diff --git a/src/internal/lexer/lexer.go b/src/internal/lexer/lexer.go index 079185a..b219d4e 100644 --- a/src/internal/lexer/lexer.go +++ b/src/internal/lexer/lexer.go @@ -178,7 +178,7 @@ func (lexer *lexer) scanToken() { case '\uFEFF': lexer.reportInvalidCharacter() case '"', '\'': - lexer.reportInvalidCharacter() + lexer.scanString(byte(char), true) default: if isDigit(byte(char)) { lexer.scanNumber() @@ -192,6 +192,227 @@ func (lexer *lexer) scanToken() { } } +func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { + openingStart := lexer.start + lexer.addToken(token.StringStart, lexer.input[lexer.start:lexer.current], nil, lexer.current) + + textStart := lexer.current + var value strings.Builder + flushText := func(end int) { + if end <= textStart { + return + } + lexer.start = textStart + lexer.addToken(token.StringText, lexer.input[textStart:end], value.String(), end) + } + + for !lexer.isAtEnd() { + char := lexer.peek() + if char == quote { + flushText(lexer.current) + lexer.start = lexer.current + lexer.current++ + lexer.addToken(token.StringEnd, lexer.input[lexer.start:lexer.current], nil, lexer.current) + return true + } + + switch char { + case '\n', '\r': + flushText(lexer.current) + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + return false + case '\\': + escapeStart := lexer.current + lexer.current++ + if lexer.isAtEnd() || lexer.peek() == '\n' || lexer.peek() == '\r' { + value.WriteByte('\\') + continue + } + + escaped := lexer.peek() + lexer.current++ + switch escaped { + case 'n': + value.WriteByte('\n') + case 't': + value.WriteByte('\t') + case '\\', '"', '\'': + value.WriteByte(escaped) + default: + lexer.report( + diagnostic.CodeInvalidEscapeSequence, + fmt.Sprintf("Invalid escape sequence: \\%c", escaped), + "Supported escapes are \\n, \\t, \\\\, \\\", and \\'.", + escapeStart, + lexer.current, + ) + value.WriteByte(escaped) + } + case '{': + if !allowInterpolation { + if lexer.peekNext() == '{' { + lexer.current += 2 + value.WriteByte('{') + continue + } + if lexer.peekNext() == '}' { + lexer.current += 2 + value.WriteString("{}") + continue + } + + lexer.report( + diagnostic.CodeInvalidCharacter, + "Nested string interpolation is not allowed.", + "Move the expression to the outer string interpolation.", + lexer.current, + lexer.current+1, + ) + lexer.current++ + value.WriteByte(char) + continue + } + if lexer.peekNext() == '{' { + lexer.current += 2 + value.WriteByte('{') + continue + } + + flushText(lexer.current) + value.Reset() + + interpolationStart := lexer.current + lexer.start = lexer.current + lexer.current++ + lexer.addToken(token.InterpStart, "{", nil, lexer.current) + if !lexer.scanInterpolation(quote, interpolationStart) { + return false + } + textStart = lexer.current + case '}': + if !allowInterpolation { + if lexer.peekNext() == '}' { + lexer.current += 2 + value.WriteByte('}') + continue + } + lexer.current++ + value.WriteByte(char) + continue + } + if lexer.peekNext() == '}' { + lexer.current += 2 + value.WriteByte('}') + continue + } + + lexer.report( + diagnostic.CodeUnescapedCloseBrace, + "Unescaped close brace in string.", + "Use }} to write a literal }.", + lexer.current, + lexer.current+1, + ) + lexer.current++ + value.WriteByte('}') + default: + decoded, size := utf8.DecodeRuneInString(lexer.input[lexer.current:]) + if decoded == utf8.RuneError && size == 1 { + flushText(lexer.current) + lexer.report(diagnostic.CodeInvalidUTF8, "File is not valid UTF-8.", "Save the file as UTF-8.", lexer.current, lexer.current+1) + lexer.current++ + textStart = lexer.current + value.Reset() + continue + } + + value.WriteString(lexer.input[lexer.current : lexer.current+size]) + lexer.current += size + } + } + + flushText(lexer.current) + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + return false +} + +func (lexer *lexer) scanInterpolation(outerQuote byte, interpolationStart int) bool { + baseBraceDepth := lexer.braceDepth + lexer.braceDepth++ + defer func() { + lexer.braceDepth = baseBraceDepth + }() + + for !lexer.isAtEnd() { + if lexer.peek() == '\n' || lexer.peek() == '\r' { + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return false + } + + if lexer.peek() == '}' && lexer.braceDepth == baseBraceDepth+1 { + if strings.TrimSpace(lexer.input[interpolationStart+1:lexer.current]) == "" { + lexer.report( + diagnostic.CodeEmptyInterpolation, + "Empty string interpolation.", + "Put an expression inside the interpolation.", + interpolationStart, + lexer.current+1, + ) + } + + lexer.start = lexer.current + lexer.current++ + lexer.addToken(token.InterpEnd, "}", nil, lexer.current) + return true + } + + if lexer.peek() == '"' || lexer.peek() == '\'' { + quote := lexer.peek() + if quote == outerQuote && (lexer.current+1 == len(lexer.input) || lexer.peekNext() == '\n' || lexer.peekNext() == '\r') { + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return true + } + lexer.start = lexer.current + lexer.current++ + if !lexer.scanString(quote, false) { + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return false + } + continue + } + + lexer.start = lexer.current + lexer.scanToken() + } + + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return false +} + func (lexer *lexer) scanComment() { lineEnd := lexer.current for lineEnd < len(lexer.input) && lexer.input[lineEnd] != '\n' && lexer.input[lineEnd] != '\r' { diff --git a/src/internal/lexer/string_test.go b/src/internal/lexer/string_test.go new file mode 100644 index 0000000..cc5d3e3 --- /dev/null +++ b/src/internal/lexer/string_test.go @@ -0,0 +1,369 @@ +package lexer + +import ( + "testing" + + "github.com/puff-lang/puff/internal/diagnostic" + "github.com/puff-lang/puff/internal/token" +) + +func TestLexSingleAndDoubleQuotedStrings(t *testing.T) { + for _, sourceText := range []string{`"hello # world"`, `'hello # world'`} { + t.Run(sourceText, func(t *testing.T) { + result := Lex(testFile(sourceText)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.StringEnd, + token.Newline, + token.EOF, + }) + if result.Tokens[1].Lexeme != "hello # world" || result.Tokens[1].Value != "hello # world" { + t.Fatalf("unexpected string text token: %#v", result.Tokens[1]) + } + }) + } +} + +func TestLexStringEscapes(t *testing.T) { + for _, sourceText := range []string{`"A\n\t\\\"\'"`, `'A\n\t\\\"\''`} { + t.Run(sourceText, func(t *testing.T) { + result := Lex(testFile(sourceText)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + if result.Tokens[1].Value != "A\n\t\\\"'" { + t.Fatalf("expected decoded escapes, got %q", result.Tokens[1].Value) + } + }) + } +} + +func TestLexStringInterpolation(t *testing.T) { + for _, sourceText := range []string{`"Coins: {$coins + 10}"`, `'Coins: {$coins + 10}'`} { + t.Run(sourceText, func(t *testing.T) { + result := Lex(testFile(sourceText)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.Dollar, + token.Ident, + token.Plus, + token.Int, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.EOF, + }) + if result.Tokens[1].Value != "Coins: " { + t.Fatalf("expected interpolation prefix, got %q", result.Tokens[1].Value) + } + }) + } +} + +func TestLexStringLiteralBraces(t *testing.T) { + result := Lex(testFile(`"Use {{player}} and }} #"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + if result.Tokens[1].Lexeme != "Use {{player}} and }} #" { + t.Fatalf("expected raw brace lexeme, got %q", result.Tokens[1].Lexeme) + } + if result.Tokens[1].Value != "Use {player} and } #" { + t.Fatalf("expected decoded literal braces, got %q", result.Tokens[1].Value) + } +} + +func TestLexStringErrors(t *testing.T) { + tests := []struct { + name string + source string + code diagnostic.Code + message string + }{ + { + name: "invalid escape", + source: `"hello\q"`, + code: diagnostic.CodeInvalidEscapeSequence, + message: `Invalid escape sequence: \q`, + }, + { + name: "unterminated at eof", + source: `"Hello`, + code: diagnostic.CodeUnterminatedString, + message: "Unterminated string.", + }, + { + name: "unterminated at newline", + source: "\"Hello\n", + code: diagnostic.CodeUnterminatedString, + message: "Unterminated string.", + }, + { + name: "unterminated interpolation", + source: `"Coins: {$coins"`, + code: diagnostic.CodeUnterminatedInterpolation, + message: "Unterminated string interpolation.", + }, + { + name: "empty interpolation", + source: `"Value: {}"`, + code: diagnostic.CodeEmptyInterpolation, + message: "Empty string interpolation.", + }, + { + name: "empty interpolation with spaces", + source: `"Value: { }"`, + code: diagnostic.CodeEmptyInterpolation, + message: "Empty string interpolation.", + }, + { + name: "unescaped close brace", + source: `"Hello }"`, + code: diagnostic.CodeUnescapedCloseBrace, + message: "Unescaped close brace in string.", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := Lex(testFile(test.source)) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{test.code}) + if result.Diagnostics[0].Message != test.message { + t.Fatalf("expected message %q, got %q", test.message, result.Diagnostics[0].Message) + } + }) + } +} + +func TestLexStringsInsideInterpolation(t *testing.T) { + result := Lex(testFile(`"Result: {format("Value } }} {} here", $coins)}"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.Ident, + token.LParen, + token.StringStart, + token.StringText, + token.StringEnd, + token.Comma, + token.Dollar, + token.Ident, + token.RParen, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.EOF, + }) + if result.Tokens[6].Value != "Value } } {} here" { + t.Fatalf("expected inner string text, got %q", result.Tokens[6].Value) + } +} + +func TestLexListInsideInterpolation(t *testing.T) { + result := Lex(testFile(`"Items: {["sword", "apple"]}"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.LBracket, + token.StringStart, + token.StringText, + token.StringEnd, + token.Comma, + token.StringStart, + token.StringText, + token.StringEnd, + token.RBracket, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.EOF, + }) +} + +func TestLexDoesNotNestInterpolationInInnerString(t *testing.T) { + result := Lex(testFile(`"Result: {format("Coins: {$coins}")}"`)) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeInvalidCharacter}) + if result.Diagnostics[0].Message != "Nested string interpolation is not allowed." { + t.Fatalf("unexpected nested interpolation message: %q", result.Diagnostics[0].Message) + } + + interpolationCount := 0 + foundInnerText := false + for _, tok := range result.Tokens { + if tok.Type == token.InterpStart { + interpolationCount++ + } + if tok.Type == token.StringText && tok.Value == "Coins: {$coins}" { + foundInnerText = true + } + } + if interpolationCount != 1 { + t.Fatalf("expected one interpolation, got %d", interpolationCount) + } + if !foundInnerText { + t.Fatal("expected nested interpolation syntax to remain inner string text") + } +} + +func TestLexStringTokenOffsets(t *testing.T) { + result := Lex(testFile(`"é\n{{x}} {$a}!"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + + prefix := result.Tokens[1] + if prefix.Lexeme != `é\n{{x}} ` || prefix.Value != "é\n{x} " { + t.Fatalf("unexpected prefix token: %#v", prefix) + } + if prefix.StartOffset != 1 || prefix.EndOffset != 11 { + t.Fatalf("expected prefix offsets 1..11, got %d..%d", prefix.StartOffset, prefix.EndOffset) + } + + if result.Tokens[2].Type != token.InterpStart || result.Tokens[2].StartOffset != 11 || result.Tokens[2].EndOffset != 12 { + t.Fatalf("unexpected interpolation start: %#v", result.Tokens[2]) + } + if result.Tokens[5].Type != token.InterpEnd || result.Tokens[5].StartOffset != 14 || result.Tokens[5].EndOffset != 15 { + t.Fatalf("unexpected interpolation end: %#v", result.Tokens[5]) + } + + suffix := result.Tokens[6] + if suffix.Lexeme != "!" || suffix.Value != "!" || suffix.StartOffset != 15 || suffix.EndOffset != 16 { + t.Fatalf("unexpected suffix token: %#v", suffix) + } + if result.Tokens[7].Type != token.StringEnd || result.Tokens[7].StartOffset != 16 || result.Tokens[7].EndOffset != 17 { + t.Fatalf("unexpected string end: %#v", result.Tokens[7]) + } +} + +func TestLexRecoversAfterUnterminatedString(t *testing.T) { + result := Lex(testFile("\"bad\n$ok = 1\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeUnterminatedString}) + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.Newline, + token.Dollar, + token.Ident, + token.Equal, + token.Int, + token.Newline, + token.EOF, + }) +} + +func TestLexRestoresStateAfterMalformedSameQuoteInterpolation(t *testing.T) { + result := Lex(testFile("\"x: {$a + \" + \"next\"\n$ok = 1\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeUnterminatedInterpolation}) + if len(result.Tokens) < 6 { + t.Fatalf("expected recovery tokens, got %v", tokenTypes(result.Tokens)) + } + + foundNextLine := false + for index, tok := range result.Tokens { + if tok.Type == token.Dollar && index+1 < len(result.Tokens) && result.Tokens[index+1].Lexeme == "ok" { + foundNextLine = true + break + } + } + if !foundNextLine { + t.Fatalf("expected lexer to resume on the next line, got %v", tokenTypes(result.Tokens)) + } +} + +func TestLexReportsUnterminatedInnerStringAndInterpolation(t *testing.T) { + result := Lex(testFile("\"x: {format('bad\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{ + diagnostic.CodeUnterminatedString, + diagnostic.CodeUnterminatedInterpolation, + }) +} + +func TestLexRecoversAfterUnterminatedInterpolation(t *testing.T) { + result := Lex(testFile("\"bad: {$value\n$ok = 1\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeUnterminatedInterpolation}) + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.Dollar, + token.Ident, + token.Newline, + token.Dollar, + token.Ident, + token.Equal, + token.Int, + token.Newline, + token.EOF, + }) +} + +func TestLexMultipleInterpolationsRestoreBraceDepth(t *testing.T) { + result := Lex(testFile("\"{$a} {$b}\"\n$items = [\n1\n]\n")) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + + interpolationCount := 0 + for _, tok := range result.Tokens { + if tok.Type == token.InterpStart { + interpolationCount++ + } + } + if interpolationCount != 2 { + t.Fatalf("expected two interpolations, got %d", interpolationCount) + } + + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.InterpStart, + token.Dollar, + token.Ident, + token.InterpEnd, + token.StringText, + token.InterpStart, + token.Dollar, + token.Ident, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.Dollar, + token.Ident, + token.Equal, + token.LBracket, + token.Int, + token.RBracket, + token.Newline, + token.EOF, + }) +}