From 18b00d1e5868ec1475439b1a82d289eba8554c15 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:41:54 -0300 Subject: [PATCH 1/8] feat(lexer): add string interpolation lexing --- src/internal/lexer/lexer.go | 237 +++++++++++++++++++++++++++++++++++- 1 file changed, 236 insertions(+), 1 deletion(-) diff --git a/src/internal/lexer/lexer.go b/src/internal/lexer/lexer.go index 079185a..1ddfec9 100644 --- a/src/internal/lexer/lexer.go +++ b/src/internal/lexer/lexer.go @@ -178,7 +178,7 @@ func (lexer *lexer) scanToken() { case '\uFEFF': lexer.reportInvalidCharacter() case '"', '\'': - lexer.reportInvalidCharacter() + lexer.scanString(byte(char), true) default: if isDigit(byte(char)) { lexer.scanNumber() @@ -192,6 +192,241 @@ func (lexer *lexer) scanToken() { } } +func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { + openingStart := lexer.start + lexer.addToken(token.StringStart, lexer.input[lexer.start:lexer.current], nil, lexer.current) + + textStart := lexer.current + var value strings.Builder + flushText := func(end int) { + if end <= textStart { + return + } + lexer.start = textStart + lexer.addToken(token.StringText, lexer.input[textStart:end], value.String(), end) + } + + for !lexer.isAtEnd() { + char := lexer.peek() + if char == quote { + flushText(lexer.current) + lexer.start = lexer.current + lexer.current++ + lexer.addToken(token.StringEnd, lexer.input[lexer.start:lexer.current], nil, lexer.current) + return true + } + + switch char { + case '\n', '\r': + flushText(lexer.current) + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + return false + case '\\': + escapeStart := lexer.current + lexer.current++ + if lexer.isAtEnd() || lexer.peek() == '\n' || lexer.peek() == '\r' { + value.WriteByte('\\') + continue + } + + escaped := lexer.peek() + lexer.current++ + switch escaped { + case 'n': + value.WriteByte('\n') + case 't': + value.WriteByte('\t') + case '\\', '"', '\'': + value.WriteByte(escaped) + default: + lexer.report( + diagnostic.CodeInvalidEscapeSequence, + fmt.Sprintf("Invalid escape sequence: \\%c", escaped), + "Supported escapes are \\n, \\t, \\\\, \\\", and \\'.", + escapeStart, + lexer.current, + ) + value.WriteByte(escaped) + } + case '{': + if !allowInterpolation { + lexer.current++ + value.WriteByte(char) + continue + } + if lexer.peekNext() == '{' { + lexer.current += 2 + value.WriteByte('{') + continue + } + + flushText(lexer.current) + value.Reset() + + interpolationStart := lexer.current + lexer.start = lexer.current + lexer.current++ + lexer.addToken(token.InterpStart, "{", nil, lexer.current) + if !lexer.scanInterpolation(quote, interpolationStart) { + return false + } + textStart = lexer.current + case '}': + if !allowInterpolation { + lexer.current++ + value.WriteByte(char) + continue + } + if lexer.peekNext() == '}' { + lexer.current += 2 + value.WriteByte('}') + continue + } + + lexer.report( + diagnostic.CodeUnescapedCloseBrace, + "Unescaped close brace in string.", + "Use }} to write a literal }.", + lexer.current, + lexer.current+1, + ) + lexer.current++ + value.WriteByte('}') + default: + decoded, size := utf8.DecodeRuneInString(lexer.input[lexer.current:]) + if decoded == utf8.RuneError && size == 1 { + flushText(lexer.current) + lexer.report(diagnostic.CodeInvalidUTF8, "File is not valid UTF-8.", "Save the file as UTF-8.", lexer.current, lexer.current+1) + lexer.current++ + textStart = lexer.current + value.Reset() + continue + } + + value.WriteString(lexer.input[lexer.current : lexer.current+size]) + lexer.current += size + } + } + + flushText(lexer.current) + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + return false +} + +func (lexer *lexer) scanInterpolation(outerQuote byte, interpolationStart int) bool { + baseBraceDepth := lexer.braceDepth + lexer.braceDepth++ + defer func() { + lexer.braceDepth = baseBraceDepth + }() + + tokenStart := len(lexer.tokens) + for !lexer.isAtEnd() { + if lexer.peek() == '}' && lexer.braceDepth == baseBraceDepth+1 { + if strings.TrimSpace(lexer.input[interpolationStart+1:lexer.current]) == "" { + lexer.report( + diagnostic.CodeEmptyInterpolation, + "Empty string interpolation.", + "Put an expression inside the interpolation.", + interpolationStart, + lexer.current+1, + ) + } + + lexer.start = lexer.current + lexer.current++ + lexer.addToken(token.InterpEnd, "}", nil, lexer.current) + return true + } + + if lexer.peek() == '"' || lexer.peek() == '\'' { + quote := lexer.peek() + if quote == outerQuote && (!canStartInnerString(lexer.tokens[tokenStart:]) || !lexer.hasClosingQuote(quote)) { + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return true + } + + lexer.start = lexer.current + lexer.current++ + if !lexer.scanString(quote, false) { + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return false + } + continue + } + + lexer.start = lexer.current + lexer.scanToken() + } + + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return false +} + +func (lexer *lexer) hasClosingQuote(quote byte) bool { + for offset := lexer.current + 1; offset < len(lexer.input); offset++ { + switch lexer.input[offset] { + case '\\': + offset++ + case '\n', '\r': + return false + case quote: + return true + } + } + return false +} + +func canStartInnerString(tokens []token.Token) bool { + if len(tokens) == 0 { + return true + } + + switch tokens[len(tokens)-1].Type { + case token.LParen, + token.LBracket, + token.LBrace, + token.Comma, + token.Colon, + token.Equal, + token.EqualEqual, + token.BangEqual, + token.Greater, + token.GreaterEq, + token.Less, + token.LessEq, + token.Plus, + token.Minus, + token.Star, + token.Slash, + token.Percent, + token.And, + token.Or, + token.Not: + return true + default: + return false + } +} + func (lexer *lexer) scanComment() { lineEnd := lexer.current for lineEnd < len(lexer.input) && lexer.input[lineEnd] != '\n' && lexer.input[lineEnd] != '\r' { From 050b11ff9b27601356310d49e2a61bbebf4395ca Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:42:12 -0300 Subject: [PATCH 2/8] test(lexer): cover string interpolation lexing --- src/internal/lexer/string_test.go | 230 ++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 src/internal/lexer/string_test.go diff --git a/src/internal/lexer/string_test.go b/src/internal/lexer/string_test.go new file mode 100644 index 0000000..1289538 --- /dev/null +++ b/src/internal/lexer/string_test.go @@ -0,0 +1,230 @@ +package lexer + +import ( + "testing" + + "github.com/puff-lang/puff/internal/diagnostic" + "github.com/puff-lang/puff/internal/token" +) + +func TestLexSingleAndDoubleQuotedStrings(t *testing.T) { + for _, sourceText := range []string{`"hello # world"`, `'hello # world'`} { + t.Run(sourceText, func(t *testing.T) { + result := Lex(testFile(sourceText)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.StringEnd, + token.Newline, + token.EOF, + }) + if result.Tokens[1].Lexeme != "hello # world" || result.Tokens[1].Value != "hello # world" { + t.Fatalf("unexpected string text token: %#v", result.Tokens[1]) + } + }) + } +} + +func TestLexStringEscapes(t *testing.T) { + for _, sourceText := range []string{`"A\n\t\\\"\'"`, `'A\n\t\\\"\''`} { + t.Run(sourceText, func(t *testing.T) { + result := Lex(testFile(sourceText)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + if result.Tokens[1].Value != "A\n\t\\\"'" { + t.Fatalf("expected decoded escapes, got %q", result.Tokens[1].Value) + } + }) + } +} + +func TestLexStringInterpolation(t *testing.T) { + for _, sourceText := range []string{`"Coins: {$coins + 10}"`, `'Coins: {$coins + 10}'`} { + t.Run(sourceText, func(t *testing.T) { + result := Lex(testFile(sourceText)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.Dollar, + token.Ident, + token.Plus, + token.Int, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.EOF, + }) + if result.Tokens[1].Value != "Coins: " { + t.Fatalf("expected interpolation prefix, got %q", result.Tokens[1].Value) + } + }) + } +} + +func TestLexStringLiteralBraces(t *testing.T) { + result := Lex(testFile(`"Use {{player}} and }} #"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + if result.Tokens[1].Lexeme != "Use {{player}} and }} #" { + t.Fatalf("expected raw brace lexeme, got %q", result.Tokens[1].Lexeme) + } + if result.Tokens[1].Value != "Use {player} and } #" { + t.Fatalf("expected decoded literal braces, got %q", result.Tokens[1].Value) + } +} + +func TestLexStringErrors(t *testing.T) { + tests := []struct { + name string + source string + code diagnostic.Code + message string + }{ + { + name: "invalid escape", + source: `"hello\q"`, + code: diagnostic.CodeInvalidEscapeSequence, + message: `Invalid escape sequence: \q`, + }, + { + name: "unterminated at eof", + source: `"Hello`, + code: diagnostic.CodeUnterminatedString, + message: "Unterminated string.", + }, + { + name: "unterminated at newline", + source: "\"Hello\n", + code: diagnostic.CodeUnterminatedString, + message: "Unterminated string.", + }, + { + name: "unterminated interpolation", + source: `"Coins: {$coins"`, + code: diagnostic.CodeUnterminatedInterpolation, + message: "Unterminated string interpolation.", + }, + { + name: "empty interpolation", + source: `"Value: {}"`, + code: diagnostic.CodeEmptyInterpolation, + message: "Empty string interpolation.", + }, + { + name: "empty interpolation with spaces", + source: `"Value: { }"`, + code: diagnostic.CodeEmptyInterpolation, + message: "Empty string interpolation.", + }, + { + name: "unescaped close brace", + source: `"Hello }"`, + code: diagnostic.CodeUnescapedCloseBrace, + message: "Unescaped close brace in string.", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := Lex(testFile(test.source)) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{test.code}) + if result.Diagnostics[0].Message != test.message { + t.Fatalf("expected message %q, got %q", test.message, result.Diagnostics[0].Message) + } + }) + } +} + +func TestLexStringsInsideInterpolation(t *testing.T) { + result := Lex(testFile(`"Result: {format("Value } here", $coins)}"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.Ident, + token.LParen, + token.StringStart, + token.StringText, + token.StringEnd, + token.Comma, + token.Dollar, + token.Ident, + token.RParen, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.EOF, + }) + if result.Tokens[6].Value != "Value } here" { + t.Fatalf("expected inner string text, got %q", result.Tokens[6].Value) + } +} + +func TestLexListInsideInterpolation(t *testing.T) { + result := Lex(testFile(`"Items: {["sword", "apple"]}"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.LBracket, + token.StringStart, + token.StringText, + token.StringEnd, + token.Comma, + token.StringStart, + token.StringText, + token.StringEnd, + token.RBracket, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.EOF, + }) +} + +func TestLexDoesNotNestInterpolationInInnerString(t *testing.T) { + result := Lex(testFile(`"Result: {format("Coins: {$coins}")}"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + + interpolationCount := 0 + foundInnerText := false + for _, tok := range result.Tokens { + if tok.Type == token.InterpStart { + interpolationCount++ + } + if tok.Type == token.StringText && tok.Value == "Coins: {$coins}" { + foundInnerText = true + } + } + if interpolationCount != 1 { + t.Fatalf("expected one interpolation, got %d", interpolationCount) + } + if !foundInnerText { + t.Fatal("expected nested interpolation syntax to remain inner string text") + } +} From 4e730337b33073c3e62e0826516596a2c976114f Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:49:03 -0300 Subject: [PATCH 3/8] fix(lexer): reject nested string interpolation --- src/internal/lexer/lexer.go | 88 +++++++++++-------------------------- 1 file changed, 26 insertions(+), 62 deletions(-) diff --git a/src/internal/lexer/lexer.go b/src/internal/lexer/lexer.go index 1ddfec9..a5bb5d5 100644 --- a/src/internal/lexer/lexer.go +++ b/src/internal/lexer/lexer.go @@ -219,7 +219,9 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { switch char { case '\n', '\r': flushText(lexer.current) - lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + if allowInterpolation { + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + } return false case '\\': escapeStart := lexer.current @@ -250,6 +252,24 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { } case '{': if !allowInterpolation { + if lexer.peekNext() == '{' { + lexer.current += 2 + value.WriteByte('{') + continue + } + if lexer.peekNext() == '}' { + lexer.current += 2 + value.WriteString("{}") + continue + } + + lexer.report( + diagnostic.CodeInvalidCharacter, + "Nested string interpolation is not allowed.", + "Move the expression to the outer string interpolation.", + lexer.current, + lexer.current+1, + ) lexer.current++ value.WriteByte(char) continue @@ -267,7 +287,7 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { lexer.start = lexer.current lexer.current++ lexer.addToken(token.InterpStart, "{", nil, lexer.current) - if !lexer.scanInterpolation(quote, interpolationStart) { + if !lexer.scanInterpolation(interpolationStart) { return false } textStart = lexer.current @@ -309,18 +329,19 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { } flushText(lexer.current) - lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + if allowInterpolation { + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) + } return false } -func (lexer *lexer) scanInterpolation(outerQuote byte, interpolationStart int) bool { +func (lexer *lexer) scanInterpolation(interpolationStart int) bool { baseBraceDepth := lexer.braceDepth lexer.braceDepth++ defer func() { lexer.braceDepth = baseBraceDepth }() - tokenStart := len(lexer.tokens) for !lexer.isAtEnd() { if lexer.peek() == '}' && lexer.braceDepth == baseBraceDepth+1 { if strings.TrimSpace(lexer.input[interpolationStart+1:lexer.current]) == "" { @@ -341,17 +362,6 @@ func (lexer *lexer) scanInterpolation(outerQuote byte, interpolationStart int) b if lexer.peek() == '"' || lexer.peek() == '\'' { quote := lexer.peek() - if quote == outerQuote && (!canStartInnerString(lexer.tokens[tokenStart:]) || !lexer.hasClosingQuote(quote)) { - lexer.report( - diagnostic.CodeUnterminatedInterpolation, - "Unterminated string interpolation.", - "Close the interpolation with }.", - interpolationStart, - lexer.current, - ) - return true - } - lexer.start = lexer.current lexer.current++ if !lexer.scanString(quote, false) { @@ -381,52 +391,6 @@ func (lexer *lexer) scanInterpolation(outerQuote byte, interpolationStart int) b return false } -func (lexer *lexer) hasClosingQuote(quote byte) bool { - for offset := lexer.current + 1; offset < len(lexer.input); offset++ { - switch lexer.input[offset] { - case '\\': - offset++ - case '\n', '\r': - return false - case quote: - return true - } - } - return false -} - -func canStartInnerString(tokens []token.Token) bool { - if len(tokens) == 0 { - return true - } - - switch tokens[len(tokens)-1].Type { - case token.LParen, - token.LBracket, - token.LBrace, - token.Comma, - token.Colon, - token.Equal, - token.EqualEqual, - token.BangEqual, - token.Greater, - token.GreaterEq, - token.Less, - token.LessEq, - token.Plus, - token.Minus, - token.Star, - token.Slash, - token.Percent, - token.And, - token.Or, - token.Not: - return true - default: - return false - } -} - func (lexer *lexer) scanComment() { lineEnd := lexer.current for lineEnd < len(lexer.input) && lexer.input[lineEnd] != '\n' && lexer.input[lineEnd] != '\r' { From 47808bf434b0f011682bec676cc8a08902f44c74 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:49:03 -0300 Subject: [PATCH 4/8] test(lexer): cover string offsets and recovery --- src/internal/lexer/string_test.go | 118 +++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 4 deletions(-) diff --git a/src/internal/lexer/string_test.go b/src/internal/lexer/string_test.go index 1289538..829244b 100644 --- a/src/internal/lexer/string_test.go +++ b/src/internal/lexer/string_test.go @@ -150,7 +150,7 @@ func TestLexStringErrors(t *testing.T) { } func TestLexStringsInsideInterpolation(t *testing.T) { - result := Lex(testFile(`"Result: {format("Value } here", $coins)}"`)) + result := Lex(testFile(`"Result: {format("Value } {} here", $coins)}"`)) if len(result.Diagnostics) != 0 { t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) @@ -173,7 +173,7 @@ func TestLexStringsInsideInterpolation(t *testing.T) { token.Newline, token.EOF, }) - if result.Tokens[6].Value != "Value } here" { + if result.Tokens[6].Value != "Value } {} here" { t.Fatalf("expected inner string text, got %q", result.Tokens[6].Value) } } @@ -207,8 +207,9 @@ func TestLexListInsideInterpolation(t *testing.T) { func TestLexDoesNotNestInterpolationInInnerString(t *testing.T) { result := Lex(testFile(`"Result: {format("Coins: {$coins}")}"`)) - if len(result.Diagnostics) != 0 { - t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeInvalidCharacter}) + if result.Diagnostics[0].Message != "Nested string interpolation is not allowed." { + t.Fatalf("unexpected nested interpolation message: %q", result.Diagnostics[0].Message) } interpolationCount := 0 @@ -228,3 +229,112 @@ func TestLexDoesNotNestInterpolationInInnerString(t *testing.T) { t.Fatal("expected nested interpolation syntax to remain inner string text") } } + +func TestLexStringTokenOffsets(t *testing.T) { + result := Lex(testFile(`"é\n{{x}} {$a}!"`)) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + + prefix := result.Tokens[1] + if prefix.Lexeme != `é\n{{x}} ` || prefix.Value != "é\n{x} " { + t.Fatalf("unexpected prefix token: %#v", prefix) + } + if prefix.StartOffset != 1 || prefix.EndOffset != 11 { + t.Fatalf("expected prefix offsets 1..11, got %d..%d", prefix.StartOffset, prefix.EndOffset) + } + + if result.Tokens[2].Type != token.InterpStart || result.Tokens[2].StartOffset != 11 || result.Tokens[2].EndOffset != 12 { + t.Fatalf("unexpected interpolation start: %#v", result.Tokens[2]) + } + if result.Tokens[5].Type != token.InterpEnd || result.Tokens[5].StartOffset != 14 || result.Tokens[5].EndOffset != 15 { + t.Fatalf("unexpected interpolation end: %#v", result.Tokens[5]) + } + + suffix := result.Tokens[6] + if suffix.Lexeme != "!" || suffix.Value != "!" || suffix.StartOffset != 15 || suffix.EndOffset != 16 { + t.Fatalf("unexpected suffix token: %#v", suffix) + } + if result.Tokens[7].Type != token.StringEnd || result.Tokens[7].StartOffset != 16 || result.Tokens[7].EndOffset != 17 { + t.Fatalf("unexpected string end: %#v", result.Tokens[7]) + } +} + +func TestLexRecoversAfterUnterminatedString(t *testing.T) { + result := Lex(testFile("\"bad\n$ok = 1\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeUnterminatedString}) + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.Newline, + token.Dollar, + token.Ident, + token.Equal, + token.Int, + token.Newline, + token.EOF, + }) +} + +func TestLexRestoresStateAfterMalformedSameQuoteInterpolation(t *testing.T) { + result := Lex(testFile("\"x: {$a + \" + \"next\"\n$ok = 1\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeUnterminatedInterpolation}) + if len(result.Tokens) < 6 { + t.Fatalf("expected recovery tokens, got %v", tokenTypes(result.Tokens)) + } + + foundNextLine := false + for index, tok := range result.Tokens { + if tok.Type == token.Dollar && index+1 < len(result.Tokens) && result.Tokens[index+1].Lexeme == "ok" { + foundNextLine = true + break + } + } + if !foundNextLine { + t.Fatalf("expected lexer to resume on the next line, got %v", tokenTypes(result.Tokens)) + } +} + +func TestLexMultipleInterpolationsRestoreBraceDepth(t *testing.T) { + result := Lex(testFile("\"{$a} {$b}\"\n$items = [\n1\n]\n")) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) + } + + interpolationCount := 0 + for _, tok := range result.Tokens { + if tok.Type == token.InterpStart { + interpolationCount++ + } + } + if interpolationCount != 2 { + t.Fatalf("expected two interpolations, got %d", interpolationCount) + } + + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.InterpStart, + token.Dollar, + token.Ident, + token.InterpEnd, + token.StringText, + token.InterpStart, + token.Dollar, + token.Ident, + token.InterpEnd, + token.StringEnd, + token.Newline, + token.Dollar, + token.Ident, + token.Equal, + token.LBracket, + token.Int, + token.RBracket, + token.Newline, + token.EOF, + }) +} From 4dbe44e6dd39783d762369e8b3b0dd639835c004 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:51:20 -0300 Subject: [PATCH 5/8] fix(lexer): stop interpolation at line breaks --- src/internal/lexer/lexer.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/internal/lexer/lexer.go b/src/internal/lexer/lexer.go index a5bb5d5..5939ab0 100644 --- a/src/internal/lexer/lexer.go +++ b/src/internal/lexer/lexer.go @@ -293,6 +293,11 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { textStart = lexer.current case '}': if !allowInterpolation { + if lexer.peekNext() == '}' { + lexer.current += 2 + value.WriteByte('}') + continue + } lexer.current++ value.WriteByte(char) continue @@ -343,6 +348,17 @@ func (lexer *lexer) scanInterpolation(interpolationStart int) bool { }() for !lexer.isAtEnd() { + if lexer.peek() == '\n' || lexer.peek() == '\r' { + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return false + } + if lexer.peek() == '}' && lexer.braceDepth == baseBraceDepth+1 { if strings.TrimSpace(lexer.input[interpolationStart+1:lexer.current]) == "" { lexer.report( From e6b452f64c67457b7755327b6b957385cad676d2 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:51:21 -0300 Subject: [PATCH 6/8] test(lexer): cover interpolation recovery --- src/internal/lexer/string_test.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/internal/lexer/string_test.go b/src/internal/lexer/string_test.go index 829244b..81cb518 100644 --- a/src/internal/lexer/string_test.go +++ b/src/internal/lexer/string_test.go @@ -150,7 +150,7 @@ func TestLexStringErrors(t *testing.T) { } func TestLexStringsInsideInterpolation(t *testing.T) { - result := Lex(testFile(`"Result: {format("Value } {} here", $coins)}"`)) + result := Lex(testFile(`"Result: {format("Value } }} {} here", $coins)}"`)) if len(result.Diagnostics) != 0 { t.Fatalf("expected no diagnostics, got %v", result.Diagnostics) @@ -173,7 +173,7 @@ func TestLexStringsInsideInterpolation(t *testing.T) { token.Newline, token.EOF, }) - if result.Tokens[6].Value != "Value } {} here" { + if result.Tokens[6].Value != "Value } } {} here" { t.Fatalf("expected inner string text, got %q", result.Tokens[6].Value) } } @@ -298,6 +298,26 @@ func TestLexRestoresStateAfterMalformedSameQuoteInterpolation(t *testing.T) { } } +func TestLexRecoversAfterUnterminatedInterpolation(t *testing.T) { + result := Lex(testFile("\"bad: {$value\n$ok = 1\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{diagnostic.CodeUnterminatedInterpolation}) + assertTokenTypes(t, result.Tokens, []token.Type{ + token.StringStart, + token.StringText, + token.InterpStart, + token.Dollar, + token.Ident, + token.Newline, + token.Dollar, + token.Ident, + token.Equal, + token.Int, + token.Newline, + token.EOF, + }) +} + func TestLexMultipleInterpolationsRestoreBraceDepth(t *testing.T) { result := Lex(testFile("\"{$a} {$b}\"\n$items = [\n1\n]\n")) From cc3d45b15cad11ecf216a4d52225eee5999bd4ad Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:57:25 -0300 Subject: [PATCH 7/8] fix(lexer): report unterminated inner strings --- src/internal/lexer/lexer.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/internal/lexer/lexer.go b/src/internal/lexer/lexer.go index 5939ab0..b219d4e 100644 --- a/src/internal/lexer/lexer.go +++ b/src/internal/lexer/lexer.go @@ -219,9 +219,7 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { switch char { case '\n', '\r': flushText(lexer.current) - if allowInterpolation { - lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) - } + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) return false case '\\': escapeStart := lexer.current @@ -287,7 +285,7 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { lexer.start = lexer.current lexer.current++ lexer.addToken(token.InterpStart, "{", nil, lexer.current) - if !lexer.scanInterpolation(interpolationStart) { + if !lexer.scanInterpolation(quote, interpolationStart) { return false } textStart = lexer.current @@ -334,13 +332,11 @@ func (lexer *lexer) scanString(quote byte, allowInterpolation bool) bool { } flushText(lexer.current) - if allowInterpolation { - lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) - } + lexer.report(diagnostic.CodeUnterminatedString, "Unterminated string.", "Close the string or use \\n for line breaks.", openingStart, lexer.current) return false } -func (lexer *lexer) scanInterpolation(interpolationStart int) bool { +func (lexer *lexer) scanInterpolation(outerQuote byte, interpolationStart int) bool { baseBraceDepth := lexer.braceDepth lexer.braceDepth++ defer func() { @@ -378,6 +374,16 @@ func (lexer *lexer) scanInterpolation(interpolationStart int) bool { if lexer.peek() == '"' || lexer.peek() == '\'' { quote := lexer.peek() + if quote == outerQuote && (lexer.current+1 == len(lexer.input) || lexer.peekNext() == '\n' || lexer.peekNext() == '\r') { + lexer.report( + diagnostic.CodeUnterminatedInterpolation, + "Unterminated string interpolation.", + "Close the interpolation with }.", + interpolationStart, + lexer.current, + ) + return true + } lexer.start = lexer.current lexer.current++ if !lexer.scanString(quote, false) { From 6ca4aaeb8dfc4a6f19cad38b0dd2783c93640e8d Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Sun, 26 Jul 2026 11:57:25 -0300 Subject: [PATCH 8/8] test(lexer): require inner string diagnostics --- src/internal/lexer/string_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/internal/lexer/string_test.go b/src/internal/lexer/string_test.go index 81cb518..cc5d3e3 100644 --- a/src/internal/lexer/string_test.go +++ b/src/internal/lexer/string_test.go @@ -298,6 +298,15 @@ func TestLexRestoresStateAfterMalformedSameQuoteInterpolation(t *testing.T) { } } +func TestLexReportsUnterminatedInnerStringAndInterpolation(t *testing.T) { + result := Lex(testFile("\"x: {format('bad\n")) + + assertDiagnosticCodes(t, result.Diagnostics, []diagnostic.Code{ + diagnostic.CodeUnterminatedString, + diagnostic.CodeUnterminatedInterpolation, + }) +} + func TestLexRecoversAfterUnterminatedInterpolation(t *testing.T) { result := Lex(testFile("\"bad: {$value\n$ok = 1\n"))