From dc65f95aca221aefa5df5b6b855a645284f57ea6 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Mon, 27 Jul 2026 11:28:01 -0300 Subject: [PATCH 1/5] feat(parser): parse top-level declarations --- src/internal/parser/parser.go | 218 +++++++++++++++ src/internal/parser/top_level.go | 446 +++++++++++++++++++++++++++++++ 2 files changed, 664 insertions(+) create mode 100644 src/internal/parser/parser.go create mode 100644 src/internal/parser/top_level.go diff --git a/src/internal/parser/parser.go b/src/internal/parser/parser.go new file mode 100644 index 0000000..6eca471 --- /dev/null +++ b/src/internal/parser/parser.go @@ -0,0 +1,218 @@ +package parser + +import ( + "fmt" + "strings" + + "github.com/puff-lang/puff/internal/ast" + "github.com/puff-lang/puff/internal/diagnostic" + "github.com/puff-lang/puff/internal/lexer" + "github.com/puff-lang/puff/internal/source" + "github.com/puff-lang/puff/internal/token" +) + +type Result struct { + File *ast.File + Diagnostics []diagnostic.Diagnostic +} + +type parser struct { + file source.File + tokens []token.Token + current int + diagnostics []diagnostic.Diagnostic +} + +func Parse(file source.File, lexed lexer.Result) Result { + parser := &parser{ + file: file, + tokens: lexed.Tokens, + diagnostics: append([]diagnostic.Diagnostic(nil), lexed.Diagnostics...), + } + + return Result{ + File: parser.parseFile(lexed.Metadata), + Diagnostics: parser.diagnostics, + } +} + +func (parser *parser) parseFile(metadata lexer.Metadata) *ast.File { + file := &ast.File{ + NodeBase: ast.NodeBase{SourceSpan: parser.spanFromOffsets(0, len(parser.file.Text))}, + Metadata: metadataEntries(metadata), + } + + parser.skipNewlines() + seenDeclaration := false + for !parser.atEnd() { + switch { + case parser.check(token.Require) && !seenDeclaration: + file.Requirements = append(file.Requirements, parser.parseRequire()) + case parser.check(token.Require): + parser.reportUnexpected(parser.peek(), "") + parser.synchronizeLine() + case parser.check(token.Fun): + seenDeclaration = true + file.Declarations = append(file.Declarations, parser.parseFunction(false)) + case parser.check(token.On): + seenDeclaration = true + file.Declarations = append(file.Declarations, parser.parseEvent()) + case parser.check(token.Dollar): + seenDeclaration = true + file.Declarations = append(file.Declarations, parser.parseGlobal(false)) + case parser.check(token.Pub): + seenDeclaration = true + if parser.peekNext().Type == token.Fun { + file.Declarations = append(file.Declarations, parser.parseFunction(true)) + } else if parser.peekNext().Type == token.Dollar { + file.Declarations = append(file.Declarations, parser.parseGlobal(true)) + } else { + parser.reportUnexpected(parser.peek(), "") + parser.synchronizeLine() + } + case parser.check(token.Else), parser.check(token.End): + hint := "" + if parser.check(token.Else) { + hint = "else can only appear inside an if block." + } + parser.reportUnexpected(parser.peek(), hint) + parser.synchronizeLine() + default: + parser.reportInvalidTopLevel() + parser.synchronizeLine() + } + parser.skipNewlines() + } + + return file +} + +func metadataEntries(metadata lexer.Metadata) []ast.MetadataEntry { + entries := make([]ast.MetadataEntry, 0, 2) + if metadata.Namespace != "" { + entries = append(entries, ast.MetadataEntry{Key: "namespace", Value: metadata.Namespace}) + } + if len(metadata.Tags) > 0 { + entries = append(entries, ast.MetadataEntry{Key: "tags", Value: strings.Join(metadata.Tags, ", ")}) + } + return entries +} + +func (parser *parser) reportExpected(expected string, hint string) { + tok := parser.peek() + parser.report( + diagnostic.CodeExpectedToken, + fmt.Sprintf("Expected %q.", expected), + hint, + tok.StartOffset, + tok.EndOffset, + ) +} + +func (parser *parser) reportUnexpected(tok token.Token, hint string) { + parser.report( + diagnostic.CodeUnexpectedToken, + fmt.Sprintf("Unexpected token: %s", tok.Lexeme), + hint, + tok.StartOffset, + tok.EndOffset, + ) +} + +func (parser *parser) reportInvalidTopLevel() { + start := parser.peek().StartOffset + end := start + for !parser.check(token.Newline) && !parser.atEnd() { + end = parser.advance().EndOffset + } + parser.report( + diagnostic.CodeInvalidTopLevelStatement, + "Executable statements are not allowed at the top level.", + "Move this statement into an event or function.", + start, + end, + ) +} + +func (parser *parser) report(code diagnostic.Code, message string, hint string, start int, end int) { + parser.diagnostics = append(parser.diagnostics, diagnostic.Diagnostic{ + Code: code, + Phase: diagnostic.PhaseParser, + Severity: diagnostic.SeverityError, + Message: message, + Hint: hint, + File: parser.file.RelPath, + Span: parser.spanFromOffsets(start, end), + }) +} + +func (parser *parser) spanFromOffsets(start int, end int) diagnostic.Span { + startLine, startColumn, _ := parser.file.Map.LineColumn(start) + endLine, endColumn, _ := parser.file.Map.LineColumn(end) + return diagnostic.Span{ + StartLine: startLine, + StartColumn: startColumn, + EndLine: endLine, + EndColumn: endColumn, + StartOffset: start, + EndOffset: end, + } +} + +func (parser *parser) base(start int, end int) ast.NodeBase { + return ast.NodeBase{SourceSpan: parser.spanFromOffsets(start, end)} +} + +func (parser *parser) skipNewlines() { + for parser.match(token.Newline) { + } +} + +func (parser *parser) synchronizeLine() { + for !parser.check(token.Newline) && !parser.atEnd() { + parser.advance() + } +} + +func (parser *parser) match(types ...token.Type) bool { + for _, tokenType := range types { + if parser.check(tokenType) { + parser.advance() + return true + } + } + return false +} + +func (parser *parser) check(tokenType token.Type) bool { + return parser.peek().Type == tokenType +} + +func (parser *parser) advance() token.Token { + if !parser.atEnd() { + parser.current++ + } + return parser.tokens[parser.current-1] +} + +func (parser *parser) peek() token.Token { + if len(parser.tokens) == 0 { + return token.Token{Type: token.EOF} + } + return parser.tokens[parser.current] +} + +func (parser *parser) peekNext() token.Token { + if parser.current+1 >= len(parser.tokens) { + return parser.peek() + } + return parser.tokens[parser.current+1] +} + +func (parser *parser) previous() token.Token { + return parser.tokens[parser.current-1] +} + +func (parser *parser) atEnd() bool { + return parser.peek().Type == token.EOF +} diff --git a/src/internal/parser/top_level.go b/src/internal/parser/top_level.go new file mode 100644 index 0000000..b7b35d0 --- /dev/null +++ b/src/internal/parser/top_level.go @@ -0,0 +1,446 @@ +package parser + +import ( + "strings" + + "github.com/puff-lang/puff/internal/ast" + "github.com/puff-lang/puff/internal/diagnostic" + "github.com/puff-lang/puff/internal/token" +) + +func (parser *parser) parseRequire() *ast.RequireDecl { + start := parser.advance().StartOffset + path := parser.parseString() + + var alias *ast.Identifier + if parser.match(token.As) { + if parser.check(token.Ident) { + alias = parser.parseIdentifier() + } else { + parser.reportExpected("identifier", "") + } + } + + end := parser.lineEndOffset(start) + parser.requireLineEnd() + return &ast.RequireDecl{ + NodeBase: parser.base(start, end), + Path: path, + Alias: alias, + } +} + +func (parser *parser) parseFunction(public bool) *ast.FunctionDecl { + start := parser.peek().StartOffset + if public { + parser.advance() + } + parser.advance() + + name := ast.Identifier{} + if parser.check(token.Ident) { + name = *parser.parseIdentifier() + } else { + parser.reportExpected("function name", "") + } + + var parameters []ast.Parameter + if parser.match(token.LParen) { + parameters = parser.parseParameters() + } + + var returnType *ast.TypeRef + if parser.match(token.Arrow) { + returnType = parser.parseType() + } + + parser.requireLineEnd() + body, end := parser.scanBlock(start) + return &ast.FunctionDecl{ + NodeBase: parser.base(start, end), + Public: public, + Name: name, + Parameters: parameters, + ReturnType: returnType, + Body: body, + } +} + +func (parser *parser) parseParameters() []ast.Parameter { + var parameters []ast.Parameter + for !parser.check(token.RParen) && !parser.check(token.Arrow) && !parser.check(token.Newline) && !parser.atEnd() { + start := parser.peek().StartOffset + if !parser.check(token.Ident) { + parser.reportExpected("parameter name", "") + parser.synchronizeUntil(token.Comma, token.RParen, token.Newline) + } else { + name := parser.parseIdentifier() + var parameterType *ast.TypeRef + if parser.match(token.Colon) { + parameterType = parser.parseType() + } + end := parser.previous().EndOffset + parameters = append(parameters, ast.Parameter{ + NodeBase: parser.base(start, end), + Name: *name, + Type: parameterType, + }) + } + + if !parser.match(token.Comma) { + break + } + } + + if !parser.match(token.RParen) { + parser.reportExpected(")", "Close the parameter list before the return type.") + } + return parameters +} + +func (parser *parser) parseType() *ast.TypeRef { + start := parser.peek().StartOffset + if !parser.check(token.Ident) { + parser.reportExpected("type", "") + return nil + } + + name := parser.parseIdentifier() + typeRef := &ast.TypeRef{Name: *name} + if parser.match(token.Less) { + for !parser.check(token.Greater) && !parser.check(token.Newline) && !parser.atEnd() { + argument := parser.parseType() + if argument != nil { + typeRef.Arguments = append(typeRef.Arguments, argument) + } + if !parser.match(token.Comma) { + break + } + } + if !parser.match(token.Greater) { + parser.reportExpected(">", "") + } + } + + typeRef.NodeBase = parser.base(start, parser.previous().EndOffset) + return typeRef +} + +func (parser *parser) parseEvent() *ast.EventDecl { + start := parser.advance().StartOffset + var name []ast.Identifier + for parser.check(token.Ident) { + name = append(name, *parser.parseIdentifier()) + } + if len(name) == 0 { + parser.reportExpected("event name", "") + } + + parser.requireLineEnd() + body, end := parser.scanBlock(start) + return &ast.EventDecl{ + NodeBase: parser.base(start, end), + Name: name, + Body: body, + } +} + +func (parser *parser) parseGlobal(public bool) *ast.GlobalAssignment { + start := parser.peek().StartOffset + if public { + parser.advance() + } + target := parser.parseGlobalVariable() + if !parser.match(token.Equal) { + parser.reportExpected("=", "") + } + + expressionStart := parser.current + end := parser.lineEndOffset(start) + for !parser.check(token.Newline) && !parser.atEnd() { + parser.advance() + } + var value ast.Expression + if expressionStart < parser.current { + value = parser.parseDeferredExpression(parser.tokens[expressionStart:parser.current]) + } + parser.requireLineEnd() + + return &ast.GlobalAssignment{ + NodeBase: parser.base(start, end), + Public: public, + Target: target, + Value: value, + } +} + +func (parser *parser) parseGlobalVariable() *ast.VariableExpr { + start := parser.peek().StartOffset + if !parser.match(token.Dollar) { + parser.reportExpected("$", "") + return nil + } + if !parser.check(token.Ident) { + parser.reportExpected("global variable name", "") + return nil + } + + name := parser.parseIdentifier() + var accesses []ast.VariableAccess + for { + switch { + case parser.match(token.Dot): + if !parser.check(token.Ident) { + parser.reportExpected("field name", "") + return &ast.VariableExpr{NodeBase: parser.base(start, parser.previous().EndOffset), Name: *name, Accesses: accesses} + } + field := parser.parseIdentifier() + accesses = append(accesses, &ast.FieldAccess{ + NodeBase: parser.base(parser.tokens[parser.current-2].StartOffset, field.Span().EndOffset), + Field: *field, + }) + case parser.match(token.LBracket): + accessStart := parser.previous().StartOffset + if parser.match(token.RBracket) { + accesses = append(accesses, &ast.EmptyIndexAccess{NodeBase: parser.base(accessStart, parser.previous().EndOffset)}) + continue + } + indexStart := parser.current + for !parser.check(token.RBracket) && !parser.check(token.Newline) && !parser.atEnd() { + parser.advance() + } + var index ast.Expression + if indexStart < parser.current { + index = parser.parseDeferredExpression(parser.tokens[indexStart:parser.current]) + } + if !parser.match(token.RBracket) { + parser.reportExpected("]", "") + } + accesses = append(accesses, &ast.IndexAccess{ + NodeBase: parser.base(accessStart, parser.previous().EndOffset), + Index: index, + }) + default: + end := name.Span().EndOffset + if len(accesses) > 0 { + end = accesses[len(accesses)-1].Span().EndOffset + } + return &ast.VariableExpr{ + NodeBase: parser.base(start, end), + Name: *name, + Accesses: accesses, + } + } + } +} + +func (parser *parser) parseDeferredExpression(tokens []token.Token) ast.Expression { + start := tokens[0].StartOffset + end := tokens[len(tokens)-1].EndOffset + base := parser.base(start, end) + + if len(tokens) == 1 { + switch tokens[0].Type { + case token.Nil: + return &ast.NilLiteral{NodeBase: base} + case token.True: + return &ast.BoolLiteral{NodeBase: base, Value: true} + case token.False: + return &ast.BoolLiteral{NodeBase: base, Value: false} + case token.Int: + value, _ := tokens[0].Value.(int) + return &ast.IntLiteral{NodeBase: base, Value: int64(value)} + case token.Float: + value, _ := tokens[0].Value.(float64) + return &ast.FloatLiteral{NodeBase: base, Value: value} + } + } + if tokens[0].Type == token.StringStart && tokens[len(tokens)-1].Type == token.StringEnd { + return parser.stringFromTokens(tokens) + } + if len(tokens) == 2 && tokens[0].Type == token.LBracket && tokens[1].Type == token.RBracket { + return &ast.ListExpr{NodeBase: base} + } + + return &ast.PatternExpr{NodeBase: base, Tokens: append([]token.Token(nil), tokens...)} +} + +func (parser *parser) parseString() *ast.StringExpr { + if !parser.check(token.StringStart) { + parser.reportExpected("string", "") + return nil + } + + start := parser.current + parser.advance() + depth := 0 + for !parser.atEnd() { + switch parser.peek().Type { + case token.InterpStart: + depth++ + case token.InterpEnd: + depth-- + case token.StringEnd: + if depth == 0 { + parser.advance() + return parser.stringFromTokens(parser.tokens[start:parser.current]) + } + case token.Newline: + parser.reportExpected("closing quote", "") + return parser.stringFromTokens(parser.tokens[start:parser.current]) + } + parser.advance() + } + parser.reportExpected("closing quote", "") + return parser.stringFromTokens(parser.tokens[start:parser.current]) +} + +func (parser *parser) stringFromTokens(tokens []token.Token) *ast.StringExpr { + if len(tokens) == 0 { + return nil + } + + expression := &ast.StringExpr{ + NodeBase: parser.base(tokens[0].StartOffset, tokens[len(tokens)-1].EndOffset), + } + if len(tokens[0].Lexeme) > 0 { + expression.Quote = tokens[0].Lexeme[0] + } + + for index := 1; index < len(tokens)-1; index++ { + tok := tokens[index] + switch tok.Type { + case token.StringText: + value, _ := tok.Value.(string) + expression.Parts = append(expression.Parts, &ast.StringText{ + NodeBase: parser.base(tok.StartOffset, tok.EndOffset), + Raw: tok.Lexeme, + Value: value, + }) + case token.InterpStart: + interpolationStart := index + 1 + for index+1 < len(tokens)-1 && tokens[index+1].Type != token.InterpEnd { + index++ + } + parts := tokens[interpolationStart : index+1] + var value ast.Expression + if len(parts) > 0 { + value = parser.parseDeferredExpression(parts) + } + end := tok.EndOffset + if index+1 < len(tokens) && tokens[index+1].Type == token.InterpEnd { + index++ + end = tokens[index].EndOffset + } + expression.Parts = append(expression.Parts, &ast.StringInterpolation{ + NodeBase: parser.base(tok.StartOffset, end), + Expression: value, + }) + } + } + + return expression +} + +func (parser *parser) parseIdentifier() *ast.Identifier { + tok := parser.advance() + return &ast.Identifier{ + NodeBase: parser.base(tok.StartOffset, tok.EndOffset), + Name: tok.Lexeme, + } +} + +func (parser *parser) scanBlock(openingStart int) (ast.Block, int) { + bodyStart := parser.peek().StartOffset + depth := 0 + lineStart := true + for !parser.atEnd() { + tok := parser.peek() + if lineStart { + switch tok.Type { + case token.If, token.Loop: + depth++ + case token.Else: + if depth == 0 { + parser.reportUnexpected(tok, "else can only appear inside an if block.") + } + case token.End: + if depth == 0 { + endToken := parser.advance() + parser.requireLineEnd() + return ast.Block{NodeBase: parser.base(bodyStart, endToken.StartOffset)}, endToken.EndOffset + } + depth-- + } + lineStart = false + } + + if parser.match(token.Newline) { + lineStart = true + continue + } + parser.advance() + } + + eof := parser.peek() + parser.report( + diagnostic.CodeExpectedEnd, + `Expected "end" before end of file.`, + "Add end to close the block.", + eof.StartOffset, + eof.EndOffset, + ) + return ast.Block{NodeBase: parser.base(bodyStart, eof.StartOffset)}, eof.EndOffset +} + +func (parser *parser) requireLineEnd() { + if parser.match(token.Newline) || parser.atEnd() { + return + } + tok := parser.peek() + parser.report( + diagnostic.CodeExpectedNewline, + "Expected newline.", + "", + tok.StartOffset, + tok.EndOffset, + ) + parser.synchronizeLine() + parser.match(token.Newline) +} + +func (parser *parser) lineEndOffset(fallback int) int { + if parser.atEnd() || parser.check(token.Newline) { + if parser.current == 0 { + return fallback + } + return parser.previous().EndOffset + } + index := parser.current + for index < len(parser.tokens) && parser.tokens[index].Type != token.Newline && parser.tokens[index].Type != token.EOF { + index++ + } + if index == parser.current { + return fallback + } + return parser.tokens[index-1].EndOffset +} + +func (parser *parser) synchronizeUntil(types ...token.Type) { + for !parser.atEnd() { + for _, tokenType := range types { + if parser.check(tokenType) { + return + } + } + parser.advance() + } +} + +func tokenText(tokens []token.Token) string { + var builder strings.Builder + for _, tok := range tokens { + builder.WriteString(tok.Lexeme) + } + return builder.String() +} From 3f3e185d2fecec115bcd688b999f61ee358d45a6 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Mon, 27 Jul 2026 11:32:00 -0300 Subject: [PATCH 2/5] fix(parser): handle contextual names and recovery --- src/internal/parser/top_level.go | 60 +++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/internal/parser/top_level.go b/src/internal/parser/top_level.go index b7b35d0..25c219e 100644 --- a/src/internal/parser/top_level.go +++ b/src/internal/parser/top_level.go @@ -1,8 +1,6 @@ package parser import ( - "strings" - "github.com/puff-lang/puff/internal/ast" "github.com/puff-lang/puff/internal/diagnostic" "github.com/puff-lang/puff/internal/token" @@ -14,7 +12,7 @@ func (parser *parser) parseRequire() *ast.RequireDecl { var alias *ast.Identifier if parser.match(token.As) { - if parser.check(token.Ident) { + if parser.checkName() { alias = parser.parseIdentifier() } else { parser.reportExpected("identifier", "") @@ -38,7 +36,7 @@ func (parser *parser) parseFunction(public bool) *ast.FunctionDecl { parser.advance() name := ast.Identifier{} - if parser.check(token.Ident) { + if parser.checkName() { name = *parser.parseIdentifier() } else { parser.reportExpected("function name", "") @@ -70,7 +68,7 @@ func (parser *parser) parseParameters() []ast.Parameter { var parameters []ast.Parameter for !parser.check(token.RParen) && !parser.check(token.Arrow) && !parser.check(token.Newline) && !parser.atEnd() { start := parser.peek().StartOffset - if !parser.check(token.Ident) { + if !parser.checkName() { parser.reportExpected("parameter name", "") parser.synchronizeUntil(token.Comma, token.RParen, token.Newline) } else { @@ -100,7 +98,7 @@ func (parser *parser) parseParameters() []ast.Parameter { func (parser *parser) parseType() *ast.TypeRef { start := parser.peek().StartOffset - if !parser.check(token.Ident) { + if !parser.checkName() { parser.reportExpected("type", "") return nil } @@ -129,7 +127,7 @@ func (parser *parser) parseType() *ast.TypeRef { func (parser *parser) parseEvent() *ast.EventDecl { start := parser.advance().StartOffset var name []ast.Identifier - for parser.check(token.Ident) { + for parser.checkName() { name = append(name, *parser.parseIdentifier()) } if len(name) == 0 { @@ -180,7 +178,7 @@ func (parser *parser) parseGlobalVariable() *ast.VariableExpr { parser.reportExpected("$", "") return nil } - if !parser.check(token.Ident) { + if !parser.checkName() { parser.reportExpected("global variable name", "") return nil } @@ -190,7 +188,7 @@ func (parser *parser) parseGlobalVariable() *ast.VariableExpr { for { switch { case parser.match(token.Dot): - if !parser.check(token.Ident) { + if !parser.checkName() { parser.reportExpected("field name", "") return &ast.VariableExpr{NodeBase: parser.base(start, parser.previous().EndOffset), Name: *name, Accesses: accesses} } @@ -350,6 +348,39 @@ func (parser *parser) parseIdentifier() *ast.Identifier { } } +func (parser *parser) checkName() bool { + switch parser.peek().Type { + case token.Ident, + token.Require, + token.As, + token.Pub, + token.Fun, + token.On, + token.End, + token.If, + token.Else, + token.Loop, + token.Times, + token.Numbers, + token.Players, + token.Entities, + token.From, + token.To, + token.In, + token.Radius, + token.Around, + token.Return, + token.Stop, + token.Add, + token.And, + token.Or, + token.Not: + return true + default: + return false + } +} + func (parser *parser) scanBlock(openingStart int) (ast.Block, int) { bodyStart := parser.peek().StartOffset depth := 0 @@ -397,6 +428,9 @@ func (parser *parser) requireLineEnd() { if parser.match(token.Newline) || parser.atEnd() { return } + if parser.current > 0 && parser.peek().Line > parser.previous().Line { + return + } tok := parser.peek() parser.report( diagnostic.CodeExpectedNewline, @@ -436,11 +470,3 @@ func (parser *parser) synchronizeUntil(types ...token.Type) { parser.advance() } } - -func tokenText(tokens []token.Token) string { - var builder strings.Builder - for _, tok := range tokens { - builder.WriteString(tok.Lexeme) - } - return builder.String() -} From 1763547dbf2a78df352f2bc59dd34e61f242bd40 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Mon, 27 Jul 2026 11:32:00 -0300 Subject: [PATCH 3/5] test(parser): cover top-level parsing --- src/internal/parser/testdata/top_level.golden | 12 + src/internal/parser/testdata/top_level.puff | 22 + src/internal/parser/top_level_test.go | 379 ++++++++++++++++++ 3 files changed, 413 insertions(+) create mode 100644 src/internal/parser/testdata/top_level.golden create mode 100644 src/internal/parser/testdata/top_level.puff create mode 100644 src/internal/parser/top_level_test.go diff --git a/src/internal/parser/testdata/top_level.golden b/src/internal/parser/testdata/top_level.golden new file mode 100644 index 0000000..c0d9317 --- /dev/null +++ b/src/internal/parser/testdata/top_level.golden @@ -0,0 +1,12 @@ +metadata namespace="example" +metadata tags="load, tick" +require "abc/shop" +require "github.com/123/123" as lib123 +global $coins = int(100) +global $shop.name = string("Main Shop") +global $players[] = list() +pub global $tax = float(0.1) +fun setup() +pub fun finalPrice(price: float) -> float +event load +event scoreboard update diff --git a/src/internal/parser/testdata/top_level.puff b/src/internal/parser/testdata/top_level.puff new file mode 100644 index 0000000..36a7c81 --- /dev/null +++ b/src/internal/parser/testdata/top_level.puff @@ -0,0 +1,22 @@ +# namespace: example +# tags: load, tick + +require "abc/shop" +require "github.com/123/123" as lib123 + +$coins = 100 +$shop.name = "Main Shop" +$players[] = [] +pub $tax = 0.1 + +fun setup +end + +pub fun finalPrice(price: float) -> float +end + +on load +end + +on scoreboard update +end diff --git a/src/internal/parser/top_level_test.go b/src/internal/parser/top_level_test.go new file mode 100644 index 0000000..ca8b94c --- /dev/null +++ b/src/internal/parser/top_level_test.go @@ -0,0 +1,379 @@ +package parser + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/puff-lang/puff/internal/ast" + "github.com/puff-lang/puff/internal/diagnostic" + "github.com/puff-lang/puff/internal/lexer" + "github.com/puff-lang/puff/internal/source" + "github.com/puff-lang/puff/internal/token" +) + +func TestParseTopLevelGolden(t *testing.T) { + sourcePath := filepath.Join("testdata", "top_level.puff") + input, err := os.ReadFile(sourcePath) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + want, err := os.ReadFile(filepath.Join("testdata", "top_level.golden")) + if err != nil { + t.Fatalf("read golden: %v", err) + } + + result := parseTestSource(sourcePath, string(input)) + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %#v", result.Diagnostics) + } + if got := renderFile(result.File); got != string(want) { + t.Fatalf("unexpected AST\nwant:\n%s\ngot:\n%s", want, got) + } +} + +func TestParseFunctionSignaturesAndNestedTypes(t *testing.T) { + result := parseTestSource("functions.puff", ` +fun noParams +end +fun explicit() +end +pub fun transform(value, items: map>) -> list +end +`) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %#v", result.Diagnostics) + } + if len(result.File.Declarations) != 3 { + t.Fatalf("expected three functions, got %d", len(result.File.Declarations)) + } + + function := result.File.Declarations[2].(*ast.FunctionDecl) + if !function.Public || function.Name.Name != "transform" || len(function.Parameters) != 2 { + t.Fatalf("unexpected function: %#v", function) + } + if function.Parameters[0].Name.Name != "value" || function.Parameters[0].Type != nil { + t.Fatalf("unexpected untyped parameter: %#v", function.Parameters[0]) + } + mapType := function.Parameters[1].Type + if mapType.Name.Name != "map" || len(mapType.Arguments) != 2 { + t.Fatalf("unexpected map type: %#v", mapType) + } + listType := mapType.Arguments[1] + if listType.Name.Name != "list" || len(listType.Arguments) != 1 || listType.Arguments[0].Name.Name != "int" { + t.Fatalf("unexpected nested list type: %#v", listType) + } + if function.ReturnType.Name.Name != "list" || function.ReturnType.Arguments[0].Name.Name != "string" { + t.Fatalf("unexpected return type: %#v", function.ReturnType) + } +} + +func TestParseGlobalsPreservesTargetsAndSimpleValues(t *testing.T) { + result := parseTestSource("globals.puff", ` +$shop.name = "Main Shop" +$players[] = [] +$stats[$key] = nil +pub $tax = 0.1 +`) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %#v", result.Diagnostics) + } + if len(result.File.Declarations) != 4 { + t.Fatalf("expected four globals, got %d", len(result.File.Declarations)) + } + + shop := result.File.Declarations[0].(*ast.GlobalAssignment) + if shop.Target.Name.Name != "shop" || shop.Target.Accesses[0].(*ast.FieldAccess).Field.Name != "name" { + t.Fatalf("unexpected field target: %#v", shop.Target) + } + shopName := shop.Value.(*ast.StringExpr) + if shopName.Parts[0].(*ast.StringText).Value != "Main Shop" { + t.Fatalf("unexpected string value: %#v", shopName) + } + + players := result.File.Declarations[1].(*ast.GlobalAssignment) + if _, ok := players.Target.Accesses[0].(*ast.EmptyIndexAccess); !ok { + t.Fatalf("expected empty index access, got %T", players.Target.Accesses[0]) + } + if _, ok := players.Value.(*ast.ListExpr); !ok { + t.Fatalf("expected empty list, got %T", players.Value) + } + + stats := result.File.Declarations[2].(*ast.GlobalAssignment) + index := stats.Target.Accesses[0].(*ast.IndexAccess).Index.(*ast.PatternExpr) + if len(index.Tokens) != 2 || index.Tokens[0].Type != token.Dollar || index.Tokens[1].Lexeme != "key" { + t.Fatalf("unexpected index expression: %#v", index) + } + if _, ok := stats.Value.(*ast.NilLiteral); !ok { + t.Fatalf("expected nil value, got %T", stats.Value) + } + + tax := result.File.Declarations[3].(*ast.GlobalAssignment) + if !tax.Public || tax.Value.(*ast.FloatLiteral).Value != 0.1 { + t.Fatalf("unexpected public global: %#v", tax) + } +} + +func TestParseEventsAndBalancedBodies(t *testing.T) { + result := parseTestSource("events.puff", ` +fun nested +if true +end +end +on scoreboard update +end +`) + + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %#v", result.Diagnostics) + } + if len(result.File.Declarations) != 2 { + t.Fatalf("expected function and event, got %d declarations", len(result.File.Declarations)) + } + event := result.File.Declarations[1].(*ast.EventDecl) + if len(event.Name) != 2 || event.Name[0].Name != "scoreboard" || event.Name[1].Name != "update" { + t.Fatalf("unexpected event name: %#v", event.Name) + } +} + +func TestParseRequiredDiagnostics(t *testing.T) { + tests := []struct { + name string + source string + code diagnostic.Code + message string + hint string + }{ + { + name: "invalid top-level statement", + source: "send \"Hello\" to player\n", + code: diagnostic.CodeInvalidTopLevelStatement, + message: "Executable statements are not allowed at the top level.", + hint: "Move this statement into an event or function.", + }, + { + name: "expected end", + source: "on load\nsend \"Loaded\" to player\n", + code: diagnostic.CodeExpectedEnd, + message: `Expected "end" before end of file.`, + hint: "Add end to close the block.", + }, + { + name: "expected token", + source: "fun add(a: int, b: int -> int\nend\n", + code: diagnostic.CodeExpectedToken, + message: `Expected ")".`, + hint: "Close the parameter list before the return type.", + }, + { + name: "unexpected token", + source: "fun example\nelse\nend\n", + code: diagnostic.CodeUnexpectedToken, + message: "Unexpected token: else", + hint: "else can only appear inside an if block.", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := parseTestSource("invalid.puff", test.source) + if len(result.Diagnostics) != 1 { + t.Fatalf("expected one diagnostic, got %#v", result.Diagnostics) + } + got := result.Diagnostics[0] + if got.Code != test.code || got.Phase != diagnostic.PhaseParser || got.Severity != diagnostic.SeverityError { + t.Fatalf("unexpected diagnostic identity: %#v", got) + } + if got.Message != test.message || got.Hint != test.hint { + t.Fatalf("unexpected diagnostic text: %#v", got) + } + if got.File != "invalid.puff" || got.Span.StartOffset > got.Span.EndOffset { + t.Fatalf("unexpected diagnostic location: %#v", got) + } + }) + } +} + +func TestParseRecoversAtNextTopLevelDeclaration(t *testing.T) { + result := parseTestSource("recovery.puff", ` +return 1 +on tick +end +`) + + if len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != diagnostic.CodeInvalidTopLevelStatement { + t.Fatalf("expected invalid top-level diagnostic, got %#v", result.Diagnostics) + } + if len(result.File.Declarations) != 1 { + t.Fatalf("expected parser to recover one declaration, got %d", len(result.File.Declarations)) + } + event := result.File.Declarations[0].(*ast.EventDecl) + if len(event.Name) != 1 || event.Name[0].Name != "tick" { + t.Fatalf("unexpected recovered declaration: %#v", event) + } +} + +func TestParseRejectsLateRequireAsUnexpected(t *testing.T) { + result := parseTestSource("late-require.puff", ` +on load +end +require "abc/shop" +on tick +end +`) + + if len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != diagnostic.CodeUnexpectedToken { + t.Fatalf("expected unexpected token, got %#v", result.Diagnostics) + } + if result.Diagnostics[0].Message != "Unexpected token: require" { + t.Fatalf("unexpected message: %q", result.Diagnostics[0].Message) + } + if len(result.File.Requirements) != 0 || len(result.File.Declarations) != 2 { + t.Fatalf("unexpected recovery result: %#v", result.File) + } +} + +func TestParseDeclarationSpansCoverSourceConstructs(t *testing.T) { + input := "require \"abc/shop\"\nfun setup\nend\n" + result := parseTestSource("spans.puff", input) + if len(result.Diagnostics) != 0 { + t.Fatalf("expected no diagnostics, got %#v", result.Diagnostics) + } + + requirement := result.File.Requirements[0] + if requirement.Span().StartOffset != 0 || requirement.Span().EndOffset != 18 { + t.Fatalf("unexpected require span: %#v", requirement.Span()) + } + function := result.File.Declarations[0].(*ast.FunctionDecl) + if function.Span().StartOffset != 19 || function.Span().EndOffset != 32 { + t.Fatalf("unexpected function span: %#v", function.Span()) + } + if function.Body.Span().StartOffset != 29 || function.Body.Span().EndOffset != 29 { + t.Fatalf("unexpected empty body span: %#v", function.Body.Span()) + } +} + +func parseTestSource(path string, text string) Result { + file := source.NewFile(path, path, text) + return Parse(file, lexer.Lex(file)) +} + +func renderFile(file *ast.File) string { + var builder strings.Builder + for _, metadata := range file.Metadata { + fmt.Fprintf(&builder, "metadata %s=%s\n", metadata.Key, strconv.Quote(metadata.Value)) + } + for _, requirement := range file.Requirements { + fmt.Fprintf(&builder, "require %s", strconv.Quote(stringValue(requirement.Path))) + if requirement.Alias != nil { + fmt.Fprintf(&builder, " as %s", requirement.Alias.Name) + } + builder.WriteByte('\n') + } + for _, declaration := range file.Declarations { + switch node := declaration.(type) { + case *ast.GlobalAssignment: + if node.Public { + builder.WriteString("pub ") + } + fmt.Fprintf(&builder, "global %s = %s\n", renderVariable(node.Target), renderExpression(node.Value)) + case *ast.FunctionDecl: + if node.Public { + builder.WriteString("pub ") + } + fmt.Fprintf(&builder, "fun %s(", node.Name.Name) + for index, parameter := range node.Parameters { + if index > 0 { + builder.WriteString(", ") + } + builder.WriteString(parameter.Name.Name) + if parameter.Type != nil { + fmt.Fprintf(&builder, ": %s", renderType(parameter.Type)) + } + } + builder.WriteByte(')') + if node.ReturnType != nil { + fmt.Fprintf(&builder, " -> %s", renderType(node.ReturnType)) + } + builder.WriteByte('\n') + case *ast.EventDecl: + names := make([]string, len(node.Name)) + for index, name := range node.Name { + names[index] = name.Name + } + fmt.Fprintf(&builder, "event %s\n", strings.Join(names, " ")) + } + } + return builder.String() +} + +func stringValue(expression *ast.StringExpr) string { + var value strings.Builder + for _, part := range expression.Parts { + if text, ok := part.(*ast.StringText); ok { + value.WriteString(text.Value) + } + } + return value.String() +} + +func renderVariable(variable *ast.VariableExpr) string { + var builder strings.Builder + builder.WriteByte('$') + builder.WriteString(variable.Name.Name) + for _, access := range variable.Accesses { + switch node := access.(type) { + case *ast.FieldAccess: + builder.WriteByte('.') + builder.WriteString(node.Field.Name) + case *ast.EmptyIndexAccess: + builder.WriteString("[]") + case *ast.IndexAccess: + builder.WriteByte('[') + builder.WriteString(renderExpression(node.Index)) + builder.WriteByte(']') + } + } + return builder.String() +} + +func renderExpression(expression ast.Expression) string { + switch node := expression.(type) { + case *ast.NilLiteral: + return "nil" + case *ast.BoolLiteral: + return fmt.Sprintf("bool(%t)", node.Value) + case *ast.IntLiteral: + return fmt.Sprintf("int(%d)", node.Value) + case *ast.FloatLiteral: + return fmt.Sprintf("float(%g)", node.Value) + case *ast.StringExpr: + return "string(" + strconv.Quote(stringValue(node)) + ")" + case *ast.ListExpr: + return "list()" + case *ast.PatternExpr: + var builder strings.Builder + for _, tok := range node.Tokens { + builder.WriteString(tok.Lexeme) + } + return "deferred(" + builder.String() + ")" + default: + return fmt.Sprintf("%T", expression) + } +} + +func renderType(typeRef *ast.TypeRef) string { + if len(typeRef.Arguments) == 0 { + return typeRef.Name.Name + } + arguments := make([]string, len(typeRef.Arguments)) + for index, argument := range typeRef.Arguments { + arguments[index] = renderType(argument) + } + return typeRef.Name.Name + "<" + strings.Join(arguments, ", ") + ">" +} From 9705b270ff140e0e0da2e04495849ff3bf90c587 Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Mon, 27 Jul 2026 11:40:21 -0300 Subject: [PATCH 4/5] fix(parser): improve top-level recovery --- src/internal/lexer/lexer.go | 27 ++++++++++++++++++++ src/internal/parser/parser.go | 42 ++++++++++++++++++++++++++------ src/internal/parser/top_level.go | 19 ++++++++++++--- 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/src/internal/lexer/lexer.go b/src/internal/lexer/lexer.go index b219d4e..c790171 100644 --- a/src/internal/lexer/lexer.go +++ b/src/internal/lexer/lexer.go @@ -20,6 +20,13 @@ var ( type Metadata struct { Namespace string Tags []string + Entries []MetadataEntry +} + +type MetadataEntry struct { + Key string + Value string + Span diagnostic.Span } type Result struct { @@ -457,6 +464,7 @@ func (lexer *lexer) scanMetadata(comment string) { } lexer.metadata.Namespace = value + lexer.addMetadataEntry(key, value, len(comment)) return } @@ -468,6 +476,25 @@ func (lexer *lexer) scanMetadata(comment string) { } } lexer.metadata.Tags = tags + lexer.addMetadataEntry(key, strings.Join(tags, ", "), len(comment)) +} + +func (lexer *lexer) addMetadataEntry(key string, value string, length int) { + startLine, startColumn, _ := lexer.file.Map.LineColumn(lexer.start) + endOffset := lexer.start + length + endLine, endColumn, _ := lexer.file.Map.LineColumn(endOffset) + lexer.metadata.Entries = append(lexer.metadata.Entries, MetadataEntry{ + Key: key, + Value: value, + Span: diagnostic.Span{ + StartLine: startLine, + StartColumn: startColumn, + EndLine: endLine, + EndColumn: endColumn, + StartOffset: lexer.start, + EndOffset: endOffset, + }, + }) } func (lexer *lexer) emitNewline() { diff --git a/src/internal/parser/parser.go b/src/internal/parser/parser.go index 6eca471..22c70a1 100644 --- a/src/internal/parser/parser.go +++ b/src/internal/parser/parser.go @@ -2,7 +2,6 @@ package parser import ( "fmt" - "strings" "github.com/puff-lang/puff/internal/ast" "github.com/puff-lang/puff/internal/diagnostic" @@ -78,8 +77,11 @@ func (parser *parser) parseFile(metadata lexer.Metadata) *ast.File { parser.reportUnexpected(parser.peek(), hint) parser.synchronizeLine() default: + block := parser.check(token.If) || parser.check(token.Loop) parser.reportInvalidTopLevel() - parser.synchronizeLine() + if block { + parser.synchronizeInvalidBlock() + } } parser.skipNewlines() } @@ -88,12 +90,13 @@ func (parser *parser) parseFile(metadata lexer.Metadata) *ast.File { } func metadataEntries(metadata lexer.Metadata) []ast.MetadataEntry { - entries := make([]ast.MetadataEntry, 0, 2) - if metadata.Namespace != "" { - entries = append(entries, ast.MetadataEntry{Key: "namespace", Value: metadata.Namespace}) - } - if len(metadata.Tags) > 0 { - entries = append(entries, ast.MetadataEntry{Key: "tags", Value: strings.Join(metadata.Tags, ", ")}) + entries := make([]ast.MetadataEntry, 0, len(metadata.Entries)) + for _, entry := range metadata.Entries { + entries = append(entries, ast.MetadataEntry{ + NodeBase: ast.NodeBase{SourceSpan: entry.Span}, + Key: entry.Key, + Value: entry.Value, + }) } return entries } @@ -174,6 +177,29 @@ func (parser *parser) synchronizeLine() { } } +func (parser *parser) synchronizeInvalidBlock() { + parser.match(token.Newline) + depth := 1 + lineStart := true + for !parser.atEnd() && depth > 0 { + if lineStart { + switch parser.peek().Type { + case token.If, token.Loop: + depth++ + case token.End: + depth-- + } + lineStart = false + } + if parser.match(token.Newline) { + lineStart = true + continue + } + parser.advance() + } + parser.synchronizeLine() +} + func (parser *parser) match(types ...token.Type) bool { for _, tokenType := range types { if parser.check(tokenType) { diff --git a/src/internal/parser/top_level.go b/src/internal/parser/top_level.go index 25c219e..d8ea2d9 100644 --- a/src/internal/parser/top_level.go +++ b/src/internal/parser/top_level.go @@ -88,6 +88,10 @@ func (parser *parser) parseParameters() []ast.Parameter { if !parser.match(token.Comma) { break } + if parser.check(token.RParen) { + parser.reportExpected("parameter name", "") + break + } } if !parser.match(token.RParen) { @@ -106,6 +110,9 @@ func (parser *parser) parseType() *ast.TypeRef { name := parser.parseIdentifier() typeRef := &ast.TypeRef{Name: *name} if parser.match(token.Less) { + if parser.check(token.Greater) { + parser.reportExpected("type", "") + } for !parser.check(token.Greater) && !parser.check(token.Newline) && !parser.atEnd() { argument := parser.parseType() if argument != nil { @@ -114,6 +121,10 @@ func (parser *parser) parseType() *ast.TypeRef { if !parser.match(token.Comma) { break } + if parser.check(token.Greater) { + parser.reportExpected("type", "") + break + } } if !parser.match(token.Greater) { parser.reportExpected(">", "") @@ -284,12 +295,10 @@ func (parser *parser) parseString() *ast.StringExpr { return parser.stringFromTokens(parser.tokens[start:parser.current]) } case token.Newline: - parser.reportExpected("closing quote", "") return parser.stringFromTokens(parser.tokens[start:parser.current]) } parser.advance() } - parser.reportExpected("closing quote", "") return parser.stringFromTokens(parser.tokens[start:parser.current]) } @@ -305,7 +314,11 @@ func (parser *parser) stringFromTokens(tokens []token.Token) *ast.StringExpr { expression.Quote = tokens[0].Lexeme[0] } - for index := 1; index < len(tokens)-1; index++ { + endIndex := len(tokens) + if tokens[len(tokens)-1].Type == token.StringEnd { + endIndex-- + } + for index := 1; index < endIndex; index++ { tok := tokens[index] switch tok.Type { case token.StringText: From 5d810432978a4d6e5a6e8f1ea7c3875af429597f Mon Sep 17 00:00:00 2001 From: ofabiodev Date: Mon, 27 Jul 2026 11:40:22 -0300 Subject: [PATCH 5/5] test(parser): cover malformed top-level syntax --- src/internal/lexer/lexer_test.go | 11 ++++++ src/internal/parser/top_level_test.go | 51 ++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/internal/lexer/lexer_test.go b/src/internal/lexer/lexer_test.go index 67a1149..9bbfea6 100644 --- a/src/internal/lexer/lexer_test.go +++ b/src/internal/lexer/lexer_test.go @@ -75,6 +75,17 @@ func TestLexCollectsFrontMatterMetadata(t *testing.T) { if result.Metadata.Tags[0] != "load" || result.Metadata.Tags[1] != "tick" { t.Fatalf("expected tags [load tick], got %v", result.Metadata.Tags) } + if len(result.Metadata.Entries) != 2 { + t.Fatalf("expected two metadata entries, got %d", len(result.Metadata.Entries)) + } + namespace := result.Metadata.Entries[0] + if namespace.Key != "namespace" || namespace.Value != "example" || namespace.Span.StartOffset != 1 || namespace.Span.EndOffset != 21 { + t.Fatalf("unexpected namespace entry: %#v", namespace) + } + tags := result.Metadata.Entries[1] + if tags.Key != "tags" || tags.Value != "load, tick" || tags.Span.StartLine != 3 || tags.Span.EndLine != 3 { + t.Fatalf("unexpected tags entry: %#v", tags) + } assertTokenTypes(t, result.Tokens, []token.Type{ token.On, diff --git a/src/internal/parser/top_level_test.go b/src/internal/parser/top_level_test.go index ca8b94c..cb00267 100644 --- a/src/internal/parser/top_level_test.go +++ b/src/internal/parser/top_level_test.go @@ -33,11 +33,14 @@ func TestParseTopLevelGolden(t *testing.T) { if got := renderFile(result.File); got != string(want) { t.Fatalf("unexpected AST\nwant:\n%s\ngot:\n%s", want, got) } + if result.File.Metadata[0].Span().StartOffset != 0 || result.File.Metadata[0].Span().EndOffset != 20 { + t.Fatalf("unexpected metadata span: %#v", result.File.Metadata[0].Span()) + } } func TestParseFunctionSignaturesAndNestedTypes(t *testing.T) { result := parseTestSource("functions.puff", ` -fun noParams +fun add end fun explicit() end @@ -72,6 +75,24 @@ end } } +func TestParseRejectsIncompleteParameterAndGenericLists(t *testing.T) { + tests := []string{ + "fun f(a,)\nend\n", + "fun f(value: list<>)\nend\n", + "fun f(value: map)\nend\n", + } + + for _, input := range tests { + result := parseTestSource("signature.puff", input) + if len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != diagnostic.CodeExpectedToken { + t.Fatalf("source %q: expected one EXPECTED_TOKEN, got %#v", input, result.Diagnostics) + } + if len(result.File.Declarations) != 1 { + t.Fatalf("source %q: expected partial function declaration, got %#v", input, result.File.Declarations) + } + } +} + func TestParseGlobalsPreservesTargetsAndSimpleValues(t *testing.T) { result := parseTestSource("globals.puff", ` $shop.name = "Main Shop" @@ -218,6 +239,34 @@ end } } +func TestParseSkipsInvalidTopLevelBlockWithoutCascading(t *testing.T) { + result := parseTestSource("invalid-block.puff", ` +if true +send "inside" to player +end +on tick +end +`) + + if len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != diagnostic.CodeInvalidTopLevelStatement { + t.Fatalf("expected one invalid top-level diagnostic, got %#v", result.Diagnostics) + } + if len(result.File.Declarations) != 1 || result.File.Declarations[0].(*ast.EventDecl).Name[0].Name != "tick" { + t.Fatalf("expected recovery at tick event, got %#v", result.File.Declarations) + } +} + +func TestParseDoesNotDuplicateUnterminatedStringDiagnostic(t *testing.T) { + result := parseTestSource("unterminated.puff", "require \"abc\non load\nend\n") + + if len(result.Diagnostics) != 1 || result.Diagnostics[0].Code != diagnostic.CodeUnterminatedString { + t.Fatalf("expected only lexer string diagnostic, got %#v", result.Diagnostics) + } + if len(result.File.Declarations) != 1 || result.File.Declarations[0].(*ast.EventDecl).Name[0].Name != "load" { + t.Fatalf("expected recovery at load event, got %#v", result.File.Declarations) + } +} + func TestParseRejectsLateRequireAsUnexpected(t *testing.T) { result := parseTestSource("late-require.puff", ` on load