diff --git a/crates/compiler/src/parser/grammar.rs b/crates/compiler/src/parser/grammar.rs index 53bc3f8..dcd2e42 100644 --- a/crates/compiler/src/parser/grammar.rs +++ b/crates/compiler/src/parser/grammar.rs @@ -3,16 +3,13 @@ //! Builds `Ast<&'a str, ParseMetadata>` directly from the token stream in a //! single pass — no intermediate concrete syntax tree. Identifier and string //! text is borrowed straight from the source (`&'a str`); every node records a -//! byte-offset `cfgrammar::Span` that indexes the original (untrimmed) input, -//! matching the spans the ANTLR integration produced. +//! byte-offset `cfgrammar::Span` that indexes the original (untrimmed) input. //! -//! Expression precedence/associativity mirrors the ANTLR `expr` rule exactly -//! (validated against the generated `expr_rec`/`precpred`): prefix unary binds -//! tightest for its operand; the suffix cluster (`.field`, `.idx`, `[]`, `!`, -//! `as`) binds tighter than the binary operators; `* / %` > `+ -` > comparisons; -//! all binary operators are left-associative. Boolean operations are lower precedence -//! than comparisons, as in Rust: -//! comparisons > `&&` > `||`. +//! Expression precedence/associativity: prefix unary binds tightest for its +//! operand; the suffix cluster (`.field`, `.idx`, `[]`, `!`, `as`) binds tighter +//! than the binary operators; `* / %` > `+ -` > comparisons; all binary +//! operators are left-associative. Boolean operations are lower precedence than +//! comparisons, as in Rust: comparisons > `&&` > `||`. use std::str::FromStr; @@ -236,7 +233,7 @@ impl<'a> Parser<'a> { /// or recovery path a rule may consume nothing after capturing `lo`, leaving /// `prev_end < lo`; clamp so the span is never inverted (`cfgrammar::Span::new` /// panics when `end < start`). For well-formed nodes `prev_end >= lo`, so this - /// is a no-op and spans match the byte ranges ANTLR produced. + /// is a no-op. #[inline] fn finish_span(&self, lo: u32) -> Span { Span::new(lo as usize, self.prev_end.max(lo) as usize) @@ -290,7 +287,7 @@ impl<'a> Parser<'a> { // Distinct diagnostics at the same offset are kept: they describe // independent problems (e.g. a token that is simultaneously not an // expression and not the expected `)`), so collapsing them by position - // alone dropped diagnostics ANTLR reported. + // alone would lose real diagnostics. if let Some(last) = self.errors.last() && last.span.start() == span.start() && last.message == message @@ -335,9 +332,9 @@ impl<'a> Parser<'a> { self.bump(); } } - // Like ANTLR's `ast : decl* EOF` context, the root span runs to the EOF - // token, i.e. the end of the (untrimmed) input — `src` is the trimmed - // buffer, so `src.len() + base` is the original length. + // The root span runs to the EOF token, i.e. the end of the (untrimmed) + // input — `src` is the trimmed buffer, so `src.len() + base` is the + // original length. let end = self.src.len() + self.base; Ast { decls, @@ -345,13 +342,13 @@ impl<'a> Parser<'a> { } } - /// `callExpr EOF` as a standalone entry (used by `parse_cell`). Returns - /// `None` (with an error recorded) unless the input is *exactly* one call - /// expression: the whole input must parse to an `Expr::Call` and reach EOF. + /// A single call expression followed by EOF, as a standalone entry (used by + /// `parse_cell`). Returns `None` (with an error recorded) unless the input is + /// *exactly* one call expression: the whole input must parse to an + /// `Expr::Call` and reach EOF. /// This rejects both trailing garbage (`f() junk`) and suffixed calls /// (`f()!`, `f().x`, `f()[0]`, which parse to an `Emit`/`FieldAccess`/`Index` - /// root rather than a `Call`), keeping the "parses exactly a callExpr" - /// contract the old ANTLR `callExpr()` entry had. + /// root rather than a `Call`). pub fn parse_cell_entry(&mut self) -> Option> { let expr = self.parse_expr(0); let Expr::Call(call) = expr else { @@ -705,9 +702,7 @@ impl<'a> Parser<'a> { // routes a trailing un-semicoloned expression into `tail` (the // `at(RBrace) || at(Eof)` arm), and only ever pushes a `semicolon: false` // statement when more tokens follow it — so a `semicolon: false` - // statement is never the last element here. (ANTLR's - // `build_unannotated_scope` built statements and the tail separately and - // did need the fixup; this single-pass loop does not.) + // statement is never the last element here. self.exit_depth(); Scope { @@ -849,7 +844,7 @@ impl<'a> Parser<'a> { // Lexical start of this expression (the first token). Composite-node // spans start here, not at `lhs.span().start()`: a parenthesized // operand is unwrapped to its inner node (whose span excludes the - // parens), but ANTLR spans the enclosing operator from the `(`. + // parens), while the enclosing operator's span must start at the `(`. let lhs_start = self.cur.start; let mut lhs = self.parse_prefix(); diff --git a/crates/compiler/src/parser/mod.rs b/crates/compiler/src/parser/mod.rs index 8a5a275..1d4e1c4 100644 --- a/crates/compiler/src/parser/mod.rs +++ b/crates/compiler/src/parser/mod.rs @@ -1,9 +1,9 @@ //! Hand-written, zero-copy parser for the Argon language. //! -//! Replaces the ANTLR-generated parser: a streaming byte lexer ([`lexer`]) feeds -//! a single-pass recursive-descent + Pratt parser ([`grammar`]) that builds the -//! AST directly, borrowing all identifier/string text from the source. The two -//! public entry points match the contract the rest of the compiler expects. +//! A streaming byte lexer ([`lexer`]) feeds a single-pass recursive-descent + +//! Pratt parser ([`grammar`]) that builds the AST directly, borrowing all +//! identifier/string text from the source. The two public entry points match +//! the contract the rest of the compiler expects. mod grammar; mod lexer; @@ -19,7 +19,6 @@ use crate::ast::{CallExpr, Decl}; use crate::parse::{AnnotatedParseAst, ParseMetadata}; /// A syntax error with the byte span (into the original input) it occurred at. -/// Shape-compatible with the old `antlr::AntlrParseError`. #[derive(Debug, Clone)] pub struct ParseError { pub span: Span, @@ -342,7 +341,7 @@ mod tests { #[test] fn leading_comment_is_allowed() { // The lexer skips `//` comments as trivia everywhere, so a comment - // before the first declaration parses fine (ANTLR rejected this). + // before the first declaration parses fine. assert!(parse("// header\ncell c() {}\n").is_ok()); assert!(parse(" \n// c1\n// c2\nfn f() -> Float { 1. }\n").is_ok()); } diff --git a/crates/compiler/src/parser/token.rs b/crates/compiler/src/parser/token.rs index 93d7557..f717556 100644 --- a/crates/compiler/src/parser/token.rs +++ b/crates/compiler/src/parser/token.rs @@ -4,8 +4,8 @@ //! own any text. The parser slices identifier/string text directly from the //! input by span, so lexing and parsing are entirely copy-free. -/// The lexical category of a token. Mirrors the lexer rules in -/// `grammar/Argon.g4` (kept as the language reference). +/// The lexical category of a token. The lexer in `lexer.rs` produces exactly +/// these kinds; `describe` gives the human-readable name used in diagnostics. #[derive(Clone, Copy, PartialEq, Eq, Debug)] #[repr(u8)] pub enum TokenKind { @@ -136,8 +136,7 @@ impl TokenKind { /// A lexed token: a kind plus the half-open byte range `[start, end)` it covers /// in the original source. Offsets already include the `offset_base` (the count /// of leading whitespace bytes trimmed before lexing), so they index the -/// original (untrimmed) input — matching the spans the ANTLR integration -/// produced. +/// original (untrimmed) input. #[derive(Clone, Copy, Debug)] pub struct Token { pub kind: TokenKind, diff --git a/docs/parser.md b/docs/parser.md index 4546e07..edf5ded 100644 --- a/docs/parser.md +++ b/docs/parser.md @@ -13,18 +13,14 @@ The parser lives in `crates/compiler/src/parser/`: | `grammar.rs` | `Parser` — a recursive-descent + Pratt parser that builds the AST. Most of the logic lives here. | | `mod.rs` | The public entry points (`parse_ast`, `parse_cell`), the `ParseError` type, and the `#[cfg(test)]` test suite. | -The AST node definitions are in `crates/compiler/src/ast/mod.rs`. A descriptive -(non-executable) reference grammar is kept in sync at -`crates/compiler/grammar/Argon.g4`. +The AST node definitions are in `crates/compiler/src/ast/mod.rs`. --- ## 1. Overview -The parser was originally generated by [ANTLR](https://www.antlr.org/). It has -been replaced by a hand-written parser; the ANTLR tool, its generated code, and -the `antlr-rust` runtime dependency are gone, which also removes the Java/Maven -build step. The hand-written parser is: +The parser is hand-written — no parser generator, generated code, or external +grammar tooling is involved in the build. It is: - **Single-pass.** There is no intermediate concrete syntax tree (CST). The token stream is walked once and AST nodes are constructed directly. @@ -101,8 +97,8 @@ the AST are always relative to the original input. > **Known leniency:** the trim uses `char::is_whitespace` (Unicode-aware) while > the lexer's `skip_trivia` only skips ASCII ` \t\r\n`. A leading non-ASCII > whitespace byte (NBSP, BOM, …) is therefore folded into `offset_base` and the -> input parses, where the old ANTLR `WS` rule would have rejected it. This is -> harmless and currently accepted. +> input parses, even though the same byte *between* tokens is an error token. +> This is harmless and currently accepted. ### Byte-exact spans @@ -187,8 +183,8 @@ mid-character). So every offset the lexer emits is a valid UTF-8 char boundary. `skip_trivia` is called at the start of every `next_token`. It skips ASCII whitespace and `//` line comments (everything to the next `\n`/`\r`). Comments -and whitespace are *trivia* — they never become tokens. (A comment before the -first declaration therefore parses fine, which ANTLR's grammar rejected.) +and whitespace are *trivia* — they never become tokens, so a comment before the +first declaration parses fine. ### Dispatch @@ -317,7 +313,7 @@ The `.max(lo)` is the panic-safety net: on an error/recovery path a rule may capture `lo = cur.start` and then consume nothing (a bad token, or EOF), leaving `prev_end < lo`, which would make `Span::new` panic. Clamping `end = max(prev_end, lo)` yields a zero-width span instead. For well-formed nodes `prev_end >= lo`, so -it is a no-op and spans match the byte ranges ANTLR produced. +it is a no-op. Text is read back from spans with `slice_tok(token)` and `slice_span(span)`, both of which subtract `base` to index the trimmed buffer. @@ -552,9 +548,8 @@ The same check covers a tuple-index suffix (`t.0`). `parse_expr` records `lhs_start = self.cur.start` *before* parsing the prefix, and operator nodes span from there rather than from `lhs.span().start()`. The reason: a parenthesized operand is unwrapped to its inner node, whose span -excludes the `(`; but ANTLR (and the rest of the compiler) expects the enclosing -operator to span from the `(`. Using the lexical start of the first token gets -this right. +excludes the `(`; but the rest of the compiler expects the enclosing operator to +span from the `(`. Using the lexical start of the first token gets this right. A consequence of `PREFIX_BP (13) > SUFFIX_BP (11)` is that a prefix operator's operand is a bare primary — suffixes do **not** bind inside it. So `-a.b` parses @@ -705,17 +700,7 @@ declaration doesn't poison the rest of the file. --- -## 13. Relationship to `Argon.g4` - -`crates/compiler/grammar/Argon.g4` is a **descriptive reference grammar**, not an -input to any build step (nothing generates code from it any more). It documents -the intended language and should be kept in sync when the parser's accepted -language changes. Expression precedence and associativity in the Pratt table -were validated against the precedence ANTLR's generated `expr` rule produced. - ---- - -## 14. Performance +## 13. Performance The parser is designed to be fast and allocation-light: @@ -735,7 +720,7 @@ cargo test -p argonc --release -- --ignored --nocapture parser_throughput --- -## 15. Testing +## 14. Testing Parser tests live in `mod.rs` under `#[cfg(test)] mod tests`: @@ -753,9 +738,9 @@ Parser tests live in `mod.rs` under `#[cfg(test)] mod tests`: `pdks/`, and the compiler's `src/std/` must parse without error. End-to-end coverage (parse → compile → solve) lives in `crates/compiler/src/lib.rs` -as the `argon_*` and `stress_*_smoke` tests. There is no longer a differential -test against ANTLR (it has been removed), so behavioral changes should be locked -in with a focused `parser::tests` case. +as the `argon_*` and `stress_*_smoke` tests. The parser is the only definition +of the accepted language, so behavioral changes should be locked in with a +focused `parser::tests` case. Verify any change with: @@ -767,7 +752,7 @@ cargo fmt -p argonc -- --check --- -## 16. Gotchas and invariants +## 15. Gotchas and invariants - **Spans must be byte-exact** and must not invert (`end >= start`). Prefer `finish_span(lo)` over a raw `Span::new(lo, prev_end)` on any path that might