rewrite llparse from the book and split up modules.

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "llparse"
 dynamic = ["version"]
-description = "A Parody of llparse written for writing C Parsers with Python"
+description = "A Parody of typescript llparse written for generating C Parsers using Python"
 readme = "README.md"
 authors = [
     { name = "Vizonex", email = "VizonexBusiness@gmail.com" }
@@ -11,11 +11,18 @@ dependencies = [
     "typing_extensions; python_version < '3.13'"
 ]
 
+
 [tool.setuptools.dynamic]
 version = {attr = "llparse.__version__"}
 
 [build-system]
 requires = ["setuptools"]
 
+[dependency-groups]
+dev = [
+    "pytest>=9.1.1",
+    "typer>=0.27.0",
+]
+
 [tool.ruff]
 target-version = "py310"

src/llparse/__init__.py

@@ -0,0 +1,5 @@
+from .api import LLParse
+
+__version__ = "1.0.0"
+
+__all__ = ("LLParse",)

src/llparse/api.py

@@ -0,0 +1,98 @@
+from llparse_builder import builder as source
+from llparse_frontend import frontend
+
+from .compiler import Compiler, CompilerResult
+
+
+class LLParse(source.Builder):
+    """
+
+    The prefix controls the names of methods and state struct in generated
+    public C headers:
+
+    ```c
+    // state struct
+    struct PREFIX_t {
+      ...
+    }
+
+    int PREFIX_init(PREFIX_t* state);
+    int PREFIX_execute(PREFIX_t* state, const char p, const char endp);
+    ```
+    """
+
+    def __init__(self, prefix: str = "llparse") -> None:
+        """
+        :param prefix: Prefix to be used when generating public API default is "llparse".
+        """
+        self.prefix = prefix
+        super().__init__()
+
+    def get_compiler(
+        self,
+        headerGuard: str | None = None,
+        debug: str | None = None,
+        max_table_elem_width: int | None = None,
+        min_table_size: int | None = None,
+    ) -> Compiler:
+        return Compiler(
+            self.prefix,
+            headerGuard,
+            debug,
+            max_table_elem_width
+            if max_table_elem_width
+            else frontend.DEFAULT_MAX_TABLE_WIDTH,
+            min_table_size if min_table_size else frontend.DEFAULT_MIN_TABLE_SIZE,
+        )
+
+    def build(
+        self,
+        root: source.node.Node,
+        headerGuard: str | None = None,
+        debug: str | None = None,
+        max_table_elem_width: int | None = None,
+        min_table_size: int | None = None,
+        header_name: str | None = None,
+        override_llparse_name: bool = False,
+    ) -> CompilerResult:
+        """Builds Graph and then compiles the data into C code , returns with the header and C file inside of a Dataclass"""
+
+        compiler = Compiler(
+            self.prefix,
+            headerGuard,
+            debug,
+            max_table_elem_width
+            if max_table_elem_width
+            else frontend.DEFAULT_MAX_TABLE_WIDTH,
+            min_table_size if min_table_size else frontend.DEFAULT_MIN_TABLE_SIZE,
+        )
+
+        return compiler.compile(
+            root,
+            self.properties(),
+            header_name=header_name,
+            override_llparse_name=override_llparse_name,
+        )
+
+    def to_frontend(
+        self,
+        root: source.node.Node,
+        headerGuard: str | None = None,
+        debug: str | None = None,
+        max_table_elem_width: int | None = None,
+        min_table_size: int | None = None,
+    ) -> Compiler:
+        """Used as an external hack to get access to the frontend of llparse and extract
+        it's contents to compile the libraries you make other things like cython, This is not in llparse
+        specifically (Yet...)"""
+        return Compiler(
+            self.prefix,
+            headerGuard,
+            debug,
+            max_table_elem_width
+            if max_table_elem_width
+            else frontend.DEFAULT_MAX_TABLE_WIDTH,
+            min_table_size if min_table_size else frontend.DEFAULT_MIN_TABLE_SIZE,
+        ).to_frontend(root, self.properties)
+
+    # capi will return soon...

src/llparse/compiler/__init__.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass
+from logging import getLogger
+from pathlib import Path
+
+from llparse_builder import builder as source
+from llparse_frontend.frontend import Frontend
+
+from ..implementation import c
+from .header_builder import HeaderBuilder
+
+logger = getLogger()
+
+debug = logger.debug
+
+
+@dataclass(slots=True)
+class CompilerResult:
+    c: str
+    """Textual C code"""
+    header: str
+    """Textual C header file"""
+
+    def write(self, c: Path | str, header: Path | str) -> None:
+        """
+        Writes the output to the chosen file locations
+
+        :param c: Output for where to write the C File
+        :type c: Path | str
+        :param header: Output for where to write the Header File
+        :type header: Path | str
+        """
+        Path(c).write_text(self.c)
+        Path(header).write_text(self.header)
+
+
+@dataclass
+class Compiler:
+    prefix: str
+    header_guard: str | None = None
+    debug: str | None = None
+    max_table_elem_width: int | None = None
+    min_table_size: int | None = None
+
+    def to_frontend(
+        self,
+        root: source.node.Node,
+        properties: list[source.Property],
+        impl=c,
+    ):
+        """compiles up the frontend and brings you back the frontend's results.
+        I added documentation to this function so that you can do creative things
+        with the library beyond C..."""
+        return Frontend(
+            self.prefix,
+            impl,
+            max_table_elem_width=self.max_table_elem_width,
+            min_table_size=self.min_table_size,
+        ).compile(root, properties)
+
+    def compile(
+        self,
+        root: source.node.Node,
+        properties: list[source.Property],
+        header_name: str | None = None,
+        impl=c,
+        override_llparse_name: bool = False,
+    ) -> CompilerResult:
+        """Creates the C and header file..."""
+        info = self.to_frontend(root, properties, impl)
+        hb = HeaderBuilder(self.prefix, self.header_guard, properties, info.spans)
+        cdata = c.CCompiler(header_name, self.debug).compile(info)
+        if override_llparse_name:
+            # sometimes users want to combine parsers together when compiling with C
+            # to make up for conflicts with other parsers example: llhttp
+            # there should be a fair way of compiling everything.
+            cdata = cdata.replace("llparse", self.prefix)
+
+        return CompilerResult(cdata, hb.build())

src/llparse/compiler/header_builder.py

@@ -0,0 +1,75 @@
+from dataclasses import dataclass, field
+
+from llparse_builder.builder import Property
+from llparse_frontend.span_field import SpanField
+
+from ..error import Error
+
+TYPE_LOOKUP = {
+    "i8": "uint8_t",
+    "i16": "uint16_t",
+    "i32": "uint32_t",
+    "i64": "uint64_t",
+    "ptr": "void*",
+}
+
+
+@dataclass(slots=True)
+class HeaderBuilder:
+    prefix: str
+    header_guard: str | None = field(default=None)
+    properties: list[Property] = field(default_factory=list)
+    spans: list[SpanField] = field(default_factory=list)
+
+    def build(self) -> str:
+        """Builds The string to create the header file"""
+        res = ""
+        PREFIX = self.prefix.upper()
+        DEFINE = f"INCLUDE_{PREFIX}_H_" if not self.header_guard else self.header_guard
+
+        res += f"#ifndef {DEFINE}\n"
+        res += f"#define {DEFINE}\n"
+        res += "#ifdef __cplusplus\n"
+        res += 'extern "C" {\n'
+        res += "#endif\n"
+        res += "\n"
+
+        res += "#include <stdint.h>\n"
+        res += "\n"
+
+        # Main Structure
+        res += f"typedef struct {self.prefix}_s {self.prefix}_t;\n"
+        res += f"struct {self.prefix}_s " + "{\n"
+        res += "  int32_t _index;\n"
+
+        for index, f in enumerate(self.spans):
+            res += f"  void* _span_pos{index};\n"
+            if len(f.callbacks) > 1:
+                res += f"  void* _span_cb{index};\n"
+
+        # TODO: Reorganize fields for better heap/memory and performance.
+        res += "  int32_t error;\n"
+        res += "  const char* reason;\n"
+        res += "  const char* error_pos;\n"
+        res += "  void* data;\n"
+        res += "  void* _current;\n"
+
+        for prop in self.properties:
+            if not (ty := TYPE_LOOKUP.get(prop.ty)):
+                raise Error(f'Unknown state property type: "{prop.ty}"')
+
+            res += f"  {ty}  {prop.name};\n"
+        res += "};"
+
+        res += "\n"
+
+        res += f"int {self.prefix}_init({self.prefix}_t* s);\n"
+        res += f"int {self.prefix}_execute({self.prefix}_t* s, const char* p, const char* endp);\n"
+
+        res += "\n"
+
+        res += "#ifdef __cplusplus\n"
+        res += '} /* extern "C" */\n'
+        res += "#endif\n"
+        res += f"#endif /* {DEFINE} */"
+        return res

src/llparse/error.py

@@ -0,0 +1,2 @@
+class Error(Exception):
+    """llparse compiler-related error"""

src/llparse/ext.py

@@ -0,0 +1,90 @@
+from collections.abc import Generator
+from contextlib import contextmanager
+from io import StringIO
+
+
+# Modified from Cython's version with a much smoother system to utilize.
+class LinesResult:
+    __slots__ = ("io",)
+
+    def __init__(self, io: StringIO | None = None):
+        self.io = io or StringIO()
+
+    def put(self, s: str) -> None:
+        self.io.write(s)
+
+    def newline(self) -> None:
+        self.io.write("\n")
+
+    def putline(self, s: str) -> None:
+        self.io.write(s)
+        self.io.write("\n")
+
+    def __str__(self):
+        return self.io.getvalue()
+
+
+# Based off Cython's CodeWriter module in the DeclarationWriter class
+class Writer:
+    """A Simplistic Code Writer tool for outputting
+    and writing clean code. It also allows users to cutomize
+    how big the indent size of the output should be."""
+
+    __slots__ = ("_indent_size", "_indent_str", "_numindents", "_result")
+
+    def __init__(self, indent_size: int = 2, result: LinesResult | None = None):
+        if indent_size < 1:
+            raise ValueError("Indent size requires a number at least greater than 1.")
+        self._indent_size = indent_size
+        self._result = result if result is not None else LinesResult()
+        self._numindents = 0
+        self._indent_str = " " * indent_size
+
+    def indent(self) -> None:
+        self._numindents += 1
+
+    def dedent(self) -> None:
+        if self._numindents < 0:
+            raise RuntimeError("number of indents is out of bounds.")
+        self._numindents -= 1
+
+    def startline(self, s: str = "") -> None:
+        self._result.put(self._indent_str * self._numindents + s)
+
+    def put(self, s: str) -> None:
+        self._result.put(s)
+
+    def putline(self, s: str) -> None:
+        self._result.putline(self._indent_str * self._numindents + s)
+
+    def putline_indented(self, s: str) -> None:
+        with self.tab():
+            self.putline(s)
+
+    def endline(self, s: str = "") -> None:
+        self._result.putline(s)
+
+    def line(self, s: str) -> None:
+        self.startline(s)
+        self.endline()
+
+    @contextmanager
+    def tab(self) -> Generator[None, None, None]:
+        """Indents and later dedents the number of indents
+        used on each given line as a context manager."""
+        self.indent()
+        yield
+        self.dedent()
+
+    def skipline(self) -> None:
+        self.endline()
+
+    def skiplines(self, amount: int = 2):
+        if amount < 1:
+            raise RuntimeError("skiplines amount must be greater or equal to 1")
+        for _ in range(amount):
+            self.skipline()
+
+    def result(self) -> str:
+        """Obtains the written data from the writer"""
+        return self._result.io.getvalue()