From 3a96c5dd523cb6c271808ec7126982d2de3c0959 Mon Sep 17 00:00:00 2001 From: Vizonex Date: Sun, 2 Aug 2026 01:18:07 -0500 Subject: [PATCH] rewrite llparse from the book and split up modules. --- pyproject.toml | 9 +- src/llparse/__init__.py | 5 + src/llparse/api.py | 98 ++++ src/llparse/compiler/__init__.py | 78 +++ src/llparse/compiler/header_builder.py | 75 +++ src/llparse/error.py | 2 + src/llparse/ext.py | 90 +++ src/llparse/implementation/__init__.py | 0 src/llparse/implementation/c/__init__.py | 196 +++++++ src/llparse/implementation/c/code/__init__.py | 32 + src/llparse/implementation/c/code/and_.py | 10 + src/llparse/implementation/c/code/base.py | 16 + src/llparse/implementation/c/code/external.py | 15 + src/llparse/implementation/c/code/field.py | 27 + src/llparse/implementation/c/code/is_equal.py | 9 + src/llparse/implementation/c/code/load.py | 9 + src/llparse/implementation/c/code/mul_add.py | 63 ++ src/llparse/implementation/c/code/or_.py | 10 + src/llparse/implementation/c/code/store.py | 10 + src/llparse/implementation/c/code/test.py | 10 + src/llparse/implementation/c/code/update.py | 10 + src/llparse/implementation/c/code/value.py | 8 + src/llparse/implementation/c/compilation.py | 287 +++++++++ src/llparse/implementation/c/constants.py | 42 ++ .../c/helpers/match_sequence.py | 69 +++ src/llparse/implementation/c/node/__init__.py | 33 ++ src/llparse/implementation/c/node/base.py | 77 +++ src/llparse/implementation/c/node/consume.py | 40 ++ src/llparse/implementation/c/node/empty.py | 11 + src/llparse/implementation/c/node/error.py | 23 + src/llparse/implementation/c/node/invoke.py | 34 ++ src/llparse/implementation/c/node/pause.py | 15 + src/llparse/implementation/c/node/sequence.py | 54 ++ src/llparse/implementation/c/node/single.py | 39 ++ src/llparse/implementation/c/node/span_end.py | 60 ++ .../implementation/c/node/span_start.py | 24 + .../implementation/c/node/table_lookup.py | 9 + src/llparse/implementation/c/node/unpack.py | 161 ++++++ .../implementation/c/transform/__init__.py | 16 + .../implementation/c/transform/base.py | 14 + src/llparse/implementation/c/transform/id.py | 8 + .../implementation/c/transform/to_lower.py | 10 + .../c/transform/to_lower_unsafe.py | 8 + src/llparse_builder/__init__.py | 8 + src/llparse_builder/binary_search.py | 59 ++ src/llparse_builder/builder.py | 173 ++++++ src/llparse_builder/code/__init__.py | 33 ++ src/llparse_builder/code/_and.py | 8 + src/llparse_builder/code/_or.py | 8 + src/llparse_builder/code/base.py | 9 + src/llparse_builder/code/creator.py | 54 ++ src/llparse_builder/code/field.py | 12 + src/llparse_builder/code/field_value.py | 13 + src/llparse_builder/code/is_equal.py | 8 + src/llparse_builder/code/load.py | 8 + src/llparse_builder/code/match.py | 8 + src/llparse_builder/code/mul_add.py | 20 + src/llparse_builder/code/span.py | 5 + src/llparse_builder/code/store.py | 8 + src/llparse_builder/code/test.py | 8 + src/llparse_builder/code/update.py | 8 + src/llparse_builder/code/value.py | 8 + src/llparse_builder/edge.py | 59 ++ src/llparse_builder/errors.py | 6 + src/llparse_builder/loop_checker/__init__.py | 163 ++++++ src/llparse_builder/loop_checker/lattice.py | 117 ++++ src/llparse_builder/node/__init__.py | 21 + src/llparse_builder/node/base.py | 64 ++ src/llparse_builder/node/consume.py | 9 + src/llparse_builder/node/error.py | 25 + src/llparse_builder/node/invoke.py | 18 + src/llparse_builder/node/match.py | 77 +++ src/llparse_builder/node/pause.py | 12 + src/llparse_builder/node/span_end.py | 14 + src/llparse_builder/node/span_start.py | 14 + src/llparse_builder/node/unpack.py | 167 ++++++ src/llparse_builder/property.py | 19 + src/llparse_builder/py.typed | 0 src/llparse_builder/reachability.py | 24 + src/llparse_builder/span.py | 34 ++ src/llparse_builder/span_allocator.py | 163 ++++++ src/llparse_builder/transform/__init__.py | 5 + src/llparse_builder/transform/base.py | 8 + src/llparse_builder/transform/creator.py | 12 + src/llparse_builder/transform/to_lower.py | 8 + .../transform/to_lower_unsafe.py | 8 + src/llparse_builder/unpack_creator.py | 54 ++ src/llparse_builder/utils.py | 10 + src/llparse_frontend/__init__.py | 0 src/llparse_frontend/code/__init__.py | 35 ++ src/llparse_frontend/code/and_.py | 9 + src/llparse_frontend/code/base.py | 14 + src/llparse_frontend/code/external.py | 6 + src/llparse_frontend/code/field.py | 9 + src/llparse_frontend/code/field_value.py | 12 + src/llparse_frontend/code/is_equal.py | 9 + src/llparse_frontend/code/load.py | 6 + src/llparse_frontend/code/match.py | 6 + src/llparse_frontend/code/mul_add.py | 30 + src/llparse_frontend/code/or_.py | 9 + src/llparse_frontend/code/span.py | 6 + src/llparse_frontend/code/store.py | 6 + src/llparse_frontend/code/test.py | 9 + src/llparse_frontend/code/update.py | 9 + src/llparse_frontend/code/value.py | 6 + src/llparse_frontend/container/__init__.py | 71 +++ src/llparse_frontend/container/wrap.py | 24 + src/llparse_frontend/enumerator.py | 21 + src/llparse_frontend/errors.py | 2 + src/llparse_frontend/frontend.py | 547 ++++++++++++++++++ .../implementation/__init__.py | 11 + src/llparse_frontend/implementation/code.py | 20 + src/llparse_frontend/implementation/full.py | 12 + src/llparse_frontend/implementation/node.py | 20 + .../implementation/transform.py | 12 + .../implementation/typedefs.py | 7 + src/llparse_frontend/namespace/frontend.py | 3 + src/llparse_frontend/node/__init__.py | 36 ++ src/llparse_frontend/node/base.py | 66 +++ src/llparse_frontend/node/consume.py | 10 + src/llparse_frontend/node/empty.py | 5 + src/llparse_frontend/node/error.py | 11 + src/llparse_frontend/node/invoke.py | 45 ++ src/llparse_frontend/node/match.py | 14 + src/llparse_frontend/node/pause.py | 5 + src/llparse_frontend/node/sequence.py | 50 ++ src/llparse_frontend/node/single.py | 66 +++ src/llparse_frontend/node/slot.py | 21 + src/llparse_frontend/node/span_end.py | 14 + src/llparse_frontend/node/span_start.py | 14 + src/llparse_frontend/node/table_lookup.py | 62 ++ src/llparse_frontend/node/unpack.py | 29 + src/llparse_frontend/peephole.py | 43 ++ src/llparse_frontend/span_field.py | 11 + src/llparse_frontend/transform/__init__.py | 6 + src/llparse_frontend/transform/base.py | 5 + src/llparse_frontend/transform/id.py | 6 + src/llparse_frontend/transform/to_lower.py | 6 + .../transform/to_lower_unsafe.py | 6 + src/llparse_frontend/trie/__init__.py | 143 +++++ src/llparse_frontend/trie/empty.py | 13 + src/llparse_frontend/trie/node.py | 5 + src/llparse_frontend/trie/sequence.py | 9 + src/llparse_frontend/trie/single.py | 22 + src/llparse_frontend/utils/__init__.py | 14 + src/llparse_frontend/utils/identifier.py | 29 + src/llparse_frontend/wrap.py | 17 + tests/builder/conftest.py | 8 + tests/builder/test_binary_search.py | 52 ++ tests/builder/test_builder.py | 60 ++ tests/{ => builder}/test_loop_checker.py | 48 +- tests/{ => builder}/test_span_allocator.py | 19 +- tests/builder/test_unpack.py | 150 +++++ tests/builder/test_unpack_creator.py | 1 + tests/frontend/__init__.py | 1 + tests/frontend/conftest.py | 8 + tests/frontend/fixtures/__init__.py | 0 .../fixtures/implementation/__init__.py | 5 + .../fixtures/implementation/code/__init__.py | 36 ++ .../fixtures/implementation/code/and_.py | 8 + .../fixtures/implementation/code/base.py | 14 + .../fixtures/implementation/code/external.py | 7 + .../fixtures/implementation/code/field.py | 7 + .../fixtures/implementation/code/is_equal.py | 8 + .../fixtures/implementation/code/load.py | 8 + .../fixtures/implementation/code/match.py | 8 + .../fixtures/implementation/code/mul_add.py | 8 + .../fixtures/implementation/code/or_.py | 8 + .../fixtures/implementation/code/span.py | 8 + .../fixtures/implementation/code/store.py | 9 + .../fixtures/implementation/code/test.py | 9 + .../fixtures/implementation/code/update.py | 9 + .../fixtures/implementation/code/value.py | 8 + .../fixtures/implementation/node/__init__.py | 32 + .../fixtures/implementation/node/base.py | 42 ++ .../fixtures/implementation/node/consume.py | 8 + .../fixtures/implementation/node/empty.py | 9 + .../fixtures/implementation/node/error.py | 8 + .../fixtures/implementation/node/invoke.py | 8 + .../fixtures/implementation/node/pause.py | 6 + .../fixtures/implementation/node/sequence.py | 17 + .../fixtures/implementation/node/single.py | 23 + .../fixtures/implementation/node/span_end.py | 8 + .../implementation/node/span_start.py | 8 + .../implementation/node/table_lookup.py | 8 + .../fixtures/implementation/node/unpack.py | 14 + .../implementation/transform/__init__.py | 16 + .../fixtures/implementation/transform/base.py | 14 + .../fixtures/implementation/transform/id.py | 8 + .../implementation/transform/to_lower.py | 8 + .../transform/to_lower_unsafe.py | 8 + tests/frontend/test_container.py | 32 + tests/frontend/test_frontend.py | 142 +++++ tests/test_capi.py | 121 ---- tests/test_compilator.py | 53 -- tests/test_frontend.py | 180 ------ 196 files changed, 5989 insertions(+), 390 deletions(-) create mode 100644 src/llparse/__init__.py create mode 100644 src/llparse/api.py create mode 100644 src/llparse/compiler/__init__.py create mode 100644 src/llparse/compiler/header_builder.py create mode 100644 src/llparse/error.py create mode 100644 src/llparse/ext.py create mode 100644 src/llparse/implementation/__init__.py create mode 100644 src/llparse/implementation/c/__init__.py create mode 100644 src/llparse/implementation/c/code/__init__.py create mode 100644 src/llparse/implementation/c/code/and_.py create mode 100644 src/llparse/implementation/c/code/base.py create mode 100644 src/llparse/implementation/c/code/external.py create mode 100644 src/llparse/implementation/c/code/field.py create mode 100644 src/llparse/implementation/c/code/is_equal.py create mode 100644 src/llparse/implementation/c/code/load.py create mode 100644 src/llparse/implementation/c/code/mul_add.py create mode 100644 src/llparse/implementation/c/code/or_.py create mode 100644 src/llparse/implementation/c/code/store.py create mode 100644 src/llparse/implementation/c/code/test.py create mode 100644 src/llparse/implementation/c/code/update.py create mode 100644 src/llparse/implementation/c/code/value.py create mode 100644 src/llparse/implementation/c/compilation.py create mode 100644 src/llparse/implementation/c/constants.py create mode 100644 src/llparse/implementation/c/helpers/match_sequence.py create mode 100644 src/llparse/implementation/c/node/__init__.py create mode 100644 src/llparse/implementation/c/node/base.py create mode 100644 src/llparse/implementation/c/node/consume.py create mode 100644 src/llparse/implementation/c/node/empty.py create mode 100644 src/llparse/implementation/c/node/error.py create mode 100644 src/llparse/implementation/c/node/invoke.py create mode 100644 src/llparse/implementation/c/node/pause.py create mode 100644 src/llparse/implementation/c/node/sequence.py create mode 100644 src/llparse/implementation/c/node/single.py create mode 100644 src/llparse/implementation/c/node/span_end.py create mode 100644 src/llparse/implementation/c/node/span_start.py create mode 100644 src/llparse/implementation/c/node/table_lookup.py create mode 100644 src/llparse/implementation/c/node/unpack.py create mode 100644 src/llparse/implementation/c/transform/__init__.py create mode 100644 src/llparse/implementation/c/transform/base.py create mode 100644 src/llparse/implementation/c/transform/id.py create mode 100644 src/llparse/implementation/c/transform/to_lower.py create mode 100644 src/llparse/implementation/c/transform/to_lower_unsafe.py create mode 100644 src/llparse_builder/__init__.py create mode 100644 src/llparse_builder/binary_search.py create mode 100644 src/llparse_builder/builder.py create mode 100644 src/llparse_builder/code/__init__.py create mode 100644 src/llparse_builder/code/_and.py create mode 100644 src/llparse_builder/code/_or.py create mode 100644 src/llparse_builder/code/base.py create mode 100644 src/llparse_builder/code/creator.py create mode 100644 src/llparse_builder/code/field.py create mode 100644 src/llparse_builder/code/field_value.py create mode 100644 src/llparse_builder/code/is_equal.py create mode 100644 src/llparse_builder/code/load.py create mode 100644 src/llparse_builder/code/match.py create mode 100644 src/llparse_builder/code/mul_add.py create mode 100644 src/llparse_builder/code/span.py create mode 100644 src/llparse_builder/code/store.py create mode 100644 src/llparse_builder/code/test.py create mode 100644 src/llparse_builder/code/update.py create mode 100644 src/llparse_builder/code/value.py create mode 100644 src/llparse_builder/edge.py create mode 100644 src/llparse_builder/errors.py create mode 100644 src/llparse_builder/loop_checker/__init__.py create mode 100644 src/llparse_builder/loop_checker/lattice.py create mode 100644 src/llparse_builder/node/__init__.py create mode 100644 src/llparse_builder/node/base.py create mode 100644 src/llparse_builder/node/consume.py create mode 100644 src/llparse_builder/node/error.py create mode 100644 src/llparse_builder/node/invoke.py create mode 100644 src/llparse_builder/node/match.py create mode 100644 src/llparse_builder/node/pause.py create mode 100644 src/llparse_builder/node/span_end.py create mode 100644 src/llparse_builder/node/span_start.py create mode 100644 src/llparse_builder/node/unpack.py create mode 100644 src/llparse_builder/property.py create mode 100644 src/llparse_builder/py.typed create mode 100644 src/llparse_builder/reachability.py create mode 100644 src/llparse_builder/span.py create mode 100644 src/llparse_builder/span_allocator.py create mode 100644 src/llparse_builder/transform/__init__.py create mode 100644 src/llparse_builder/transform/base.py create mode 100644 src/llparse_builder/transform/creator.py create mode 100644 src/llparse_builder/transform/to_lower.py create mode 100644 src/llparse_builder/transform/to_lower_unsafe.py create mode 100644 src/llparse_builder/unpack_creator.py create mode 100644 src/llparse_builder/utils.py create mode 100644 src/llparse_frontend/__init__.py create mode 100644 src/llparse_frontend/code/__init__.py create mode 100644 src/llparse_frontend/code/and_.py create mode 100644 src/llparse_frontend/code/base.py create mode 100644 src/llparse_frontend/code/external.py create mode 100644 src/llparse_frontend/code/field.py create mode 100644 src/llparse_frontend/code/field_value.py create mode 100644 src/llparse_frontend/code/is_equal.py create mode 100644 src/llparse_frontend/code/load.py create mode 100644 src/llparse_frontend/code/match.py create mode 100644 src/llparse_frontend/code/mul_add.py create mode 100644 src/llparse_frontend/code/or_.py create mode 100644 src/llparse_frontend/code/span.py create mode 100644 src/llparse_frontend/code/store.py create mode 100644 src/llparse_frontend/code/test.py create mode 100644 src/llparse_frontend/code/update.py create mode 100644 src/llparse_frontend/code/value.py create mode 100644 src/llparse_frontend/container/__init__.py create mode 100644 src/llparse_frontend/container/wrap.py create mode 100644 src/llparse_frontend/enumerator.py create mode 100644 src/llparse_frontend/errors.py create mode 100644 src/llparse_frontend/frontend.py create mode 100644 src/llparse_frontend/implementation/__init__.py create mode 100644 src/llparse_frontend/implementation/code.py create mode 100644 src/llparse_frontend/implementation/full.py create mode 100644 src/llparse_frontend/implementation/node.py create mode 100644 src/llparse_frontend/implementation/transform.py create mode 100644 src/llparse_frontend/implementation/typedefs.py create mode 100644 src/llparse_frontend/namespace/frontend.py create mode 100644 src/llparse_frontend/node/__init__.py create mode 100644 src/llparse_frontend/node/base.py create mode 100644 src/llparse_frontend/node/consume.py create mode 100644 src/llparse_frontend/node/empty.py create mode 100644 src/llparse_frontend/node/error.py create mode 100644 src/llparse_frontend/node/invoke.py create mode 100644 src/llparse_frontend/node/match.py create mode 100644 src/llparse_frontend/node/pause.py create mode 100644 src/llparse_frontend/node/sequence.py create mode 100644 src/llparse_frontend/node/single.py create mode 100644 src/llparse_frontend/node/slot.py create mode 100644 src/llparse_frontend/node/span_end.py create mode 100644 src/llparse_frontend/node/span_start.py create mode 100644 src/llparse_frontend/node/table_lookup.py create mode 100644 src/llparse_frontend/node/unpack.py create mode 100644 src/llparse_frontend/peephole.py create mode 100644 src/llparse_frontend/span_field.py create mode 100644 src/llparse_frontend/transform/__init__.py create mode 100644 src/llparse_frontend/transform/base.py create mode 100644 src/llparse_frontend/transform/id.py create mode 100644 src/llparse_frontend/transform/to_lower.py create mode 100644 src/llparse_frontend/transform/to_lower_unsafe.py create mode 100644 src/llparse_frontend/trie/__init__.py create mode 100644 src/llparse_frontend/trie/empty.py create mode 100644 src/llparse_frontend/trie/node.py create mode 100644 src/llparse_frontend/trie/sequence.py create mode 100644 src/llparse_frontend/trie/single.py create mode 100644 src/llparse_frontend/utils/__init__.py create mode 100644 src/llparse_frontend/utils/identifier.py create mode 100644 src/llparse_frontend/wrap.py create mode 100644 tests/builder/conftest.py create mode 100644 tests/builder/test_binary_search.py create mode 100644 tests/builder/test_builder.py rename tests/{ => builder}/test_loop_checker.py (66%) rename tests/{ => builder}/test_span_allocator.py (89%) create mode 100644 tests/builder/test_unpack.py create mode 100644 tests/builder/test_unpack_creator.py create mode 100644 tests/frontend/__init__.py create mode 100644 tests/frontend/conftest.py create mode 100644 tests/frontend/fixtures/__init__.py create mode 100644 tests/frontend/fixtures/implementation/__init__.py create mode 100644 tests/frontend/fixtures/implementation/code/__init__.py create mode 100644 tests/frontend/fixtures/implementation/code/and_.py create mode 100644 tests/frontend/fixtures/implementation/code/base.py create mode 100644 tests/frontend/fixtures/implementation/code/external.py create mode 100644 tests/frontend/fixtures/implementation/code/field.py create mode 100644 tests/frontend/fixtures/implementation/code/is_equal.py create mode 100644 tests/frontend/fixtures/implementation/code/load.py create mode 100644 tests/frontend/fixtures/implementation/code/match.py create mode 100644 tests/frontend/fixtures/implementation/code/mul_add.py create mode 100644 tests/frontend/fixtures/implementation/code/or_.py create mode 100644 tests/frontend/fixtures/implementation/code/span.py create mode 100644 tests/frontend/fixtures/implementation/code/store.py create mode 100644 tests/frontend/fixtures/implementation/code/test.py create mode 100644 tests/frontend/fixtures/implementation/code/update.py create mode 100644 tests/frontend/fixtures/implementation/code/value.py create mode 100644 tests/frontend/fixtures/implementation/node/__init__.py create mode 100644 tests/frontend/fixtures/implementation/node/base.py create mode 100644 tests/frontend/fixtures/implementation/node/consume.py create mode 100644 tests/frontend/fixtures/implementation/node/empty.py create mode 100644 tests/frontend/fixtures/implementation/node/error.py create mode 100644 tests/frontend/fixtures/implementation/node/invoke.py create mode 100644 tests/frontend/fixtures/implementation/node/pause.py create mode 100644 tests/frontend/fixtures/implementation/node/sequence.py create mode 100644 tests/frontend/fixtures/implementation/node/single.py create mode 100644 tests/frontend/fixtures/implementation/node/span_end.py create mode 100644 tests/frontend/fixtures/implementation/node/span_start.py create mode 100644 tests/frontend/fixtures/implementation/node/table_lookup.py create mode 100644 tests/frontend/fixtures/implementation/node/unpack.py create mode 100644 tests/frontend/fixtures/implementation/transform/__init__.py create mode 100644 tests/frontend/fixtures/implementation/transform/base.py create mode 100644 tests/frontend/fixtures/implementation/transform/id.py create mode 100644 tests/frontend/fixtures/implementation/transform/to_lower.py create mode 100644 tests/frontend/fixtures/implementation/transform/to_lower_unsafe.py create mode 100644 tests/frontend/test_container.py create mode 100644 tests/frontend/test_frontend.py delete mode 100644 tests/test_capi.py delete mode 100644 tests/test_compilator.py delete mode 100644 tests/test_frontend.py diff --git a/pyproject.toml b/pyproject.toml index 5cb9779..28f42d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "llparse" dynamic = ["version"] -description = "A Parody of llparse written for writing C Parsers with Python" +description = "A Parody of typescript llparse written for generating C Parsers using Python" readme = "README.md" authors = [ { name = "Vizonex", email = "VizonexBusiness@gmail.com" } @@ -11,11 +11,18 @@ dependencies = [ "typing_extensions; python_version < '3.13'" ] + [tool.setuptools.dynamic] version = {attr = "llparse.__version__"} [build-system] requires = ["setuptools"] +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "typer>=0.27.0", +] + [tool.ruff] target-version = "py310" diff --git a/src/llparse/__init__.py b/src/llparse/__init__.py new file mode 100644 index 0000000..d2b1bc0 --- /dev/null +++ b/src/llparse/__init__.py @@ -0,0 +1,5 @@ +from .api import LLParse + +__version__ = "1.0.0" + +__all__ = ("LLParse",) diff --git a/src/llparse/api.py b/src/llparse/api.py new file mode 100644 index 0000000..886d468 --- /dev/null +++ b/src/llparse/api.py @@ -0,0 +1,98 @@ +from llparse_builder import builder as source +from llparse_frontend import frontend + +from .compiler import Compiler, CompilerResult + + +class LLParse(source.Builder): + """ + + The prefix controls the names of methods and state struct in generated + public C headers: + + ```c + // state struct + struct PREFIX_t { + ... + } + + int PREFIX_init(PREFIX_t* state); + int PREFIX_execute(PREFIX_t* state, const char p, const char endp); + ``` + """ + + def __init__(self, prefix: str = "llparse") -> None: + """ + :param prefix: Prefix to be used when generating public API default is "llparse". + """ + self.prefix = prefix + super().__init__() + + def get_compiler( + self, + headerGuard: str | None = None, + debug: str | None = None, + max_table_elem_width: int | None = None, + min_table_size: int | None = None, + ) -> Compiler: + return Compiler( + self.prefix, + headerGuard, + debug, + max_table_elem_width + if max_table_elem_width + else frontend.DEFAULT_MAX_TABLE_WIDTH, + min_table_size if min_table_size else frontend.DEFAULT_MIN_TABLE_SIZE, + ) + + def build( + self, + root: source.node.Node, + headerGuard: str | None = None, + debug: str | None = None, + max_table_elem_width: int | None = None, + min_table_size: int | None = None, + header_name: str | None = None, + override_llparse_name: bool = False, + ) -> CompilerResult: + """Builds Graph and then compiles the data into C code , returns with the header and C file inside of a Dataclass""" + + compiler = Compiler( + self.prefix, + headerGuard, + debug, + max_table_elem_width + if max_table_elem_width + else frontend.DEFAULT_MAX_TABLE_WIDTH, + min_table_size if min_table_size else frontend.DEFAULT_MIN_TABLE_SIZE, + ) + + return compiler.compile( + root, + self.properties(), + header_name=header_name, + override_llparse_name=override_llparse_name, + ) + + def to_frontend( + self, + root: source.node.Node, + headerGuard: str | None = None, + debug: str | None = None, + max_table_elem_width: int | None = None, + min_table_size: int | None = None, + ) -> Compiler: + """Used as an external hack to get access to the frontend of llparse and extract + it's contents to compile the libraries you make other things like cython, This is not in llparse + specifically (Yet...)""" + return Compiler( + self.prefix, + headerGuard, + debug, + max_table_elem_width + if max_table_elem_width + else frontend.DEFAULT_MAX_TABLE_WIDTH, + min_table_size if min_table_size else frontend.DEFAULT_MIN_TABLE_SIZE, + ).to_frontend(root, self.properties) + + # capi will return soon... diff --git a/src/llparse/compiler/__init__.py b/src/llparse/compiler/__init__.py new file mode 100644 index 0000000..2165ecd --- /dev/null +++ b/src/llparse/compiler/__init__.py @@ -0,0 +1,78 @@ +from dataclasses import dataclass +from logging import getLogger +from pathlib import Path + +from llparse_builder import builder as source +from llparse_frontend.frontend import Frontend + +from ..implementation import c +from .header_builder import HeaderBuilder + +logger = getLogger() + +debug = logger.debug + + +@dataclass(slots=True) +class CompilerResult: + c: str + """Textual C code""" + header: str + """Textual C header file""" + + def write(self, c: Path | str, header: Path | str) -> None: + """ + Writes the output to the chosen file locations + + :param c: Output for where to write the C File + :type c: Path | str + :param header: Output for where to write the Header File + :type header: Path | str + """ + Path(c).write_text(self.c) + Path(header).write_text(self.header) + + +@dataclass +class Compiler: + prefix: str + header_guard: str | None = None + debug: str | None = None + max_table_elem_width: int | None = None + min_table_size: int | None = None + + def to_frontend( + self, + root: source.node.Node, + properties: list[source.Property], + impl=c, + ): + """compiles up the frontend and brings you back the frontend's results. + I added documentation to this function so that you can do creative things + with the library beyond C...""" + return Frontend( + self.prefix, + impl, + max_table_elem_width=self.max_table_elem_width, + min_table_size=self.min_table_size, + ).compile(root, properties) + + def compile( + self, + root: source.node.Node, + properties: list[source.Property], + header_name: str | None = None, + impl=c, + override_llparse_name: bool = False, + ) -> CompilerResult: + """Creates the C and header file...""" + info = self.to_frontend(root, properties, impl) + hb = HeaderBuilder(self.prefix, self.header_guard, properties, info.spans) + cdata = c.CCompiler(header_name, self.debug).compile(info) + if override_llparse_name: + # sometimes users want to combine parsers together when compiling with C + # to make up for conflicts with other parsers example: llhttp + # there should be a fair way of compiling everything. + cdata = cdata.replace("llparse", self.prefix) + + return CompilerResult(cdata, hb.build()) diff --git a/src/llparse/compiler/header_builder.py b/src/llparse/compiler/header_builder.py new file mode 100644 index 0000000..25ca115 --- /dev/null +++ b/src/llparse/compiler/header_builder.py @@ -0,0 +1,75 @@ +from dataclasses import dataclass, field + +from llparse_builder.builder import Property +from llparse_frontend.span_field import SpanField + +from ..error import Error + +TYPE_LOOKUP = { + "i8": "uint8_t", + "i16": "uint16_t", + "i32": "uint32_t", + "i64": "uint64_t", + "ptr": "void*", +} + + +@dataclass(slots=True) +class HeaderBuilder: + prefix: str + header_guard: str | None = field(default=None) + properties: list[Property] = field(default_factory=list) + spans: list[SpanField] = field(default_factory=list) + + def build(self) -> str: + """Builds The string to create the header file""" + res = "" + PREFIX = self.prefix.upper() + DEFINE = f"INCLUDE_{PREFIX}_H_" if not self.header_guard else self.header_guard + + res += f"#ifndef {DEFINE}\n" + res += f"#define {DEFINE}\n" + res += "#ifdef __cplusplus\n" + res += 'extern "C" {\n' + res += "#endif\n" + res += "\n" + + res += "#include \n" + res += "\n" + + # Main Structure + res += f"typedef struct {self.prefix}_s {self.prefix}_t;\n" + res += f"struct {self.prefix}_s " + "{\n" + res += " int32_t _index;\n" + + for index, f in enumerate(self.spans): + res += f" void* _span_pos{index};\n" + if len(f.callbacks) > 1: + res += f" void* _span_cb{index};\n" + + # TODO: Reorganize fields for better heap/memory and performance. + res += " int32_t error;\n" + res += " const char* reason;\n" + res += " const char* error_pos;\n" + res += " void* data;\n" + res += " void* _current;\n" + + for prop in self.properties: + if not (ty := TYPE_LOOKUP.get(prop.ty)): + raise Error(f'Unknown state property type: "{prop.ty}"') + + res += f" {ty} {prop.name};\n" + res += "};" + + res += "\n" + + res += f"int {self.prefix}_init({self.prefix}_t* s);\n" + res += f"int {self.prefix}_execute({self.prefix}_t* s, const char* p, const char* endp);\n" + + res += "\n" + + res += "#ifdef __cplusplus\n" + res += '} /* extern "C" */\n' + res += "#endif\n" + res += f"#endif /* {DEFINE} */" + return res diff --git a/src/llparse/error.py b/src/llparse/error.py new file mode 100644 index 0000000..66a4b7d --- /dev/null +++ b/src/llparse/error.py @@ -0,0 +1,2 @@ +class Error(Exception): + """llparse compiler-related error""" diff --git a/src/llparse/ext.py b/src/llparse/ext.py new file mode 100644 index 0000000..f9bad46 --- /dev/null +++ b/src/llparse/ext.py @@ -0,0 +1,90 @@ +from collections.abc import Generator +from contextlib import contextmanager +from io import StringIO + + +# Modified from Cython's version with a much smoother system to utilize. +class LinesResult: + __slots__ = ("io",) + + def __init__(self, io: StringIO | None = None): + self.io = io or StringIO() + + def put(self, s: str) -> None: + self.io.write(s) + + def newline(self) -> None: + self.io.write("\n") + + def putline(self, s: str) -> None: + self.io.write(s) + self.io.write("\n") + + def __str__(self): + return self.io.getvalue() + + +# Based off Cython's CodeWriter module in the DeclarationWriter class +class Writer: + """A Simplistic Code Writer tool for outputting + and writing clean code. It also allows users to cutomize + how big the indent size of the output should be.""" + + __slots__ = ("_indent_size", "_indent_str", "_numindents", "_result") + + def __init__(self, indent_size: int = 2, result: LinesResult | None = None): + if indent_size < 1: + raise ValueError("Indent size requires a number at least greater than 1.") + self._indent_size = indent_size + self._result = result if result is not None else LinesResult() + self._numindents = 0 + self._indent_str = " " * indent_size + + def indent(self) -> None: + self._numindents += 1 + + def dedent(self) -> None: + if self._numindents < 0: + raise RuntimeError("number of indents is out of bounds.") + self._numindents -= 1 + + def startline(self, s: str = "") -> None: + self._result.put(self._indent_str * self._numindents + s) + + def put(self, s: str) -> None: + self._result.put(s) + + def putline(self, s: str) -> None: + self._result.putline(self._indent_str * self._numindents + s) + + def putline_indented(self, s: str) -> None: + with self.tab(): + self.putline(s) + + def endline(self, s: str = "") -> None: + self._result.putline(s) + + def line(self, s: str) -> None: + self.startline(s) + self.endline() + + @contextmanager + def tab(self) -> Generator[None, None, None]: + """Indents and later dedents the number of indents + used on each given line as a context manager.""" + self.indent() + yield + self.dedent() + + def skipline(self) -> None: + self.endline() + + def skiplines(self, amount: int = 2): + if amount < 1: + raise RuntimeError("skiplines amount must be greater or equal to 1") + for _ in range(amount): + self.skipline() + + def result(self) -> str: + """Obtains the written data from the writer""" + return self._result.io.getvalue() diff --git a/src/llparse/implementation/__init__.py b/src/llparse/implementation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/llparse/implementation/c/__init__.py b/src/llparse/implementation/c/__init__.py new file mode 100644 index 0000000..eda0136 --- /dev/null +++ b/src/llparse/implementation/c/__init__.py @@ -0,0 +1,196 @@ +from llparse_frontend.frontend import FrontendResult + +from .compilation import Compilation, CompilationOptions +from .constants import * + + +class CCompiler: + """The Final HeadPeice where the Main C-Code gets compiled to...""" + + def __init__(self, header: str | None = None, debug: str | None = None) -> None: + # NOTE Unlike in typescript llparse Containers are not Required since I'm using a different methoad to translate those parts... + self.options = CompilationOptions(debug, header) + + def compile(self, info: FrontendResult): + compilation = Compilation( + info.prefix, + info.properties, + list(info.resumption_targets), + options=self.options, + ) + + out: list[str] = [] + + out.append("#include ") + out.append("#include ") + out.append("#include ") + out.append("") + # Seems LLParse was updated from /* UNREACHABLE */ abort(); to a Macro, Intresting... + out.append("#ifdef __SSE4_2__") + out.append(" #ifdef _MSC_VER") + out.append(" #include ") + out.append(" #else /* !_MSC_VER */") + out.append(" #include ") + out.append(" #endif /* _MSC_VER */") + out.append("#endif /* __SSE4_2__ */") + out.append("") + + out.append("#ifdef __ARM_NEON__") + out.append(" #include ") + out.append("#endif /* __ARM_NEON__ */") + out.append("") + + out.append("#ifdef __wasm__") + out.append(" #include ") + out.append("#endif /* __wasm__ */") + out.append("") + + out.append("#ifdef _MSC_VER") + out.append(" #define ALIGN(n) _declspec(align(n))") + out.append(" #define UNREACHABLE __assume(0)") + out.append("#else /* !_MSC_VER */") + out.append(" #define ALIGN(n) __attribute__((aligned(n)))") + out.append(" #define UNREACHABLE __builtin_unreachable()") + out.append("#endif /* _MSC_VER */") + + out.append("") + out.append( + f'#include "{self.options.header if self.options.header else info.prefix}.h"' + ) + out.append("") + out.append(f"typedef int (*{info.prefix}__span_cb)(") + out.append(f" {info.prefix}_t*, const char*, const char*);") + out.append("") + + # Start Queuing span callbacks + # otherwise we will have nothing + # but mess which is not what we want - Vizonex + compilation.reserve_spans(info.spans) + + rootState = compilation.unwrap_node(info.root) + rootName = rootState.build(compilation) + # Bring in the rest of the variables... + compilation.build_globals(out) + out.append("") + + out.append(f"int {info.prefix}_init({info.prefix}_t* {ARG_STATE}) " + "{") + out.append(f" memset({ARG_STATE}, 0, sizeof(*{ARG_STATE}));") + out.append(f" {ARG_STATE}->_current = (void*) (intptr_t) {rootName};") + out.append(" return 0;") + out.append("}") + out.append("") + + # TODO (Vizonex) Make llparse_state_t's Name Optional and alterable incase mixed with + # llhttp or another parser + out.append(f"static llparse_state_t {info.prefix}__run(") + out.append(f" {info.prefix}_t* {ARG_STATE},") + out.append(f" const unsigned char* {ARG_POS},") + out.append(f" const unsigned char* {ARG_ENDPOS}) " + "{") + out.append(f" int {VAR_MATCH};") + out.append( + " switch ((llparse_state_t) (intptr_t) " + + f"{compilation.current_field()}) " + + "{" + ) + + # Now build resumption states... These are states what will have a 'case block' next to them... + # However I'm not refering to the characters those will be handles in thier inner switches, + # I'm talking about the major states... + tmp = [] + compilation.build_resumption_states(tmp) + compilation.indent(out, tmp, " ") + + # Final Resumption State... Very important! + out.append(" default:") + out.append(" UNREACHABLE;") + out.append(" }") + + tmp = [] + compilation.build_internal_states(tmp) + compilation.indent(out, tmp, " ") + + out.append("}") + out.append("") + + out.append( + f"int {info.prefix}_execute({info.prefix}_t* {ARG_STATE}, " + + f"const char* {ARG_POS}, const char* {ARG_ENDPOS}) " + + "{" + ) + out.append(" llparse_state_t next;") + out.append("") + + out.append(" /* check lingering errors */") + out.append(f" if ({compilation.error_field()} != 0) " + "{") + out.append(f" return {compilation.error_field()};") + out.append(" }") + out.append("") + + tmp = [] + self.restart_spans(compilation, info, tmp) + compilation.indent(out, tmp, " ") + args = [ + compilation.state_arg(), + f"(const unsigned char*) {compilation.pos_arg()}", + f"(const unsigned char*) {compilation.end_pos_arg()}", + ] + out.append(f" next = {info.prefix}__run({(', ').join(args)});") + out.append(f" if (next == {STATE_ERROR}) " + "{") + out.append(f" return {compilation.error_field()};") + out.append(" }") + out.append(f" {compilation.current_field()} = (void*) (intptr_t) next;") + out.append("") + + tmp = [] + self.execute_spans(compilation, info, tmp) + compilation.indent(out, tmp, " ") + + out.append(" return 0;") + out.append("}") + + # JOIN ALL OF THEM! + return "\n".join(out) + + def restart_spans(self, ctx: Compilation, info: FrontendResult, out: list[str]): + if not info.spans: + return + + out.append("/* restart spans */") + for span in info.spans: + pos_field = ctx.span_pos_field(span.index) + + out.append(f"if ({pos_field} != NULL) " + "{") + out.append(f" {pos_field} = (void*) {ctx.pos_arg()};") + out.append("}") + out.append("") + + def execute_spans(self, ctx: Compilation, info: FrontendResult, out: list[str]): + if not info.spans: + return + + out.append("/* execute spans */") + for span in info.spans: + pos_field = ctx.span_pos_field(span.index) + + if len(span.callbacks) == 1: + callback = ctx.build_code(ctx.unwrap_code(span.callbacks[0])) + + else: + callback = ( + f"(({info.prefix}__span_cb)" + ctx.span_cb_field(span.index) + ")" + ) + + args = [ctx.state_arg(), pos_field, f"(const char*) {ctx.end_pos_arg()}"] + + out.append(f"if ({pos_field} != NULL) " + "{") + out.append(" int error;") + out.append("") + out.append(f" error = {callback}({', '.join(args)});") + + out.append(" if (error != 0) {") + out.append(f" {ctx.error_field()} = error;") + out.append(f" {ctx.error_pos_field()} = {ctx.end_pos_arg()};") + out.append(" return error;") + out.append(" }") + out.append("}") + out.append("") diff --git a/src/llparse/implementation/c/code/__init__.py b/src/llparse/implementation/c/code/__init__.py new file mode 100644 index 0000000..83e3433 --- /dev/null +++ b/src/llparse/implementation/c/code/__init__.py @@ -0,0 +1,32 @@ +# This code was autogenerated from tools/generate_impl.py +# It is used to handle code generation for new language implementations +# And gives a useful template for the list of nodes and code objects needed. +# Editing is fine but use tools/generate_impl.py at your own risk. + +from .and_ import And +from .base import Code +from .external import External +from .field import Field +from .is_equal import IsEqual +from .load import Load +from .mul_add import MulAdd +from .or_ import Or +from .store import Store +from .test import Test +from .update import Update +from .value import Value + +__all__ = ( + "And", + "Code", + "External", + "Field", + "IsEqual", + "Load", + "MulAdd", + "Or", + "Store", + "Test", + "Update", + "Value", +) diff --git a/src/llparse/implementation/c/code/and_.py b/src/llparse/implementation/c/code/and_.py new file mode 100644 index 0000000..3e17d27 --- /dev/null +++ b/src/llparse/implementation/c/code/and_.py @@ -0,0 +1,10 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .field import Field + + +class And(Field[frontend.And]): + def build(self, ctx: "compilation.Compilation", out: list[str]) -> None: + out.append(f"{self.field(ctx)} &= {self.ref.value};") + out.append("return 0;") diff --git a/src/llparse/implementation/c/code/base.py b/src/llparse/implementation/c/code/base.py new file mode 100644 index 0000000..6aeae66 --- /dev/null +++ b/src/llparse/implementation/c/code/base.py @@ -0,0 +1,16 @@ +from abc import ABC, abstractmethod +from typing import Generic, TypeVar + +from .. import compilation + +T = TypeVar("T") + + +class Code(ABC, Generic[T]): + __slots__ = ("ref",) + + def __init__(self, ref: T) -> None: + self.ref = ref + + @abstractmethod + def build(self, ctx: "compilation.Compilation", out: list[str]) -> None: ... diff --git a/src/llparse/implementation/c/code/external.py b/src/llparse/implementation/c/code/external.py new file mode 100644 index 0000000..cacc307 --- /dev/null +++ b/src/llparse/implementation/c/code/external.py @@ -0,0 +1,15 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .base import Code + + +class External(Code[frontend.External]): + def build(self, ctx: "compilation.Compilation", out: list[str]) -> None: + out.append(f"int {self.ref.name}(") + out.append(f" {ctx.prefix}_t* s, const unsigned char* p,") + if self.ref.signature == "value": + out.append(" const unsigned char* endp,") + out.append(" int value);") + else: + out.append(" const unsigned char* endp);") diff --git a/src/llparse/implementation/c/code/field.py b/src/llparse/implementation/c/code/field.py new file mode 100644 index 0000000..5b5e918 --- /dev/null +++ b/src/llparse/implementation/c/code/field.py @@ -0,0 +1,27 @@ +from abc import abstractmethod + +from .. import compilation +from .base import Code, T + + +class Field(Code[T]): + def build(self, ctx: "compilation.Compilation", out: list[str]) -> None: + out.append(f"int {self.ref.name}(") + out.append(f" {ctx.prefix}_t* {ctx.state_arg()},") + out.append(f" const unsigned char* {ctx.pos_arg()},") + if self.ref.signature == "value": + out.append(f" const unsigned char* {ctx.end_pos_arg()},") + out.append(f" int {ctx.match_var()}) {{") + else: + out.append(f" const unsigned char* {ctx.end_pos_arg()}) {{") + + tmp = [] + self.do_build(ctx, tmp) + ctx.indent(out, tmp, " ") + out.append("}") + + @abstractmethod + def do_build(self, ctx: "compilation.Compilation", out: list[str]) -> None: ... + + def field(self, ctx: "compilation.Compilation") -> str: + return f"{ctx.state_arg()}->{self.ref.field}" diff --git a/src/llparse/implementation/c/code/is_equal.py b/src/llparse/implementation/c/code/is_equal.py new file mode 100644 index 0000000..9a41823 --- /dev/null +++ b/src/llparse/implementation/c/code/is_equal.py @@ -0,0 +1,9 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .field import Field + + +class IsEqual(Field[frontend.IsEqual]): + def do_build(self, ctx: "compilation.Compilation", out: list[str]) -> None: + out.append(f"return {self.field(ctx)} == {self.ref.value};") diff --git a/src/llparse/implementation/c/code/load.py b/src/llparse/implementation/c/code/load.py new file mode 100644 index 0000000..c655b83 --- /dev/null +++ b/src/llparse/implementation/c/code/load.py @@ -0,0 +1,9 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .field import Field + + +class Load(Field[frontend.Load]): + def do_build(self, ctx: "compilation.Compilation", out: list[str]) -> None: + out.append(f"return {self.field(ctx)};") diff --git a/src/llparse/implementation/c/code/mul_add.py b/src/llparse/implementation/c/code/mul_add.py new file mode 100644 index 0000000..7583d9a --- /dev/null +++ b/src/llparse/implementation/c/code/mul_add.py @@ -0,0 +1,63 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from ..constants import SIGNED_LIMITS, SIGNED_TYPES, UNSIGNED_LIMITS +from .field import Field + + +class MulAdd(Field[frontend.MulAdd]): + def do_build(self, ctx: "compilation.Compilation", out: list[str]) -> None: + options = self.ref.options + ty = ctx.get_field_type(self.ref.field) + field = self.field(ctx) + + if options.signed: + assert ty in SIGNED_TYPES, f'Unexpected mulAdd type "{ty}"' + target_type = SIGNED_TYPES[ty] + out.append(f"{target_type}* field = ({target_type}*) &{field};") + field = "(*field)" + + _match = ctx.match_var() + + limits = SIGNED_LIMITS if options.signed else UNSIGNED_LIMITS + assert ty in limits, f'Unexpected mulAdd type "{ty}"' + _min, _max = limits.get(ty) + + mul_max = f"{_max} / {options.base}" + mul_min = f"{_min} / {options.base}" + + out.append("/* Multiplication overflow */") + out.append(f"if ({field} > {mul_max}) {{") + out.append(" return 1;") + out.append("}") + if options.signed: + out.append(f"if ({field} < {mul_min}) {{") + out.append(" return 1;") + out.append("}") + + out.append("") + + out.append(f"{field} *= {options.base};{{") + out.append("") + + out.append("/* Addition overflow */") + out.append(f"if ({_match} >= 0) {{") + out.append(f" if ({field} > {_max} - {_match}) {{") + out.append(" return 1;") + out.append(" }") + out.append("} else {") + out.append(f" if ({field} < {_min} - {_match}) {{") + out.append(" return 1;") + out.append(" }") + out.append("}") + + out.append(f"{field} += {_match};{{") + + if options.max is not None: + out.append("") + out.append("/* Enforce maximum */") + out.append(f"if ({field} > {options.max}) {{") + out.append(" return 1;") + out.append("}") + + out.append("return 0;") diff --git a/src/llparse/implementation/c/code/or_.py b/src/llparse/implementation/c/code/or_.py new file mode 100644 index 0000000..5ffe20d --- /dev/null +++ b/src/llparse/implementation/c/code/or_.py @@ -0,0 +1,10 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .field import Field + + +class Or(Field[frontend.Or]): + def do_build(self, ctx: "compilation.Compilation", out: list[str]): + out.append(f"{self.field(ctx)} |= {self.ref.value};") + out.append("return 0;") diff --git a/src/llparse/implementation/c/code/store.py b/src/llparse/implementation/c/code/store.py new file mode 100644 index 0000000..6b6494d --- /dev/null +++ b/src/llparse/implementation/c/code/store.py @@ -0,0 +1,10 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .field import Field + + +class Store(Field[frontend.Store]): + def do_build(self, ctx: "compilation.Compilation", out: list[str]): + out.append(f"{self.field(ctx)} = {ctx.match_var()};") + out.append("return 0;") diff --git a/src/llparse/implementation/c/code/test.py b/src/llparse/implementation/c/code/test.py new file mode 100644 index 0000000..6f6ec64 --- /dev/null +++ b/src/llparse/implementation/c/code/test.py @@ -0,0 +1,10 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .field import Field + + +class Test(Field[frontend.Test]): + def do_build(self, ctx: "compilation.Compilation", out: list[str]): + value = self.ref.value + out.append(f"return ({self.field(ctx)} & {value}) == {value};") diff --git a/src/llparse/implementation/c/code/update.py b/src/llparse/implementation/c/code/update.py new file mode 100644 index 0000000..083822b --- /dev/null +++ b/src/llparse/implementation/c/code/update.py @@ -0,0 +1,10 @@ +from llparse_frontend import code as frontend + +from .. import compilation +from .field import Field + + +class Update(Field[frontend.Update]): + def do_build(self, ctx: "compilation.Compilation", out: list[str]): + out.append(f"{self.field(ctx)} = {self.ref.value};") + out.append("return 0;") diff --git a/src/llparse/implementation/c/code/value.py b/src/llparse/implementation/c/code/value.py new file mode 100644 index 0000000..f2bb728 --- /dev/null +++ b/src/llparse/implementation/c/code/value.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from ....ext import Writer +from .base import Code + + +class Value(Code[frontend.Value]): + def build(self, out: Writer) -> None: ... diff --git a/src/llparse/implementation/c/compilation.py b/src/llparse/implementation/c/compilation.py new file mode 100644 index 0000000..10df2f8 --- /dev/null +++ b/src/llparse/implementation/c/compilation.py @@ -0,0 +1,287 @@ +from dataclasses import dataclass +from typing import cast + +from llparse_frontend.namespace import frontend +from llparse_frontend.span_field import SpanField +from llparse_frontend.wrap import Wrap + +from ...error import Error +from .code import Code +from .constants import ( + ARG_ENDPOS, + ARG_POS, + ARG_STATE, + BLOB_PREFIX, + LABEL_PREFIX, + STATE_ERROR, + STATE_PREFIX, + VAR_MATCH, +) +from .helpers.match_sequence import MatchSequence +from .node import Node +from .transform import Transform + +# XXX: Should this belong in constants??? +BLOB_GROUP_SIZE = 11 + + +@dataclass +class Blob: + alignment: int | None + buffer: bytes + name: str + + +@dataclass +class CompilationOptions: + debug: str | None = None + header: str | None = None + + +@dataclass +class CompilationProperty: + name: str + ty: str + + +class Compilation: + def __init__( + self, + prefix: str, + properties: list[CompilationProperty], + resumption_targets: set[Wrap[frontend.node.Node]], + options: CompilationOptions, + ) -> None: + self.prefix = prefix + self.properties = properties + self.options = options + + # Custom optimization for llparse-typescript this is recommended + self.property_map = {p.name: p.ty for p in self.properties} + + self.state_map: dict[str, list[str]] = {} + self.blobs: dict[bytes, Blob] = {} + self.code_map: dict[str, Code[frontend.code.Code]] = {} + self.match_sequence: dict[str, MatchSequence] = {} + self.resumption_targets: set[str] = set() + + for node in resumption_targets: + self.resumption_targets.add(STATE_PREFIX + node.ref.id.name) + + def build_state_enum(self, out: list[str]) -> None: + # NOTE: These values will never cause compilation issues. + out.append("enum llparse_state_e {") + out.append(f" {STATE_ERROR},") + for state_name in self.state_map: + # if state_name in self.resumption_targets: + out.append(f" {state_name},") + + out.append("};") + out.append("typedef enum llparse_state_e llparse_state_t;") + + def build_blobs(self, out: list[str]) -> None: + if not self.blobs: + return + for blob in self.blobs.values(): + buffer = blob.buffer + align = "" + if blob.alignment: + # TODO: (Vizonex) LLParse Typescript PR + # join blob.argument to: + # if (blob.argument){ + # align = ` ALIGN(${blob.alignment})`; + # out.push('#ifdef __SSE4_2__'); + align = f" ALIGN({blob.alignment})" + out.append("#ifdef __SSE4_2__") + + # NOTE: {{}} is a bracket escape. + out.append(f"static const unsigned char{align} {blob.name}[] = {{") + buffer_len = len(buffer) + for i in range(0, BLOB_GROUP_SIZE, buffer_len): + limit = min(len(buffer), i + BLOB_GROUP_SIZE) + hx: list[str] = [] + + for j in range(i, limit): + value = buffer[j] + assert value is not None + hx.append(self.to_char(value)) + + line = " " + ", ".join(hx) + if limit != buffer_len: + line += "," + out.append(line) + + out.append("};") + if blob.alignment: + out.append("#endif /* __SSE4_2__ */") + out.append("") + + def build_match_sequence(self, out: list[str]) -> None: + if not self.match_sequence: + return + + MatchSequence.build_globals(out) + out.append("") + + for _match in self.match_sequence.values(): + _match.build(self, out) + out.append("") + + def reserve_spans(self, spans: list[SpanField]) -> None: + for span in spans: + for callback in span.callbacks: + self.build_code(self.unwrap_code(callback)) + + def debug(self, out: list[str], message: str) -> None: + if self.options.debug is None: + return + + args = [ + self.state_arg(), + f"(const char*) {self.pos_arg()}", + f"(const char*) {self.end_pos_arg()}", + ] + + out.append(f"{self.options.debug}({', '.join(args)}),") + out.append(f" {self.cstring(message)});") + + def build_globals(self, out: list[str]) -> None: + if self.options.debug != None: + out.append(f"void {self.options.debug}(") + out.append(f" {self.prefix}_t* s, const char* p, const char* endp,") + out.append(" const char* msg);") + + self.build_blobs(out) + self.build_match_sequence(out) + self.build_state_enum(out) + + for code in self.code_map.values(): + out.append("") + code.build(self, out) + + def build_resumption_states(self, out: list[str]) -> None: + for name, lines in self.state_map.items(): + if name not in self.resumption_targets: + continue + out.append(f"case {name}:") + out.append(f"{LABEL_PREFIX}{name}: {{") + for line in lines: + out.append(f" {line}") + out.append(" UNREACHABLE;") + out.append("}") + + def build_internal_states(self, out: list[str]) -> None: + for name, lines in self.state_map.items(): + if name in self.resumption_targets: + continue + out.append(f"{LABEL_PREFIX}{name}: {{") + for line in lines: + out.append(f" {line}") + + out.append(" UNREACHABLE;") + out.append("}") + + def add_state(self, state: str, lines: list[str]): + assert state not in self.state_map + self.state_map[state] = lines + + def build_code(self, code: Code[frontend.code.Code]) -> str: + if code.ref.name in self.code_map: + assert self.code_map[code.ref.name] == code, ( + f'Code name conflict for "{code.ref.name}"' + ) + else: + self.code_map[code.ref.name] = code + return code.ref.name + + def get_field_type(self, field: str) -> str: + try: + return self.property_map[field] + except KeyError: + raise Error(f'Field "{field}" not found') + + def unwrap_code(self, code: Wrap[frontend.code.Code]) -> Code[frontend.code.Code]: + # XXX: for now this is broken so we just straight up wrap the implementation used. + return cast(Code[frontend.code.Code], code) + + def unwrap_node(self, node: Wrap[frontend.node.Node]) -> Node[frontend.node.Node]: + # XXX: for now this is broken so we just straight up wrap the implementation used. + return cast(Node[frontend.node.Node], node) + + def unwrap_transform( + self, transform: Wrap[frontend.transform.Transform] + ) -> Transform[frontend.transform.Transform]: + return cast(Transform[frontend.transform.Transform], transform) + + def indent(self, out: list[str], lines: list[str], pad: str): + for line in lines: + out.append(f"{pad}{line}") + + def get_match_sequence( + self, transform: Wrap[frontend.transform.Transform], select: bytes + ): + wrap = self.unwrap_transform(transform) + if res := self.match_sequence.get(wrap.ref.name): + return res.get_name() + else: + res = MatchSequence(wrap) + self.match_sequence[wrap.ref.name] = res + return res.get_name() + + def state_arg(self) -> str: + return ARG_STATE + + def pos_arg(self) -> str: + return ARG_POS + + def end_pos_arg(self) -> str: + return ARG_ENDPOS + + def match_var(self) -> str: + return VAR_MATCH + + def index_field(self) -> str: + return self.state_field("_index") + + def current_field(self) -> str: + return self.state_field("_current") + + def reason_field(self) -> str: + return self.state_field("reason") + + def error_field(self) -> str: + return self.state_field("error") + + def error_pos_field(self) -> str: + return self.state_field("error_pos") + + def span_pos_field(self, index: int) -> str: + return self.state_field(f"_span_pos{index}") + + def span_cb_field(self, index: int) -> str: + return self.state_field(f"_span_cb{index}") + + def state_field(self, name: str) -> str: + return f"{self.state_arg()}->{name}" + + def cstring(self, value: str): + return f'"{value}"' + + def blob(self, value: bytes, alignment: int | None = None) -> str: + if value in self.blobs: + return self.blobs[value].name + + res = f"{BLOB_PREFIX}{len(self.blobs)}" + self.blobs[value] = Blob(alignment, buffer=value, name=res) + return res + + def to_char(self, value: int) -> str: + ch = chr(value) + if value in (0x27, 0x5C): + return f"'\\{ch}'" + elif value >= 0x20 and value <= 0x7E: + return f"\'{ch}\'" + else: + # NOTE: 0x prefix is already in python. + # No need to format like how typescript does it. + return hex(value) diff --git a/src/llparse/implementation/c/constants.py b/src/llparse/implementation/c/constants.py new file mode 100644 index 0000000..f9f8cb2 --- /dev/null +++ b/src/llparse/implementation/c/constants.py @@ -0,0 +1,42 @@ +CONTAINER_KEY = "c" +"""Unused due to brokenness of containers currently.""" + +LABEL_PREFIX = "" +STATE_PREFIX = "s_n_" +STATE_ERROR = "s_error" + +BLOB_PREFIX = "llparse_blob" + +ARG_STATE = "state" +ARG_POS = "p" +ARG_ENDPOS = "endp" + +VAR_MATCH = "match" + +SEQUENCE_COMPLETE = "kMatchComplete" +SEQUENCE_MISMATCH = "kMatchMismatch" +SEQUENCE_PAUSE = "kMatchPause" + +SIGNED_LIMITS = { + "i8": ("-0x80", "0x7f"), + "i16": ("-0x8000", "0x7fff"), + "i32": ("(-0x7fffffff - 1)", "0x7fffffff"), + "i64": ("(-0x7fffffffffffffffLL - 1)", "0x7fffffffffffffffLL"), +} + +UNSIGNED_LIMITS = { + "i8": ("0", "0xff"), + "i16": ("0", "0xffff"), + "i32": ("0", "0xffffffff"), + "i64": ("0ULL", "0xffffffffffffffffULL"), +} + + +UNSIGNED_TYPES = { + "i8": "uint8_t", + "i16": "uint16_t", + "i32": "uint32_t", + "i64": "uint64_t", +} + +SIGNED_TYPES = {"i8": "int8_t", "i16": "int16_t", "i32": "int32_t", "i64": "int64_t"} diff --git a/src/llparse/implementation/c/helpers/match_sequence.py b/src/llparse/implementation/c/helpers/match_sequence.py new file mode 100644 index 0000000..6774fda --- /dev/null +++ b/src/llparse/implementation/c/helpers/match_sequence.py @@ -0,0 +1,69 @@ +from dataclasses import dataclass + +from llparse_frontend.namespace import frontend + +from .. import compilation # prevent looped imports +from ..constants import SEQUENCE_COMPLETE, SEQUENCE_MISMATCH, SEQUENCE_PAUSE +from ..transform import Transform + + +@dataclass(slots=True) +class MatchSequence: + transform: Transform[frontend.transform.Transform] + + @staticmethod + def build_globals(out: list[str]) -> None: + out.append("enum llparse_match_status_e {") + out.append(f" {SEQUENCE_COMPLETE},") + out.append(f" {SEQUENCE_PAUSE},") + out.append(f" {SEQUENCE_MISMATCH}") + out.append("};") + out.append("typedef enum llparse_match_status_e llparse_match_status_t;") + out.append("") + out.append("struct llparse_match_s {") + out.append(" llparse_match_status_t status;") + out.append(" const unsigned char* current;") + out.append("};") + out.append("typedef struct llparse_match_s llparse_match_t;") + + def get_name(self) -> str: + return f"llparse__match_sequence_{self.transform.ref.name}" + + def build(self, ctx: "compilation.Compilation", out: list[str]): + out.append(f"static llparse_match_t {self.get_name()}(") + out.append(f" {ctx.prefix}_t* s, const unsigned char* p,") + out.append(" const unsigned char* endp,") + out.append(" const unsigned char* seq, uint32_t seq_len) {") + + # Vars + out.append(" uint32_t index;") + out.append(" llparse_match_t res;") + out.append("") + + out.append(" index = s->_index;") + out.append(" for (; p != endp; p++) {") + out.append(" unsigned char current;") + out.append("") + + out.append(f" current = {self.transform.build(ctx, '*p')};") + out.append(" if (current == seq[index]) {") + out.append(" if (++index == seq_len) {") + out.append(f" res.status = {SEQUENCE_COMPLETE};") + out.append(" goto reset;") + out.append(" }") + out.append(" } else {") + out.append(f" res.status = {SEQUENCE_MISMATCH};") + out.append(" goto reset;") + out.append(" }") + out.append(" }") + + out.append(" s->_index = index;") + out.append(f" res.status = {SEQUENCE_PAUSE};") + out.append(" res.current = p;") + out.append(" return res;") + + out.append("reset:") + out.append(" s->_index = 0;") + out.append(" res.current = p;") + out.append(" return res;") + out.append("}") diff --git a/src/llparse/implementation/c/node/__init__.py b/src/llparse/implementation/c/node/__init__.py new file mode 100644 index 0000000..a85b3e6 --- /dev/null +++ b/src/llparse/implementation/c/node/__init__.py @@ -0,0 +1,33 @@ +# This code was autogenerated from tools/generate_impl.py +# It is used to handle code generation for new language implementations +# And gives a useful template for the list of nodes and code objects needed. +# Editing is fine but use tools/generate_impl.py at your own risk. + +from .base import Node +from .consume import Consume +from .empty import Empty +from .error import Error, ErrorNode +from .invoke import Invoke +from .pause import Pause +from .sequence import Sequence +from .single import Single +from .span_end import SpanEnd +from .span_start import SpanStart +from .table_lookup import TableLookup +from .unpack import Unpack + +__all__ = ( + "Consume", + "Empty", + "Error", + "ErrorNode", + "Invoke", + "Node", + "Pause", + "Sequence", + "Single", + "SpanEnd", + "SpanStart", + "TableLookup", + "Unpack", +) diff --git a/src/llparse/implementation/c/node/base.py b/src/llparse/implementation/c/node/base.py new file mode 100644 index 0000000..f598259 --- /dev/null +++ b/src/llparse/implementation/c/node/base.py @@ -0,0 +1,77 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Generic, TypeVar + +from llparse_frontend.namespace import frontend +from llparse_frontend.wrap import Wrap + +from .. import compilation as comp # Don't let python freak out +from ..constants import LABEL_PREFIX, STATE_PREFIX + +T = TypeVar("T") + + +@dataclass +class NodeEdge: + node: Wrap[frontend.node.Node] + no_advance: bool + value: int | None = None + + +class Node(ABC, Generic[T]): + __slots__ = ("cached_decl", "priv_compilation", "ref") + + def __init__(self, ref: T) -> None: + self.ref = ref + self.cached_decl = None + self.priv_compilation: comp.Compilation | None = None + + def build(self, compilation: "comp.Compilation") -> str: + if self.cached_decl is not None: + return self.cached_decl + + res = STATE_PREFIX + self.ref.id.name + self.cached_decl = res + out = [] + + compilation.debug( + out, f'Entering node "{self.ref.id.original_name}" ("{self.ref.id.name}")' + ) + self.priv_compilation = compilation + self.do_build(out) + compilation.add_state(res, out) + + return res + + @property + def compilation(self): + assert self.priv_compilation + return self.priv_compilation + + def prologue(self, out: list[str]): + ctx = self.compilation + + out.append(f"if ({ctx.pos_arg()} == {ctx.end_pos_arg()}) {{") + tmp: list[str] = [] + self.pause(tmp) + self.compilation.indent(out, tmp, " ") + + out.append("}") + + def pause(self, out: list[str]) -> None: + out.append(f"return {self.cached_decl};") + + def tail_to(self, out: list[str], edge: NodeEdge) -> None: + ctx = self.compilation + target = ctx.unwrap_node(edge.node).build(ctx) + + if edge.no_advance: + out.append(f"{ctx.pos_arg()}++;") + + if edge.value is not None: + out.append(f"{ctx.match_var()} = {edge.value};") + + out.append(f"goto {LABEL_PREFIX}{target};") + + @abstractmethod + def do_build(self, out: list[str]) -> None: ... diff --git a/src/llparse/implementation/c/node/consume.py b/src/llparse/implementation/c/node/consume.py new file mode 100644 index 0000000..fac5a07 --- /dev/null +++ b/src/llparse/implementation/c/node/consume.py @@ -0,0 +1,40 @@ +from llparse_frontend import node as frontend + +from ....error import Error +from ..constants import UNSIGNED_TYPES +from .base import Node + + +class Consume(Node[frontend.Consume]): + def do_build(self, out: list[str]): + ctx = self.compilation + + index = ctx.state_field(self.ref.field) + ty = ctx.state_field(self.ref.field) + + try: + field_ty = UNSIGNED_TYPES[ty] + except KeyError: + raise Error( + f"Unsupported type {ty} of field {self.ref.field} for consume node" + ) + + out.append("size_t avail;") + out.append(f"{field_ty} need;") + out.append("") + out.append(f"avail = {ctx.endPosArg()} - {ctx.posArg()};") + out.append(f"need = {index};") + + # Note: `avail` or `need` are going to coerced to the largest + # datatype needed to hold either of the values. + out.append("if (avail >= need) {") + out.append(" p += need;") + out.append(f" {index} = 0;") + tmp = [] + self.tail_to(tmp, self.ref.otherwise) + ctx.indent(out, tmp, " ") + out.append("}") + out.append("") + + out.append(f"{index} -= avail;") + self.pause(out) diff --git a/src/llparse/implementation/c/node/empty.py b/src/llparse/implementation/c/node/empty.py new file mode 100644 index 0000000..f08dc70 --- /dev/null +++ b/src/llparse/implementation/c/node/empty.py @@ -0,0 +1,11 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class Empty(Node[frontend.Empty]): + def do_build(self, out: list[str]) -> None: + otherwise = self.ref.otherwise + if not otherwise.no_advance: + self.prologue(out) + self.tail_to(out, otherwise) diff --git a/src/llparse/implementation/c/node/error.py b/src/llparse/implementation/c/node/error.py new file mode 100644 index 0000000..fc611ff --- /dev/null +++ b/src/llparse/implementation/c/node/error.py @@ -0,0 +1,23 @@ +from ..constants import STATE_ERROR +from .base import Node, T + + +class ErrorNode(Node[T]): + def store_error(self, out: list[str]): + ctx = self.compilation + hex_code = hex(self.ref.code) + + out.append(f"{ctx.error_field()} = {hex_code};") + out.append(f"{ctx.reason_field()} = {ctx.cstring(self.ref.reason)};") + out.append(f"{ctx.error_pos_field()} = (const char*) {ctx.pos_arg()};") + + def do_build(self, out: list[str]) -> None: + self.store_error(out) + + out.append( + f"{self.compilation.current_field()} = " + + f"(void*) (intptr_t) {STATE_ERROR};" + ) + out.append(f"return {STATE_ERROR};") + +Error = ErrorNode \ No newline at end of file diff --git a/src/llparse/implementation/c/node/invoke.py b/src/llparse/implementation/c/node/invoke.py new file mode 100644 index 0000000..a0a373b --- /dev/null +++ b/src/llparse/implementation/c/node/invoke.py @@ -0,0 +1,34 @@ +from llparse_frontend import node as frontend + +from .base import Node, NodeEdge + + +class Invoke(Node[frontend.Invoke]): + def do_build(self, out: list[str]) -> None: + ctx = self.compilation + + code = ctx.unwrap_code(self.ref.code) + + code_decl = ctx.build_code(code) + + args = [ctx.state_arg(), ctx.pos_arg(), ctx.end_pos_arg()] + + signature = code.ref.signature + + if signature == "value": + args.append(ctx.match_var()) + + out.append(f"switch ({code_decl}({', '.join(args)})) {{") + + for edge in self.ref.edges: + out.append(f" case {edge.code}:") + tmp = [] + self.tail_to(tmp, NodeEdge(no_advance=True, node=edge.node, value=None)) + ctx.indent(out, tmp, " ") + + out.append(" default:") + tmp = [] + + self.tail_to(tmp, self.ref.otherwise) + ctx.indent(out, tmp, " ") + out.append("}") diff --git a/src/llparse/implementation/c/node/pause.py b/src/llparse/implementation/c/node/pause.py new file mode 100644 index 0000000..94131dc --- /dev/null +++ b/src/llparse/implementation/c/node/pause.py @@ -0,0 +1,15 @@ +from llparse_frontend import node as frontend + +from ..constants import STATE_ERROR +from .error import ErrorNode + + +class Pause(ErrorNode[frontend.Pause]): + def do_build(self, out: list[str]) -> None: + ctx = self.compilation + + self.store_error(out) + + otherwise = ctx.unwrap_node(self.ref.otherwise.node).build(ctx) + out.append(f"{ctx.current_field()} = " + f"(void*) (intptr_t) {otherwise};") + out.append(f"return {STATE_ERROR};") diff --git a/src/llparse/implementation/c/node/sequence.py b/src/llparse/implementation/c/node/sequence.py new file mode 100644 index 0000000..92c2f58 --- /dev/null +++ b/src/llparse/implementation/c/node/sequence.py @@ -0,0 +1,54 @@ +from llparse_frontend import node as frontend + +from ..constants import SEQUENCE_COMPLETE, SEQUENCE_MISMATCH, SEQUENCE_PAUSE +from .base import Node, NodeEdge + + +class Sequence(Node[frontend.Sequence]): + def do_build(self, out): + ctx = self.compilation + + out.append("llparse_match_t match_seq;") + out.append("") + + self.prologue(out) + + match_sequence = ctx.get_match_sequence(self.ref.transform, self.ref.select) + + out.append( + f"match_seq = {match_sequence}({ctx.state_arg()}, " + + f"{ctx.pos_arg()}, " + + f"{ctx.end_pos_arg()}, {ctx.blob(self.ref.select)}, " + + f"{len(self.ref.select)});" + ) + out.append("p = match_seq.current;") + + tmp = [] + + out.append("switch (match_seq.status) {") + + out.append(f" case {SEQUENCE_COMPLETE}: {{") + tmp = [] + self.tail_to( + tmp, + NodeEdge( + no_advance=False, + node=self.ref.edge.node, + value=self.ref.edge.value, + ), + ) + ctx.indent(out, tmp, " ") + out.append(" }") + + out.append(f" case {SEQUENCE_PAUSE}: {{") + tmp = [] + self.pause(tmp) + ctx.indent(out, tmp, " ") + out.append(" }") + + out.append(f" case {SEQUENCE_MISMATCH}: {{") + tmp = [] + self.tail_to(tmp, self.ref.otherwise) + ctx.indent(out, tmp, " ") + out.append(" }") + out.append("}") diff --git a/src/llparse/implementation/c/node/single.py b/src/llparse/implementation/c/node/single.py new file mode 100644 index 0000000..04ad89d --- /dev/null +++ b/src/llparse/implementation/c/node/single.py @@ -0,0 +1,39 @@ +from llparse_frontend import node as frontend + +from .base import Node, NodeEdge + + +class Single(Node[frontend.Single]): + def do_build(self, out: list[str]): + ctx = self.compilation + otherwise = self.ref.otherwise + assert otherwise + + self.prologue(out) + transform = ctx.unwrap_transform(self.ref.transform) + current = transform.build(ctx, f"*{ctx.pos_arg()}") + out.append(f"switch ({current})" + "{") + + for e in self.ref.edges: + if e.key < 0x20 or e.key > 0x7E or e.key == 0x27 or e.key == 0x5C: + ch = e.key + else: + ch = f"'{chr(e.key)}'" + + out.append(f" case {ch}: " + "{") + tmp: list[str] = [] + + # For now debug everything.... + + self.tail_to(tmp, NodeEdge(e.node, e.no_advance, e.value)) + + ctx.indent(out, tmp, " ") + out.append(" }") + + out.append(" default: {") + + tmp: list[str] = [] + self.tail_to(tmp, NodeEdge(otherwise.node, otherwise.no_advance, None)) + ctx.indent(out, tmp, " ") + out.append(" }") + out.append("}") diff --git a/src/llparse/implementation/c/node/span_end.py b/src/llparse/implementation/c/node/span_end.py new file mode 100644 index 0000000..e9aef70 --- /dev/null +++ b/src/llparse/implementation/c/node/span_end.py @@ -0,0 +1,60 @@ +from llparse_frontend import node as frontend + +from ..constants import STATE_ERROR, STATE_PREFIX +from .base import Node, NodeEdge + + +class SpanEnd(Node[frontend.SpanEnd]): + def do_build(self, out: list[str]): + out.append("const unsigned char* start;") + out.append("int err;") + out.append("") + + ctx = self.compilation + field = self.ref.field + posField = ctx.span_pos_field(field.index) + + # Loast start position + out.append(f"start = {posField};") + + # reset position + out.append(f"{posField} = NULL;") + + # Invoke callback + callback = ctx.build_code(ctx.unwrap_code(self.ref.callback)) + + out.append(f"err = {callback}({ctx.state_arg()}, start, {ctx.pos_arg()});") + + out.append("if (err != 0) {") + tmp = [] + self.build_error(tmp, "err") + ctx.indent(out, tmp, " ") + out.append("}") + + otherwise = self.ref.otherwise + self.tail_to(out, NodeEdge(otherwise.node, otherwise.no_advance, None)) + + def build_error(self, out: list[str], code: str): + ctx = self.compilation + + out.append(f"{ctx.error_field()} = {code};") + + otherwise = self.ref.otherwise + assert otherwise + + resume_pos = ctx.pos_arg() + + if not otherwise.no_advance: + resume_pos = f"({resume_pos} + 1)" + + out.append(f"{ctx.error_pos_field()} = (const char*) {resume_pos};") + + rt = ctx.unwrap_node(otherwise.node) + # check if the resumption target has already been built or not... + resumption_target = rt.build(ctx) + + out.append( + f"{ctx.current_field()} = " + + f"(void*) (intptr_t) {STATE_PREFIX + resumption_target if not resumption_target.startswith(STATE_PREFIX) else resumption_target};" + ) + out.append(f"return {STATE_ERROR};") diff --git a/src/llparse/implementation/c/node/span_start.py b/src/llparse/implementation/c/node/span_start.py new file mode 100644 index 0000000..1b1b55f --- /dev/null +++ b/src/llparse/implementation/c/node/span_start.py @@ -0,0 +1,24 @@ +from llparse_frontend import node as frontend + +from .base import Node, NodeEdge + + +class SpanStart(Node[frontend.SpanStart]): + def do_build(self, out: list[str]): + self.prologue(out) + + ctx = self.compilation + field = self.ref.field + + pos_field = ctx.span_pos_field(field.index) + out.append(f"{pos_field} = (void*) {ctx.pos_arg()};") + + if len(field.callbacks) > 1: + cbField = ctx.span_cb_field(field.index) + callback = ctx.unwrap_code(self.ref.callback) + out.append(f"{cbField} = {ctx.build_code(callback)};") + + otherwise = self.ref.otherwise + self.tail_to( + out, NodeEdge(otherwise.node, otherwise.no_advance, otherwise.value) + ) diff --git a/src/llparse/implementation/c/node/table_lookup.py b/src/llparse/implementation/c/node/table_lookup.py new file mode 100644 index 0000000..f8bca83 --- /dev/null +++ b/src/llparse/implementation/c/node/table_lookup.py @@ -0,0 +1,9 @@ +from llparse_frontend import node as frontend + +from ....ext import Writer +from .base import Node + + +class TableLookup(Node[frontend.TableLookup]): + def do_build(self, out: Writer) -> None: + super().do_build(out) diff --git a/src/llparse/implementation/c/node/unpack.py b/src/llparse/implementation/c/node/unpack.py new file mode 100644 index 0000000..2de28f5 --- /dev/null +++ b/src/llparse/implementation/c/node/unpack.py @@ -0,0 +1,161 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class Unpack(Node[frontend.Unpack]): + @property + def offset(self) -> int: + return self.ref.byte_offset + + @property + def pair(self): + return self.compilation, self.compilation.state_field(self.ref.field) + + def read_int8(self, out: list[str]) -> None: + ctx, index = self.pair + out.append(f"{index} = ((*{ctx.pos_arg()}) & 0x80);") + + def read_uint8(self, out: list[str]) -> None: + ctx, index = self.pair + out.append(f"{index} = (*{ctx.pos_arg()});") + + # LITTLE ENDIAN + + def read_int16LE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + # Since BE Belongs to performing << aka left shifts we do >> right shifts + out.append(f"{index} = ({index} >> 8) | ((*{ctx.pos_arg()}) & 0x80);") + + def read_uint16LE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} >> 8) | (*{ctx.pos_arg()});") + + def read_int24LE(self, out: list[str]) -> None: + ctx, index = self.pair + match self.offset: + case 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + case 1: + out.append(f"{index} = ({index} >> 8) | (*{ctx.pos_arg()});") + case _: + out.append(f"{index} = ({index} >> 8) | ((*{ctx.pos_arg()}) & 0x80);") + + def read_uint24LE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} >> 8) | (*{ctx.pos_arg()});") + + def read_int32LE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + elif self.offset in (1, 2): + out.append(f"{index} = ({index} >> 8) | (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} >> 8) | ((*{ctx.pos_arg()}) & 0x80);") + + def read_uint32LE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} >> 8) | (*{ctx.pos_arg()});") + + # BIG ENDIAN + + def read_int16BE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + # Since LE Belongs to >> we do "<<" instead + out.append(f"{index} = ({index} << 8) | ((*{ctx.pos_arg()}) & 0x80);") + + def read_uint16BE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} << 8) | (*{ctx.pos_arg()});") + + def read_int24BE(self, out: list[str]) -> None: + ctx, index = self.pair + match self.offset: + case 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + case 1: + out.append(f"{index} = ({index} << 8) | (*{ctx.pos_arg()});") + case _: + out.append(f"{index} = ({index} << 8) | ((*{ctx.pos_arg()}) & 0x80);") + + def read_uint24BE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} << 8) | (*{ctx.pos_arg()});") + + def read_int32BE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + elif self.offset in (1, 2): + out.append(f"{index} = ({index} << 8) | (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} << 8) | ((*{ctx.pos_arg()}) & 0x80);") + + def read_uint32BE(self, out: list[str]) -> None: + ctx, index = self.pair + if self.offset == 0: + out.append(f"{index} = (*{ctx.pos_arg()});") + else: + out.append(f"{index} = ({index} << 8) | (*{ctx.pos_arg()});") + + def do_build(self, out: list[str]): + self.prologue(out) + + if self.compilation.getFieldType(self.ref.field) == "ptr": + raise ValueError( + f'property {self.ref.field} should not use pointers but it was given "ptr"' + ) + match self.ref.bits: + case 1: + self.read_int8(out) if self.ref.signed else self.read_uint8(out) + case 2: + if self.ref.little_endian: + self.read_int16LE(out) if self.ref.signed else self.read_uint16LE( + out + ) + else: + self.read_int16BE(out) if self.ref.signed else self.read_uint16BE( + out + ) + case 3: + if self.ref.little_endian: + self.read_int24LE(out) if self.ref.signed else self.read_uint24LE( + out + ) + else: + self.read_int24BE(out) if self.ref.signed else self.read_uint24BE( + out + ) + case _: + if self.ref.little_endian: + self.read_int32LE(out) if self.ref.signed else self.read_uint32LE( + out + ) + else: + self.read_int32BE(out) if self.ref.signed else self.read_uint32BE( + out + ) + + self.tail_to(out, self.ref.otherwise.node, self.ref.otherwise.noAdvance, None) diff --git a/src/llparse/implementation/c/transform/__init__.py b/src/llparse/implementation/c/transform/__init__.py new file mode 100644 index 0000000..288dc6f --- /dev/null +++ b/src/llparse/implementation/c/transform/__init__.py @@ -0,0 +1,16 @@ +# This code was autogenerated from tools/generate_impl.py +# It is used to handle code generation for new language implementations +# And gives a useful template for the list of nodes and code objects needed. +# Editing is fine but use tools/generate_impl.py at your own risk. + +from .base import Transform +from .id import ID +from .to_lower import ToLower +from .to_lower_unsafe import ToLowerUnsafe + +__all__ = ( + "ID", + "ToLower", + "ToLowerUnsafe", + "Transform", +) diff --git a/src/llparse/implementation/c/transform/base.py b/src/llparse/implementation/c/transform/base.py new file mode 100644 index 0000000..b5b8f70 --- /dev/null +++ b/src/llparse/implementation/c/transform/base.py @@ -0,0 +1,14 @@ +from abc import ABC, abstractmethod +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class Transform(ABC, Generic[T]): + __slots__ = ("ref",) + + def __init__(self, ref: T) -> None: + self.ref = ref + + @abstractmethod + def build(self, ctx, value: str) -> str: ... diff --git a/src/llparse/implementation/c/transform/id.py b/src/llparse/implementation/c/transform/id.py new file mode 100644 index 0000000..5c6ddf4 --- /dev/null +++ b/src/llparse/implementation/c/transform/id.py @@ -0,0 +1,8 @@ +from llparse_frontend import transform as frontend + +from .base import Transform + + +class ID(Transform[frontend.ID]): + def build(self, ctx, value: str) -> str: + return value diff --git a/src/llparse/implementation/c/transform/to_lower.py b/src/llparse/implementation/c/transform/to_lower.py new file mode 100644 index 0000000..9b21a07 --- /dev/null +++ b/src/llparse/implementation/c/transform/to_lower.py @@ -0,0 +1,10 @@ +from llparse_frontend import transform as frontend + +from .base import Transform + + +class ToLower(Transform[frontend.ToLower]): + def build(self, ctx, value: str) -> str: + return ( + f"(({value}) >= 'A' && ({value}) <= 'Z' ? ({value} | 0x20) : ({value}))`;" + ) diff --git a/src/llparse/implementation/c/transform/to_lower_unsafe.py b/src/llparse/implementation/c/transform/to_lower_unsafe.py new file mode 100644 index 0000000..a930f30 --- /dev/null +++ b/src/llparse/implementation/c/transform/to_lower_unsafe.py @@ -0,0 +1,8 @@ +from llparse_frontend import transform as frontend + +from .base import Transform + + +class ToLowerUnsafe(Transform[frontend.ToLowerUnsafe]): + def build(self, ctx, value: str) -> str: + return f"(({value}) | 0x20)" diff --git a/src/llparse_builder/__init__.py b/src/llparse_builder/__init__.py new file mode 100644 index 0000000..8100b65 --- /dev/null +++ b/src/llparse_builder/__init__.py @@ -0,0 +1,8 @@ +from .builder import Builder +from .loop_checker import LoopChecker + +__version__ = "0.1.0" +__author__ = "Vizonex" +__typescript_author__ = "Indutny" + +__all__ = ("Builder", "LoopChecker") diff --git a/src/llparse_builder/binary_search.py b/src/llparse_builder/binary_search.py new file mode 100644 index 0000000..38d59e8 --- /dev/null +++ b/src/llparse_builder/binary_search.py @@ -0,0 +1,59 @@ +""" +binary_search +------------- + +based off llparse's version, It's actually unimplemented +mainly due to faster technqiues which were found with python. +But can be used if needed. +""" + +from collections.abc import Callable, Sequence +from typing import TypeVar + +from .errors import RangeError + +A = TypeVar("A") +B = TypeVar("B") + + +def binary_search( + haystack: Sequence[A], + needle: B, + comparator: Callable[[A, B, float | None, Sequence[A] | None], int], + low: float | None = None, + high: float | None = None, +) -> float: + if not isinstance(haystack, Sequence): + raise TypeError("haystack must be a valid sequence") + if not callable(comparator): + raise TypeError("comparator must be a callable") + + if low is not None: + low = low or 0 + if low < 0 or low >= len(haystack): + raise RangeError("invalid lower bound") + else: + low = 0 + + if high is not None: + high = high or 0 + if (high < low) or (high >= len(haystack)): + raise RangeError("invalid upper bound") + else: + high = len(haystack) + + if low == 0 and high == 0 and not haystack: + return -1 + + while low <= high: + mid = low + ((high - low) >> 1) + # Weird opeation PLEASE DO NOT CHANGE = +yadayada + # the space before the = sign is here for a reason!!!! + cmp = +comparator(haystack[mid], needle, mid, haystack) + if cmp < 0.0: + low = mid + 1 + elif cmp > 0.0: + high = mid - 1 + else: + return mid + return ~low diff --git a/src/llparse_builder/builder.py b/src/llparse_builder/builder.py new file mode 100644 index 0000000..811e481 --- /dev/null +++ b/src/llparse_builder/builder.py @@ -0,0 +1,173 @@ +from typing import Literal + +from . import code, transform +from . import node as _node +from .edge import Edge +from .loop_checker import LoopChecker +from .property import Property, PropertyType, PropertyTypes +from .reachability import Reachability +from .span import Span +from .span_allocator import ISpanAllocatorResult, SpanAllocator +from .unpack_creator import StructUnpack, UnpackCreator + +node = _node + +__all__ = ( + "Builder", + "Edge", + "ISpanAllocatorResult", + "LoopChecker", + "Property", + "PropertyType", + "Reachability", + "Span", + "SpanAllocator", + "code", + "node", + "transform", +) + +code_span = code.Span +Unpack = _node.Unpack + + +# TODO (Vizonex) Add more Documentation later , I got tired of it... +class Builder: + __slots__ = ("code", "priv_properties", "transform", "unpack_creator") + + def __init__(self) -> None: + self.code = code.Creator() + " API for creating external callbacks and intrinsic operations." + self.transform = transform.Creator() + self.priv_properties: dict[str, Property] = {} + self.unpack_creator = UnpackCreator() + + def node(self, name: str) -> _node.Match: + return _node.Match(name) + + def error(self, errorCode: int, reason: str): + return _node.Error(errorCode, reason) + + def invoke( + self, + fn: "code.base.Code", + map: dict[int, _node.Node] | _node.Node | None = None, + otherwise: _node.Node | None = None, + ): + if not map: + res = _node.Invoke(fn, {}) + + elif isinstance(map, node.Node): + res = _node.Invoke(fn, {}) + otherwise = map + + else: + res = _node.Invoke(fn, map) + + if otherwise: + res.otherwise(otherwise) + + return res + + def consume(self, field: str): + return _node.Consume(field) + + def pause(self, errorCode: int, reason: str): + return _node.Pause(errorCode, reason) + + def span(self, callback: code_span) -> Span: + return Span(callback) + + def property(self, ty: Literal["i8", "i16", "i32", "i64", "ptr"], name: str): + if ty not in PropertyTypes: + raise TypeError(f"ty:{ty} is not an existing Parser Property") + + if name in self.priv_properties: + raise RuntimeError(f"Duplicate property with name:{name}") + + self.priv_properties[name] = Property(ty, name) + + def properties(self) -> list[Property]: + """Return list of all allocated properties in parser's state.""" + return list(self.priv_properties.values()) + + def intBE(self, field: str, bits: int): + """ + :param field: State's property name + :param bits: Number of bits to use + + """ + return Unpack(field, bits, True, False) + + def intLE(self, field: str, bits: int): + """ + return a node for unpacking arrays to integers + + :param field: State's property name + :param bits: Number of bits to use + """ + return Unpack(field, bits, True, True) + + def uintBE(self, field: str, bits: int): + """ + return a node for unpacking arrays to integers + + :param field: State's property name + :param bits: Number of bits to use + """ + return Unpack(field, bits, False, False) + + def uintLE(self, field: str, bits: int): + """ + return a node for unpacking arrays to integers + + :param field: State's property name + :param bits: Number of bits to use + """ + return Unpack(field, bits, False, True) + + def unpack(self, field: str, fmt: str): + """Enables unpacking in python struct notation + they are mapped as. + + - value, type in rust notation, bit length + - 'b' i8, 1 + - 'c' i8, 1 + - '?' bool, 1 + - 'h' i16, 2 + - 'i' i32, 4 + - 'l' i32, 4 + - 'q' i64, 8 + - 'B' u8, 1 + - 'H' u16, 2 + - 'I' u32, 4 + - 'L' u32, 4 + - 'Q' u64, 8 + + These ones are custom and not apart of CPython at all. + They are added incase only 3 bits are needed. They + are used to describe a hybrid (H was already taken by short so will use a y) + - 'y', i24, 3 + - 'Y', u24, 3 + + :param field: the property's name + :param fmt: the struct format as described above + """ + return Unpack.from_format(field, fmt) + + def unpack_structure(self, fmt: str) -> StructUnpack: + """Provides an unpackable structure to lazily unpack into different paths. + :param fmt: format of the nodes + `name:format|name:format` + """ + return self.unpack_creator.struct(fmt) + + # XXX: String annotations needed due to VSCode + Pyright wanting to screw around. + def unpack_to(self, fmt: str, node: _node.Node) -> "_node.Unpack": + """takes a list of elements as `property_name:value|etc...` + :param fmt: format of the nodes + `name:format|name:format` + :param node: the node to skip to afterwards + it is required due to the way all of this works. + """ + return self.unpack_creator.with_format(fmt, node) diff --git a/src/llparse_builder/code/__init__.py b/src/llparse_builder/code/__init__.py new file mode 100644 index 0000000..7c68ca1 --- /dev/null +++ b/src/llparse_builder/code/__init__.py @@ -0,0 +1,33 @@ +from ._and import And +from ._or import Or +from .base import Code +from .creator import Creator +from .field import Field +from .field_value import FieldValue +from .is_equal import IsEqual +from .load import Load +from .match import Match +from .mul_add import MulAdd +from .span import Span +from .store import Store +from .test import Test +from .update import Update +from .value import Value + +__all__ = ( + "And", + "Code", + "Creator", + "Field", + "FieldValue", + "IsEqual", + "Load", + "Match", + "MulAdd", + "Or", + "Span", + "Store", + "Test", + "Update", + "Value", +) diff --git a/src/llparse_builder/code/_and.py b/src/llparse_builder/code/_and.py new file mode 100644 index 0000000..43bc423 --- /dev/null +++ b/src/llparse_builder/code/_and.py @@ -0,0 +1,8 @@ +from .field_value import FieldValue + + +class And(FieldValue): + __slots__ = ("name", "signature", "value") + + def __init__(self, field: str, value: int) -> None: + super().__init__("match", "and", field, value) diff --git a/src/llparse_builder/code/_or.py b/src/llparse_builder/code/_or.py new file mode 100644 index 0000000..ebf3788 --- /dev/null +++ b/src/llparse_builder/code/_or.py @@ -0,0 +1,8 @@ +from .field_value import FieldValue + + +class Or(FieldValue): + __slots__ = ("name", "signature", "value") + + def __init__(self, field: str, value: int): + super().__init__("match", "or", field, value) diff --git a/src/llparse_builder/code/base.py b/src/llparse_builder/code/base.py new file mode 100644 index 0000000..8fd64a6 --- /dev/null +++ b/src/llparse_builder/code/base.py @@ -0,0 +1,9 @@ +from typing import Literal + + +class Code: + __slots__ = ("name", "signature") + + def __init__(self, signature: Literal["match", "value"], name: str) -> None: + self.signature = signature + self.name = name diff --git a/src/llparse_builder/code/creator.py b/src/llparse_builder/code/creator.py new file mode 100644 index 0000000..4e37a66 --- /dev/null +++ b/src/llparse_builder/code/creator.py @@ -0,0 +1,54 @@ +from ._and import And +from ._or import Or +from .is_equal import IsEqual +from .load import Load +from .match import Match +from .mul_add import MulAdd +from .span import Span +from .store import Store +from .test import Test +from .update import Update +from .value import Value + + +class Creator: + __slots__ = () + + def match(self, name: str) -> Match: + return Match(name) + + def value(self, name: str) -> Value: + return Value(name) + + def span(self, name: str) -> Span: + return Span(name) + + def store(self, field: str) -> Store: + return Store(field) + + def load(self, field: str) -> Load: + return Load(field) + + def mul_add( + self, field: str, base: int, max: int | None = None, signed: bool | None = None + ) -> MulAdd: + return MulAdd(field, base, max, signed) + + def update(self, field: str, value: int) -> Update: + return Update(field, value) + + def is_equal(self, field: str, value: int) -> IsEqual: + return IsEqual(field, value) + + def And(self, field: str, value: int) -> And: + return And(field, value) + + def Or(self, field: str, value: int) -> Or: + return Or(field, value) + + # secondary options. + and_ = And + or_ = Or + + def test(self, field: str, value: int) -> Test: + return Test(field, value) diff --git a/src/llparse_builder/code/field.py b/src/llparse_builder/code/field.py new file mode 100644 index 0000000..8f8b124 --- /dev/null +++ b/src/llparse_builder/code/field.py @@ -0,0 +1,12 @@ +from typing import Literal + +from .base import Code + + +class Field(Code): + __slots__ = ("field", "name", "signature") + + def __init__(self, signature: Literal["match", "value"], name: str, field: str): + super().__init__(signature, name + "_" + field) + assert not field.startswith("_"), "Can't access internal field from user code" + self.field = field diff --git a/src/llparse_builder/code/field_value.py b/src/llparse_builder/code/field_value.py new file mode 100644 index 0000000..3779eff --- /dev/null +++ b/src/llparse_builder/code/field_value.py @@ -0,0 +1,13 @@ +from typing import Literal + +from .field import Field + + +class FieldValue(Field): + __slots__ = ("name", "signature", "value") + + def __init__( + self, signature: Literal["match", "value"], name: str, field: str, value: int + ): + super().__init__(signature, name, field) + self.value = value diff --git a/src/llparse_builder/code/is_equal.py b/src/llparse_builder/code/is_equal.py new file mode 100644 index 0000000..9445381 --- /dev/null +++ b/src/llparse_builder/code/is_equal.py @@ -0,0 +1,8 @@ +from .field_value import FieldValue + + +class IsEqual(FieldValue): + __slots__ = ("field", "name", "signature", "value") + + def __init__(self, field: str, value: int): + super().__init__("value", "is_equal", field, value) diff --git a/src/llparse_builder/code/load.py b/src/llparse_builder/code/load.py new file mode 100644 index 0000000..2e6b035 --- /dev/null +++ b/src/llparse_builder/code/load.py @@ -0,0 +1,8 @@ +from .field import Field + + +class Load(Field): + __slots__ = ("name", "signature") + + def __init__(self, field: str): + super().__init__("match", "load", field) diff --git a/src/llparse_builder/code/match.py b/src/llparse_builder/code/match.py new file mode 100644 index 0000000..cce4654 --- /dev/null +++ b/src/llparse_builder/code/match.py @@ -0,0 +1,8 @@ +from .base import Code + + +class Match(Code): + __slots__ = ("name", "signature") + + def __init__(self, name: str): + super().__init__("match", name) diff --git a/src/llparse_builder/code/mul_add.py b/src/llparse_builder/code/mul_add.py new file mode 100644 index 0000000..5a46761 --- /dev/null +++ b/src/llparse_builder/code/mul_add.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +from .field import Field + + +@dataclass(slots=True) +class MulAddOptions: + base: int + max: int | None = None + signed: bool | None = None + + +class MulAdd(Field): + __slots__ = ("name", "options", "signature") + + def __init__( + self, field: str, base: int, max: int | None = None, signed: int | None = None + ): + super().__init__("value", "mul_add", field) + self.options = MulAddOptions(base, max, signed) diff --git a/src/llparse_builder/code/span.py b/src/llparse_builder/code/span.py new file mode 100644 index 0000000..e35a163 --- /dev/null +++ b/src/llparse_builder/code/span.py @@ -0,0 +1,5 @@ +from .match import Match + + +class Span(Match): + __slots__ = ("name", "signature") diff --git a/src/llparse_builder/code/store.py b/src/llparse_builder/code/store.py new file mode 100644 index 0000000..6254c08 --- /dev/null +++ b/src/llparse_builder/code/store.py @@ -0,0 +1,8 @@ +from .field import Field + + +class Store(Field): + __slots__ = ("field", "name", "signature") + + def __init__(self, field: str): + super().__init__("value", "store", field) diff --git a/src/llparse_builder/code/test.py b/src/llparse_builder/code/test.py new file mode 100644 index 0000000..69168d8 --- /dev/null +++ b/src/llparse_builder/code/test.py @@ -0,0 +1,8 @@ +from .field_value import FieldValue + + +class Test(FieldValue): + __slots__ = ("field", "name", "signature", "value") + + def __init__(self, field: str, value: int) -> None: + super().__init__("match", "test", field, value) diff --git a/src/llparse_builder/code/update.py b/src/llparse_builder/code/update.py new file mode 100644 index 0000000..9a96615 --- /dev/null +++ b/src/llparse_builder/code/update.py @@ -0,0 +1,8 @@ +from .field_value import FieldValue + + +class Update(FieldValue): + __slots__ = ("name", "signature", "value") + + def __init__(self, field: str, value: int): + super().__init__("match", "update", field, value) diff --git a/src/llparse_builder/code/value.py b/src/llparse_builder/code/value.py new file mode 100644 index 0000000..7d702f7 --- /dev/null +++ b/src/llparse_builder/code/value.py @@ -0,0 +1,8 @@ +from .base import Code + + +class Value(Code): + __slots__ = ("name", "signature") + + def __init__(self, name): + super().__init__("value", name) diff --git a/src/llparse_builder/edge.py b/src/llparse_builder/edge.py new file mode 100644 index 0000000..15f7abb --- /dev/null +++ b/src/llparse_builder/edge.py @@ -0,0 +1,59 @@ +""" +Edge +---- + +Node's edge in the parser graph. +""" + +# Prevent circular importing but also enhance speedups. +from . import node as _node + + +class Edge: + """Represents an edge in the parser graph.""" + + __slots__ = ("key", "no_advance", "node", "value") + + def __eq__(a: "Edge", b: "Edge"): + if isinstance(a.key, int): + return a.key == (b.key[0] if not isinstance(b.key, int) else b.key) + return a.key == b.key + + def __init__( + self, + node: "_node.Node", + no_advance: bool, + key: bytes | str | int | None = None, + value: int | None = None, + ) -> None: + if isinstance(key, str): + key = key.encode("ascii") + + self.node = node + self.no_advance = no_advance + self.key = key + self.value = value + if isinstance(node, _node.Invoke): + match node.code.signature: + case "match": + assert value is None, "Invalid Invoke's code signature" + case "value": + assert value is not None, "Invalid Invoke's code signature" + + if isinstance(key, (bytes, str)): + assert len(key) > 0, "Invalid edge buffer length" + if no_advance: + assert len(key) == 1, ( + "Only 1-character keys are allowed with no_advance edges." + ) + + def __hash__(self): + return hash((self.node.name, self.no_advance, self.key, self.value)) + + def __repr__(self): + return ( + f"{self.__class__.__name__}" + f"(node={self.node.name!r}," + f"no_advance={self.no_advance!r}, " + f"key={self.key!r}, value={self.value!r})" + ) diff --git a/src/llparse_builder/errors.py b/src/llparse_builder/errors.py new file mode 100644 index 0000000..41d0866 --- /dev/null +++ b/src/llparse_builder/errors.py @@ -0,0 +1,6 @@ +class Error(Exception): + """Something went wrong during build process""" + + +class RangeError(Exception): + pass diff --git a/src/llparse_builder/loop_checker/__init__.py b/src/llparse_builder/loop_checker/__init__.py new file mode 100644 index 0000000..fcb09aa --- /dev/null +++ b/src/llparse_builder/loop_checker/__init__.py @@ -0,0 +1,163 @@ +import logging +from collections.abc import Sequence +from typing import TypeVar + +from ..errors import Error +from ..node import Node +from ..reachability import Reachability +from .lattice import Lattice + +logger = logging.getLogger(__name__) +debug = logger.debug +EMPTY_VALUE = Lattice("empty") +ANY_VALUE = Lattice("any") + +T = TypeVar("T") + + +def js_indexof(array: Sequence[Node], item: Node) -> int: + """mimics typescript's indexOf functionality""" + for a in array: + if a.name == item.name: + return 0 + return -1 + + +class LoopChecker: + __slots__ = ("__weakref__", "lattice", "terminated_cache") + + def __init__(self): + self.lattice: dict[Node, Lattice] = {} + self.terminated_cache: dict[Node, Lattice] = {} + + def check(self, root: Node): + r = Reachability() + nodes = r.build(root) + + for node in nodes: + debug(f"checking loops starting with {node.name!r}") + self.clear(nodes) + self.lattice[node] = ANY_VALUE + changed = {root} + while changed: + if not logger.disabled: + debug("changed %s", [other.name for other in changed]) + next_node = set() + for changed_node in changed: + self.propagate(changed_node, next_node) + changed = next_node + + debug("lattice stabilized") + self.visit(node, []) + + def clear(self, nodes: list[Node]): + for n in nodes: + self.lattice[n] = EMPTY_VALUE + + def propagate(self, node: Node, changed: set[Node]) -> None: + value = self.lattice[node] + debug(f"propagate({node.name!r}), initial value {value.to_json()}") + + # Termine values that are consume by match/select + terminated = self.terminate(node) + if not terminated.is_empty(): + debug(f"node {node.name!r} terminates {terminated.to_json()!r}") + value = value.subtract(terminated) + if value.is_empty(): + return + + keys_by_target: dict[Node, Lattice] = {} + + for edge in node.get_all_edges(): + if not edge.no_advance: + continue + + if edge.node in keys_by_target: + target_value = keys_by_target[edge.node] + else: + target_value = self.lattice[edge.node] + + if edge.key is None or isinstance(edge.key, int): + # .otherwise() + keys_by_target[edge.node] = target_value.union(value) + else: + # .peek() + edge_value = Lattice([edge.key[0]]).intersect(value) + if edge_value.is_empty(): + continue + keys_by_target[edge.node] = target_value.union(edge_value) + + for child, child_value in keys_by_target.items(): + debug( + f"node {node.name!r} propagates {child_value.to_json()!r} to {child.name!r}" + ) + self.update(child, child_value, changed) + + def update(self, node: Node, new_value: Lattice, changed: set[Node]): + value = self.lattice[node] + if new_value.is_equal(value): + return False + + self.lattice[node] = new_value + changed.add(node) + return True + + def terminate(self, node: Node) -> Lattice: + # print("determining termination of \"%s\"" % node.name) + if node.name in self.terminated_cache: + return self.terminated_cache[node] + + terminated: list[int] = [] + + for edge in node.get_all_edges(): + if edge.no_advance: + continue + + if edge.key is None or isinstance(edge.key, int): + continue + + terminated.append(edge.key[0]) + + result = Lattice(terminated) + self.terminated_cache[node] = result + return result + + def visit(self, node: Node, path: list[Node]) -> None: + value = self.lattice[node] + debug(f"enter {node.name!r}, value is {value.to_json()!r}") + + terminated = self.terminated_cache.get(node, EMPTY_VALUE) + if not terminated.is_equal(EMPTY_VALUE): + debug(f"subtract terminated {node.name} {terminated.to_json()!r}") + value = value.subtract(terminated) + if value.is_equal(EMPTY_VALUE): + debug("terminated everything") + return + + for edge in node.get_all_edges(): + if not edge.no_advance: + continue + + edge_value = value + if edge.key is None or isinstance(edge.key, int): + pass + else: + edge_value = edge_value.intersect(Lattice([edge.key[0]])) + + if edge_value.is_equal(EMPTY_VALUE): + continue + + if edge.node in path: + if not path: + raise Error( + f'Detected loop in "{edge.node.name}" through "{edge.node.name}"' + ) + raise Error( + 'Detected loop in "' + + edge.node.name + + '" through chain ' + + (" -> ").join(['"' + parent.name + '"' for parent in path]) + ) + + self.visit(edge.node, path + [edge.node]) + debug(f"leave {node.name!r}") diff --git a/src/llparse_builder/loop_checker/lattice.py b/src/llparse_builder/loop_checker/lattice.py new file mode 100644 index 0000000..8515a8d --- /dev/null +++ b/src/llparse_builder/loop_checker/lattice.py @@ -0,0 +1,117 @@ +from array import array +from collections.abc import Iterable +from typing import Literal + + +def empty_arr(): + return array("l", [0, 0, 0, 0, 0, 0, 0, 0]) + + +def any_arr(): + return array("l", [-1, -1, -1, -1, -1, -1, -1, -1]) + + +MAX_VALUE = 256 +WORD_SIZE = 32 +SIZE = MAX_VALUE // WORD_SIZE +WORD_FILL = -1 + +# Will be using arrays to take advantage of speeds +# normally these sections of code can require +# heavier amounts of calculations to be done otherwise +# cython would need to be used here. + +assert (MAX_VALUE % WORD_SIZE) == 0 + + +class Lattice: + # words: + + __slots__ = ("words",) + + def __init__(self, value: Literal["empty", "any"] | Iterable[int]): + match value: + case "empty": + self.words = empty_arr() + case "any": + self.words = any_arr() + case _: + self.words = empty_arr() + for single in value: + # inlined version of add(...) + assert 0 <= single <= MAX_VALUE, "Invalid bit" + index, off = divmod(single, WORD_SIZE) + self.words[index] |= 1 << off + + def dup(self): + e = Lattice("empty") + e.words = self.words + return e + + def check(self, bit: int) -> bool: + assert 0 <= bit < MAX_VALUE, "invalid bit" + index, off = divmod(bit, WORD_SIZE) + return (self.words[index] & (1 << off)) != 0 + + def union(self, other: "Lattice") -> "Lattice": + result = Lattice("empty") + for i in range(SIZE): + result.words[i] = self.words[i] | other.words[i] + + return result + + def intersect(self, other: "Lattice") -> "Lattice": + result = Lattice("empty") + for i in range(SIZE): + result.words[i] = self.words[i] & other.words[i] + return result + + def subtract(self, other: "Lattice") -> "Lattice": + result = Lattice("empty") + for i in range(SIZE): + result.words[i] = self.words[i] & (~other.words[i]) + return result + + def is_equal(self, value: "Lattice") -> bool: + if id(self) == id(value): + return True + + for i in range(SIZE): + if self.words[i] != value.words[i]: + return False + return True + + def is_empty(self) -> bool: + for w in self.words: + if w != 0: + return False + return True + + def __iter__(self): + for i in range(MAX_VALUE): + if self.check(i): + yield i + + # TODO: (Vizonex) use a __repr__ function instead. + def to_json(self): + is_empty = True + is_full = True + + for i in range(SIZE): + if self.words[i] != 0: + is_empty = False + if self.words[i] != WORD_FILL: + is_full = False + + if is_empty: + return "empty" + + if is_full: + return "any" + + return list(self.words) + + def add(self, bit: int): + assert 0 <= bit <= MAX_VALUE, "Invalid bit" + index, off = divmod(bit, WORD_SIZE) + self.words[index] |= 1 << off diff --git a/src/llparse_builder/node/__init__.py b/src/llparse_builder/node/__init__.py new file mode 100644 index 0000000..70063c2 --- /dev/null +++ b/src/llparse_builder/node/__init__.py @@ -0,0 +1,21 @@ +from .base import Node +from .consume import Consume +from .error import NodeError as Error +from .invoke import Invoke +from .match import Match +from .pause import Pause +from .span_end import SpanEnd +from .span_start import SpanStart +from .unpack import Unpack + +__all__ = ( + "Consume", + "Error", + "Invoke", + "Match", + "Node", + "Pause", + "SpanEnd", + "SpanStart", + "Unpack", +) diff --git a/src/llparse_builder/node/base.py b/src/llparse_builder/node/base.py new file mode 100644 index 0000000..c5ca0eb --- /dev/null +++ b/src/llparse_builder/node/base.py @@ -0,0 +1,64 @@ +import sys +from collections import deque +from collections.abc import Iterator, Sequence + +from ..edge import Edge +from ..errors import Error + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +# shortcut so we don't leak functions into the main node that gets +# subclassed quite a lot. +def add_otherwise_edge(this: "Node", other: "Node", no_advance: bool): + if this.otherwise_edge is not None: + raise Error("Node already has an `otherwise` or `skip_to`") + this.otherwise_edge = Edge(other, no_advance, None, None) + + +class Node: + __slots__ = ("name", "otherwise_edge", "priv_edges") + + def __init__(self, name: str) -> None: + self.otherwise_edge: Edge | None = None + self.priv_edges: deque[Edge] = deque() + self.name = name + + def __hash__(self) -> int: + return hash(self.name) + + def __eq__(self, value): + if not isinstance(value, Node): + return False + return ( + value.name == self.name + and value.priv_edges == self.priv_edges + and value.otherwise_edge == self.otherwise_edge + ) + + def otherwise(self, node: "Node") -> Self: + add_otherwise_edge(self, node, True) + return self + + def skip_to(self, node: "Node") -> Self: + add_otherwise_edge(self, node, False) + return self + + def get_otherwise_edge(self) -> Edge | None: + return self.otherwise_edge + + def get_all_edges(self) -> Sequence[Edge]: + if edge := self.otherwise_edge: + return [*self.priv_edges, edge] + return self.priv_edges + + def __iter__(self) -> Iterator[Edge]: + yield from self.priv_edges + + def add_edge(self, edge: Edge) -> None: + assert edge.key is not None + assert edge not in self.priv_edges, "Attempting to create duplicate edge" + self.priv_edges.appendleft(edge) diff --git a/src/llparse_builder/node/consume.py b/src/llparse_builder/node/consume.py new file mode 100644 index 0000000..ae4c136 --- /dev/null +++ b/src/llparse_builder/node/consume.py @@ -0,0 +1,9 @@ +from ..errors import Error +from .base import Node + + +class Consume(Node): + def __init__(self, name: str): + super().__init__(f"consume_{name}") + if name.startswith("_"): + raise Error("Cannot use internal field in consume") diff --git a/src/llparse_builder/node/error.py b/src/llparse_builder/node/error.py new file mode 100644 index 0000000..508b94c --- /dev/null +++ b/src/llparse_builder/node/error.py @@ -0,0 +1,25 @@ +from enum import IntEnum + +from ..errors import Error +from .base import Node + + +class NodeError(Node): + """Node that terminates execution with a given error""" + + __slots__ = ("code", "reason") + + def __init__(self, code: int | IntEnum, reason: str): + super().__init__("error") + # Decided to would go a step further and support + # enums just like the typescript version would permit + self.code = code.value if isinstance(code, IntEnum) else code + self.reason = reason + + def otherwise(self): + """`.otherwise()` is not supported with this type of Node""" + raise Error("Not supported.") + + def skip_to(self): + """`.skip_to` is not supported with this type of node.""" + raise Error("Not supported.") diff --git a/src/llparse_builder/node/invoke.py b/src/llparse_builder/node/invoke.py new file mode 100644 index 0000000..dc78f11 --- /dev/null +++ b/src/llparse_builder/node/invoke.py @@ -0,0 +1,18 @@ +from collections.abc import Mapping + +from ..code.base import Code +from ..edge import Edge +from .base import Node + + +class Invoke(Node): + __slots__ = ("code", "map") + + def __init__(self, code: Code, map: Mapping[int | str | bytes, Node]): + super().__init__(f"invoke_{code.name}") + + for k, v in map.items(): + assert isinstance(k, int), "Invoke's map keys must be integers" + self.add_edge(Edge(v, True, k)) + self.code = code + self.map = map diff --git a/src/llparse_builder/node/match.py b/src/llparse_builder/node/match.py new file mode 100644 index 0000000..125eb51 --- /dev/null +++ b/src/llparse_builder/node/match.py @@ -0,0 +1,77 @@ +import sys +from collections.abc import Sequence + +from ..edge import Edge +from ..transform import Transform +from ..utils import to_buffer +from .base import Node + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +MatchSingleValue = str | int | bytes +MatchValue = MatchSingleValue | Sequence[MatchSingleValue] + + +class Match(Node): + __slots__ = ("transform_fn",) + + def __init__(self, name: str) -> None: + super().__init__(name) + self.transform_fn: Transform | None = None + + def transform(self, transform: Transform) -> Self: + self.transform_fn = transform + return self + + def match(self, value: MatchValue, next: Node) -> Self: + if not isinstance(value, (str, bytes, int, bytearray, memoryview)): + for subvalue in value: + self.match(subvalue, next) + return self + + edge = Edge(next, False, to_buffer(value)) + self.add_edge(edge) + return self + + def peek(self, value: MatchValue, next: Node) -> Self: + + if not isinstance(value, str) and isinstance(value, Sequence): + for subvalue in value: + self.peek(subvalue, next) + return self + b = to_buffer(value) + assert len(b) == 1, "`.peek()` accepts only single character keys" + # NOTE: Numbers with peek are considered unacceptable. + # this is because of the loop-checker, therefore we + # transform it into bytes. Got stuck for weeks with this library + # because it was configured as Edge(next, True, ord(b)) Very costly + # mistake, Do NOT DO IT!!! + edge = Edge(next, True, b) + self.add_edge(edge) + return self + + def select( + self, + key_or_map: str | bytes | int | dict[str | bytes | int, int], + value_or_next: int | Node | None = None, + next: Node | None = None, + ): + if isinstance(key_or_map, dict): + assert isinstance(value_or_next, Node), "Invalid next argument of select" + assert next is None, "Inavlid argument count of select" + for k, v in key_or_map.items(): + self.select(k, v, value_or_next) + return self + + assert isinstance(value_or_next, int), "Invalid value argument of select" + assert next is not None, "Invalid next argument of select" + edge = Edge(next, False, to_buffer(key_or_map), value_or_next) + self.add_edge(edge) + return self + + def get_transform(self) -> Transform | None: + return self.transform_fn diff --git a/src/llparse_builder/node/pause.py b/src/llparse_builder/node/pause.py new file mode 100644 index 0000000..b292e13 --- /dev/null +++ b/src/llparse_builder/node/pause.py @@ -0,0 +1,12 @@ +from enum import IntEnum + +from .base import Node + + +class Pause(Node): + __slots__ = ("code", "reason") + + def __init__(self, code: int | IntEnum, reason: str) -> None: + super().__init__("pause") + self.code = code.value if isinstance(code, IntEnum) else code + self.reason = reason diff --git a/src/llparse_builder/node/span_end.py b/src/llparse_builder/node/span_end.py new file mode 100644 index 0000000..8a3dbd1 --- /dev/null +++ b/src/llparse_builder/node/span_end.py @@ -0,0 +1,14 @@ +import typing + +from .base import Node + +if typing.TYPE_CHECKING: + from ..span import Span + + +class SpanEnd(Node): + __slots__ = ("span",) + + def __init__(self, span: "Span"): + super().__init__(f"span_end_{span.callback.name}") + self.span = span diff --git a/src/llparse_builder/node/span_start.py b/src/llparse_builder/node/span_start.py new file mode 100644 index 0000000..2754c0a --- /dev/null +++ b/src/llparse_builder/node/span_start.py @@ -0,0 +1,14 @@ +import typing + +from .base import Node + +if typing.TYPE_CHECKING: + from ..span import Span + + +class SpanStart(Node): + __slots__ = ("span",) + + def __init__(self, span: "Span"): + super().__init__(f"span_start_{span.callback.name}") + self.span = span diff --git a/src/llparse_builder/node/unpack.py b/src/llparse_builder/node/unpack.py new file mode 100644 index 0000000..d91c2fe --- /dev/null +++ b/src/llparse_builder/node/unpack.py @@ -0,0 +1,167 @@ +""" +unpack +------ + +while unpack is not implemented as of yet in typescript llparse +the desire for enabling socket protocols has always been a desire +to extend llparse's usefulness in real world scenarios. + +It is an extension based on some older work that has been +previously proposed to Indutny by Arthurschreiber. + +This change uses python's struct protocols +to determine how the nodes should unpack into a +given property. Know however that it can only take a +single value at a time. + +""" + +import sys +import warnings +from enum import IntEnum +from io import StringIO + +from ..errors import Error +from .base import Node + + +class ByteOrder(IntEnum): + LITTLE = 0 + BIG = 1 + + +# based off CPython _struct.c's whichtable function +def whichtable(pfmt: StringIO) -> ByteOrder: + """Determines what kind of byte format llparse should be using + defaults to big-endian if this can't be determined. + """ + fmt = pfmt.read(1) + match fmt: + case "<": + return ByteOrder.LITTLE + case ">": + return ByteOrder.BIG + case "!": # network byte-order is always big-endian + return ByteOrder.BIG + case "@": + # Just call it big + return ByteOrder.BIG + case "=": + warnings.warn( + "llparse will be compiling based off " + "your cpu's endianness " + "output may result different parsers " + "for different users and developers," + "this may not be what you want." + ) + return ByteOrder.LITTLE if sys.byteorder == "little" else ByteOrder.BIG + case _: + # assume the format is already given to us, big is all we need. + pfmt.seek(0) + return ByteOrder.BIG + + +# SEE: https://docs.python.org/3/library/struct.html#format-characters +# TODO: A new value to map 24 bit integers would be a good idea for +# http2 parsers. +TABLE_SIZE = { + "b": (1, True), + "c": (1, True), + "?": (1, True), + "h": (2, True), + "i": (4, True), + "l": (4, True), + "q": (8, True), + "B": (1, False), + "H": (2, False), + "I": (4, False), + "L": (4, False), + "Q": (8, False), + # Entirely Custom. They are named after the word H'y'brid + "y": (3, True), + "Y": (3, False), + # Throw an issue on Github if you need any values between 5 to 8 +} + + +def whichsize(pfmt: StringIO) -> tuple[int, bool]: + """Determines the length of the byte wanted for parsing with. + It uses python's array.array/struct module number format to + determine the size of the value being unpacked. Know that floats + and doubles are not implemented as llparse has not histrocially + supported it yet. + + - value, type in rust notation, bit length + - 'b' i8, 1 + - 'c' i8, 1 + - '?' bool, 1 + - 'h' i16, 2 + - 'i' i32, 4 + - 'l' i32, 4 + - 'q' i64, 8 + - 'B' u8, 1 + - 'H' u16, 2 + - 'I' u32, 4 + - 'L' u32, 4 + - 'Q' u64, 8 + + These ones are custom and not apart of CPython at all. + They are added incase only 3 bits are needed. They + are used to describe a hybrid (H was already taken by short so will use a y) + - 'y', i24, 3 + - 'Y', u24, 3 + + Example of hybrid's usage could be making a parser for the + socks5 protocol or http2/3. + + """ + value = pfmt.read(1) + + try: + return TABLE_SIZE[value] + except KeyError: + raise Error(f"{value!r} not implemented or is invalid.") + + +def define_name(field: str, bits: int, signed: bool, little_endian: bool) -> str: + result = f"{field}_{'int' if signed else 'uint'}_{bits * 8}" + return result + ("_le" if little_endian else "_be") + + +class Unpack(Node): + __slots__ = ("bits", "field", "little_endian", "signed") + + def __init__(self, field: str, bits: int, signed: bool, little_endian: bool): + if bits <= 0: + raise ValueError("bits should be a positive integer") + if bits > 64: + raise ValueError(f"bits too large, field {field!r} could overflow") + self.field = field + self.bits = bits + self.signed = signed + self.little_endian = little_endian + super().__init__(define_name(field, bits, signed, little_endian)) + + @classmethod + def from_format(cls, field: str, fmt: str): + """Defines the format of an Unpack Node and which direction + to unpack the provided bytes in.""" + fmt_len = len(fmt) + assert fmt_len in (1, 2), "format must be greater than zero or less than 3" + pfmt = StringIO(fmt) + is_le = whichtable(pfmt) == ByteOrder.LITTLE + size, signed = whichsize(pfmt) + return cls(field, size, signed, is_le) + + def otherwise(self): + """ + WARNING! + -------- + + Otherwise is not implemented as + bytes must advance after unpacking + values.""" + raise Error( + "Otherwise is not implemented as " + "skip_to(...) on an unpack variable is required" + ) diff --git a/src/llparse_builder/property.py b/src/llparse_builder/property.py new file mode 100644 index 0000000..7496ade --- /dev/null +++ b/src/llparse_builder/property.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +from typing import Literal, TypeAlias + +from .errors import Error + +PropertyTypes = {"i8", "i16", "i32", "i64", "ptr"} + +PropertyType: TypeAlias = Literal["i8", "i16", "i32", "i64", "ptr"] + + +@dataclass(slots=True) +class Property: + ty: Literal["i8", "i16", "i32", "i64", "ptr"] + name: str + + def __post_init__(self): + assert self.ty in PropertyTypes + if self.name.startswith("_"): + raise Error(f"Can't use internal property name: {self.name!r}") diff --git a/src/llparse_builder/py.typed b/src/llparse_builder/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/llparse_builder/reachability.py b/src/llparse_builder/reachability.py new file mode 100644 index 0000000..d289b55 --- /dev/null +++ b/src/llparse_builder/reachability.py @@ -0,0 +1,24 @@ +from .node import Node + + +class Reachability: + __slots__ = () + + def build(self, root: Node) -> list[Node]: + res: set[Node] = set() + res_add = res.add + queue = [root] + while queue: + node = queue.pop() + if node in res: + continue + + res_add(node) + for edge in node: + queue.append(edge.node) + + # little shortcut that is not in the typescript code. + if otherwise := node.get_otherwise_edge(): + queue.append(otherwise.node) + + return list(res) diff --git a/src/llparse_builder/span.py b/src/llparse_builder/span.py new file mode 100644 index 0000000..f008885 --- /dev/null +++ b/src/llparse_builder/span.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass, field + +from .code.span import Span as SpanCallback +from .node import Node, SpanEnd, SpanStart + + +@dataclass(slots=True) +class Span: + callback: SpanCallback + start_cache: dict[Node, SpanStart] = field(init=False, default_factory=dict) + end_cache: dict[Node, SpanStart] = field(init=False, default_factory=dict) + + def start(self, otherwise: Node | None = None) -> SpanStart: + if otherwise is not None and otherwise in self.start_cache: + return self.start_cache[otherwise] + + res = SpanStart(self) + if otherwise is not None: + res.otherwise(otherwise) + self.start_cache[otherwise] = res + return res + + def end(self, otherwise: Node | None = None) -> SpanEnd: + if otherwise is not None and otherwise in self.end_cache: + return self.end_cache[otherwise] + + res = SpanEnd(self) + if otherwise is not None: + res.otherwise(otherwise) + self.end_cache[otherwise] = res + return res + + def __hash__(self) -> int: + return hash(self.callback.name) diff --git a/src/llparse_builder/span_allocator.py b/src/llparse_builder/span_allocator.py new file mode 100644 index 0000000..6d8297b --- /dev/null +++ b/src/llparse_builder/span_allocator.py @@ -0,0 +1,163 @@ +from collections import defaultdict +from dataclasses import dataclass, field +from logging import getLogger + +from .errors import Error +from .node import Node, SpanEnd, SpanStart +from .reachability import Reachability +from .span import Span + +LOG = getLogger(__name__) + + +def _id(node: SpanStart | SpanEnd) -> Span: + return node.span + + +@dataclass(slots=True) +class ISpanActiveInfo: + active: defaultdict[Node, set[Span]] = field( + default_factory=lambda: defaultdict(set) + ) + spans: list[Span] = field(default_factory=list) + + +@dataclass(slots=True) +class ISpanAllocatorResult: + colors: dict[Span, int] = field(default_factory=dict) + concurrency: list[list[Span]] = field(default_factory=list) + max: int = field(default_factory=int) + + +# Personally I think having this function +# outside of color(...) function is a bit cleaner +# we could always change the name of this function +# specifically in the future +def _allocate(this: "SpanAllocator", span: Span) -> int: + if span in this._colors: + return this._colors[span] + + overlap = this._overlap_map[span] + + used: set[int] = set() + for sub_span in overlap: + if sub_span in this._colors: + used.add(this._colors.get(sub_span)) + + # XXX: Is this optimizable? + i = 0 + while i in used: + i += 1 + + this._mx = max(this._mx, i) + this._colors[span] = i + return i + + +class SpanAllocator: + __slots__ = ("_colors", "_mx", "_overlap_map") + + def __init__(self) -> None: + return + + def allocate(self, root: Node): + # Less numbers needing crunching if everything + # is more glued together. + info = self.compute_active(Reachability().build(root)) + + self.check(info) + return self.color(info.spans, self.compute_overlap(info)) + + def check(self, info: ISpanActiveInfo) -> None: + for node, spans in info.active.items(): + for edge in node.get_all_edges(): + edge_node = edge.node + if isinstance(edge_node, SpanStart): + continue + + # Typescript LLParse NOTE: Skip terminal nodes + if not edge_node.get_all_edges(): + continue + + if not LOG.disabled: + LOG.debug(f"checking edge from {node.name!r} to {edge_node.name!r}") + + for sub_span in info.active[edge_node]: + if sub_span not in spans: + raise Error( + f"unmatched span end for {sub_span.callback.name!r}" + f"at {edge_node.name!r}, coming from {node.name!r}" + ) + + if isinstance(edge_node, SpanEnd): + span = edge_node.span + if span not in spans: + raise Error(f"unmatched span end for {span.callback.name!r}") + + def compute_active(self, nodes: list[Node]) -> ISpanActiveInfo: + active_map: defaultdict[Node, set[Node]] = defaultdict(set) + + queue = set(nodes) + spans: set[Span] = set() + # This fixes an issue when using a for loop which unlike in typescript + # we cannot remove items when in a for-loop in python this also ensures + # that all spans are visited. + while queue: + node = queue.pop() + active = active_map[node] + if isinstance(node, SpanStart): + span = _id(node) + spans.add(span) + active.add(span) + + for span in active: + if isinstance(node, SpanEnd) and span == _id(node): + break + + for edge in node.get_all_edges(): + edge_node = edge.node + + if isinstance(edge_node, SpanStart) and _id(edge_node) == span: + raise Error( + f"Detected loop in span {span.callback.name!r} at {node.name!r}" + ) + + edgeActive = active_map[edge_node] + if span in edgeActive: + break + + edgeActive.add(span) + queue.add(edge_node) + + return ISpanActiveInfo(active=active_map, spans=list(spans)) + + def compute_overlap(self, info: ISpanActiveInfo) -> defaultdict[Span, set[Span]]: + overlap: defaultdict[Span, set[Span]] = defaultdict(set) + for spans in info.active.values(): + # Itertools might be a good option here... + for one in spans: + for other in spans: + if other != one: + overlap[one].add(other) + return overlap + + def color( + self, spans: list[Span], overlap_map: dict[Node, set[Span]] + ) -> ISpanAllocatorResult: + # Used _max instead of max because max() is an api called function needed in a bit... + self._mx = -1 + self._colors: dict[Span, int] = {} + + self._overlap_map = overlap_map + + colors = {span: _allocate(self, span) for span in spans} + + # This algorythm is a little bit less complicated than typescript + # llparse so lets let it stay the way it is... + + # TODO: Maybe concurrency should've been a defaultdict? + + concurrency: list[list[Span]] = [[] for _ in range(self._mx + 1)] + for s in sorted(spans, key=lambda s: s.callback.name): + concurrency[_allocate(self, s)].append(s) + return ISpanAllocatorResult(colors, concurrency, self._mx) diff --git a/src/llparse_builder/transform/__init__.py b/src/llparse_builder/transform/__init__.py new file mode 100644 index 0000000..409dcd3 --- /dev/null +++ b/src/llparse_builder/transform/__init__.py @@ -0,0 +1,5 @@ +from .base import Transform +from .creator import Creator +from .to_lower_unsafe import ToLowerUnsafe + +__all__ = ("Creator", "ToLowerUnsafe", "Transform") diff --git a/src/llparse_builder/transform/base.py b/src/llparse_builder/transform/base.py new file mode 100644 index 0000000..9413d6d --- /dev/null +++ b/src/llparse_builder/transform/base.py @@ -0,0 +1,8 @@ +from typing import Literal + + +class Transform: + __slots__ = ("name",) + + def __init__(self, name: Literal["to_lower_unsafe", "to_lower"]): + self.name = name diff --git a/src/llparse_builder/transform/creator.py b/src/llparse_builder/transform/creator.py new file mode 100644 index 0000000..5cf73dc --- /dev/null +++ b/src/llparse_builder/transform/creator.py @@ -0,0 +1,12 @@ +from .to_lower import ToLower +from .to_lower_unsafe import ToLowerUnsafe + + +class Creator: + __slots__ = () + + def to_lower_unsafe(self): + return ToLowerUnsafe() + + def to_lower(self): + return ToLower() diff --git a/src/llparse_builder/transform/to_lower.py b/src/llparse_builder/transform/to_lower.py new file mode 100644 index 0000000..54d7bfb --- /dev/null +++ b/src/llparse_builder/transform/to_lower.py @@ -0,0 +1,8 @@ +from .base import Transform + + +class ToLower(Transform): + __slots__ = ("name",) + + def __init__(self) -> None: + super().__init__("to_lower") diff --git a/src/llparse_builder/transform/to_lower_unsafe.py b/src/llparse_builder/transform/to_lower_unsafe.py new file mode 100644 index 0000000..a705968 --- /dev/null +++ b/src/llparse_builder/transform/to_lower_unsafe.py @@ -0,0 +1,8 @@ +from .base import Transform + + +class ToLowerUnsafe(Transform): + __slots__ = ("name",) + + def __init__(self) -> None: + super().__init__("to_lower_unsafe") diff --git a/src/llparse_builder/unpack_creator.py b/src/llparse_builder/unpack_creator.py new file mode 100644 index 0000000..d2ea8a4 --- /dev/null +++ b/src/llparse_builder/unpack_creator.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass, field + +from .node import Node, Unpack + + +@dataclass(slots=True) +class StructUnpack: + fmt: str + cache: dict[Node, list[Unpack]] = field(default_factory=dict, init=False) + + def _unpack_items(self, node: Node) -> list[Unpack]: + if node not in self.cache: + unpacks: list[Unpack] = [] + for single in self.fmt.split("|"): + name, single_format = single.split(":", 1) + unpacks.append(Unpack.from_format(name, single_format)) + self.cache[node] = unpacks + return unpacks + else: + return self.cache[node] + + def unpack(self, node: Node) -> Unpack: + """unpack from given format and then move onto this node""" + items = self._unpack_items(node) + items[-1].skip_to(node) + return items[0] + + +@dataclass(slots=True) +class UnpackCreator: + cache: dict[str, StructUnpack] = field(default_factory=dict, init=False) + + def struct(self, fmt: str) -> StructUnpack: + """Uses a custom structure for providing multiple ways to handle + unpackable formats + + :param fmt: format of the nodes + `name:format|name:format` + """ + if fmt in self.cache: + return self.cache[fmt] + s = StructUnpack(fmt) + self.cache[fmt] = s + return s + + def with_format(self, fmt: str, node: Node) -> Unpack: + """Unpack using this format and then move onto this node. + :param fmt: format of the nodes + `name:format|name:format` + :param node: the node to move to after finishing. + + :returns: the starting unpack node. + """ + return self.struct(fmt).unpack(node) diff --git a/src/llparse_builder/utils.py b/src/llparse_builder/utils.py new file mode 100644 index 0000000..37362a7 --- /dev/null +++ b/src/llparse_builder/utils.py @@ -0,0 +1,10 @@ +def to_buffer(value: int | str | bytes): + if isinstance(value, bytes): + res = value + elif isinstance(value, str): + res = value.encode("utf-8") + else: + assert value <= 0 and value <= 0xFF, "Invalid byte value" + res = bytes([value]) + assert len(res), "Invalid key length" + return res diff --git a/src/llparse_frontend/__init__.py b/src/llparse_frontend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/llparse_frontend/code/__init__.py b/src/llparse_frontend/code/__init__.py new file mode 100644 index 0000000..7bc1e82 --- /dev/null +++ b/src/llparse_frontend/code/__init__.py @@ -0,0 +1,35 @@ +from .and_ import And +from .base import Code, Signature +from .external import External +from .field import Field +from .field_value import FieldValue +from .is_equal import IsEqual +from .load import Load +from .match import Match +from .mul_add import MulAdd, MulAddOptions +from .or_ import Or +from .span import Span +from .store import Store +from .test import Test +from .update import Update +from .value import Value + +__all__ = ( + "And", + "Code", + "External", + "Field", + "FieldValue", + "IsEqual", + "Load", + "Match", + "MulAdd", + "MulAddOptions", + "Or", + "Signature", + "Span", + "Store", + "Test", + "Update", + "Value", +) diff --git a/src/llparse_frontend/code/and_.py b/src/llparse_frontend/code/and_.py new file mode 100644 index 0000000..fafe539 --- /dev/null +++ b/src/llparse_frontend/code/and_.py @@ -0,0 +1,9 @@ +from ..utils import to_cache_key +from .field_value import FieldValue + + +class And(FieldValue): + def __init__(self, name: str, field: str, value: int): + super().__init__( + "match", f"and_{field}_{to_cache_key(value)}", name, field, value + ) diff --git a/src/llparse_frontend/code/base.py b/src/llparse_frontend/code/base.py new file mode 100644 index 0000000..1d54a3a --- /dev/null +++ b/src/llparse_frontend/code/base.py @@ -0,0 +1,14 @@ +from typing import Literal, TypeAlias + +Signature: TypeAlias = Literal["match", "value", "span"] + + +class Code: + __slots__ = ("cache_key", "name", "signature") + + def __init__( + self, signature: Literal["match", "value", "span"], cache_key: str, name: str + ) -> None: + self.signature = signature + self.cache_key = cache_key + self.name = name diff --git a/src/llparse_frontend/code/external.py b/src/llparse_frontend/code/external.py new file mode 100644 index 0000000..3cf66fb --- /dev/null +++ b/src/llparse_frontend/code/external.py @@ -0,0 +1,6 @@ +from .base import Code, Signature + + +class External(Code): + def __init__(self, signature: Signature, name: str): + super().__init__(signature, f"external_{name}", name) diff --git a/src/llparse_frontend/code/field.py b/src/llparse_frontend/code/field.py new file mode 100644 index 0000000..393f19e --- /dev/null +++ b/src/llparse_frontend/code/field.py @@ -0,0 +1,9 @@ +from .base import Code, Signature + + +class Field(Code): + __slots__ = ("field",) + + def __init__(self, signature: Signature, cache_key: str, name: str, field: str): + super().__init__(signature, cache_key, name) + self.field = field diff --git a/src/llparse_frontend/code/field_value.py b/src/llparse_frontend/code/field_value.py new file mode 100644 index 0000000..27ff836 --- /dev/null +++ b/src/llparse_frontend/code/field_value.py @@ -0,0 +1,12 @@ +from .base import Signature +from .field import Field + + +class FieldValue(Field): + __slots__ = ("value",) + + def __init__( + self, signature: Signature, cache_key: str, name: str, field: str, value: int + ): + super().__init__(signature, cache_key, name, field) + self.value = value diff --git a/src/llparse_frontend/code/is_equal.py b/src/llparse_frontend/code/is_equal.py new file mode 100644 index 0000000..2548866 --- /dev/null +++ b/src/llparse_frontend/code/is_equal.py @@ -0,0 +1,9 @@ +from ..utils import to_cache_key +from .field_value import FieldValue + + +class IsEqual(FieldValue): + def __init__(self, name: str, field: str, value: int) -> None: + super().__init__( + "match", f"is_equal_{field}_{to_cache_key(value)}", name, field, value + ) diff --git a/src/llparse_frontend/code/load.py b/src/llparse_frontend/code/load.py new file mode 100644 index 0000000..36c2b3f --- /dev/null +++ b/src/llparse_frontend/code/load.py @@ -0,0 +1,6 @@ +from .field import Field + + +class Load(Field): + def __init__(self, name: str, field: str) -> None: + super().__init__("match", f"load_{field}", name, field) diff --git a/src/llparse_frontend/code/match.py b/src/llparse_frontend/code/match.py new file mode 100644 index 0000000..6e0cfd3 --- /dev/null +++ b/src/llparse_frontend/code/match.py @@ -0,0 +1,6 @@ +from .external import External + + +class Match(External): + def __init__(self, name: str): + super().__init__("match", name) diff --git a/src/llparse_frontend/code/mul_add.py b/src/llparse_frontend/code/mul_add.py new file mode 100644 index 0000000..2833a2e --- /dev/null +++ b/src/llparse_frontend/code/mul_add.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass + +from ..utils import to_cache_key +from .field import Field + + +@dataclass(slots=True, frozen=True) +class MulAddOptions: + base: int + max: int | None + signed: bool + + +def to_options_key(options: MulAddOptions) -> str: + res = f"base_{to_cache_key(options.base)}" + if options.max is not None: + res += f"_max_{to_cache_key(options.max)}" + if options.signed is not None: + res += f"_signed_{to_cache_key(options.signed)}" + return res + + +class MulAdd(Field): + __slots__ = ("options",) + + def __init__(self, name: str, field: str, options: MulAddOptions): + super().__init__( + "value", f"mul_add_{field}_{to_options_key(options)}", name, field + ) + self.options = options diff --git a/src/llparse_frontend/code/or_.py b/src/llparse_frontend/code/or_.py new file mode 100644 index 0000000..dbed1ed --- /dev/null +++ b/src/llparse_frontend/code/or_.py @@ -0,0 +1,9 @@ +from ..utils import to_cache_key +from .field_value import FieldValue + + +class Or(FieldValue): + def __init__(self, name: str, field: str, value: int): + super().__init__( + "match", f"or_{field}_{to_cache_key(value)}", name, field, value + ) diff --git a/src/llparse_frontend/code/span.py b/src/llparse_frontend/code/span.py new file mode 100644 index 0000000..cb23b09 --- /dev/null +++ b/src/llparse_frontend/code/span.py @@ -0,0 +1,6 @@ +from .external import External + + +class Span(External): + def __init__(self, name: str) -> None: + super().__init__("span", name) diff --git a/src/llparse_frontend/code/store.py b/src/llparse_frontend/code/store.py new file mode 100644 index 0000000..8f03d00 --- /dev/null +++ b/src/llparse_frontend/code/store.py @@ -0,0 +1,6 @@ +from .field_value import Field + + +class Store(Field): + def __init__(self, name: str, field: str): + super().__init__("value", f"store_{field}", name, field) diff --git a/src/llparse_frontend/code/test.py b/src/llparse_frontend/code/test.py new file mode 100644 index 0000000..0505a82 --- /dev/null +++ b/src/llparse_frontend/code/test.py @@ -0,0 +1,9 @@ +from ..utils import to_cache_key +from .field_value import FieldValue + + +class Test(FieldValue): + def __init__(self, name: str, field: str, value: int): + super().__init__( + "match", f"test_{field}_{to_cache_key(value)}", name, field, value + ) diff --git a/src/llparse_frontend/code/update.py b/src/llparse_frontend/code/update.py new file mode 100644 index 0000000..2fc6a73 --- /dev/null +++ b/src/llparse_frontend/code/update.py @@ -0,0 +1,9 @@ +from ..utils import to_cache_key +from .field_value import FieldValue + + +class Update(FieldValue): + def __init__(self, name: str, field: str, value: int): + super().__init__( + "match", f"update_{field}_{to_cache_key(value)}", name, field, value + ) diff --git a/src/llparse_frontend/code/value.py b/src/llparse_frontend/code/value.py new file mode 100644 index 0000000..21afe52 --- /dev/null +++ b/src/llparse_frontend/code/value.py @@ -0,0 +1,6 @@ +from .external import External + + +class Value(External): + def __init__(self, name: str) -> None: + super().__init__("value", name) diff --git a/src/llparse_frontend/container/__init__.py b/src/llparse_frontend/container/__init__.py new file mode 100644 index 0000000..98e932d --- /dev/null +++ b/src/llparse_frontend/container/__init__.py @@ -0,0 +1,71 @@ +from collections.abc import Callable +from functools import partial + +from ..implementation import ( + CodeImplementation, + Implementation, + NodeImplementation, + TransformImplementation, +) +from ..wrap import T, Wrap +from .wrap import ContainerWrap + + +class Container: + __slots__ = ("map",) + + def __init__(self) -> None: + self.map: dict[str, Implementation] = {} + + def add(self, key: str, impl: Implementation) -> None: + assert key not in self.map, f'Duplicate implementation key: "{key}"' + self.map[key] = impl + + def build(self) -> Implementation: + return Implementation( + self.build_code(), self.build_node(), self.build_transform() + ) + + def build_code(self): + return CodeImplementation( + And=self.combine(lambda impl: impl.code.And), + IsEqual=self.combine(lambda impl: impl.code.IsEqual), + Load=self.combine(lambda impl: impl.code.Load), + Match=self.combine(lambda impl: impl.code.Match), + MulAdd=self.combine(lambda impl: impl.code.MulAdd), + Or=self.combine(lambda impl: impl.code.Or), + Span=self.combine(lambda impl: impl.code.Span), + Store=self.combine(lambda impl: impl.code.Store), + Test=self.combine(lambda impl: impl.code.Test), + Update=self.combine(lambda impl: impl.code.Update), + Value=self.combine(lambda impl: impl.code.Value), + ) + + def build_node(self): + return NodeImplementation( + Consume=self.combine(lambda impl: impl.node.Consume), + Empty=self.combine(lambda impl: impl.node.Empty), + Error=self.combine(lambda impl: impl.node.Error), + Invoke=self.combine(lambda impl: impl.node.Invoke), + Pause=self.combine(lambda impl: impl.node.Pause), + Sequence=self.combine(lambda impl: impl.node.Sequence), + Single=self.combine(lambda impl: impl.node.Single), + SpanEnd=self.combine(lambda impl: impl.node.SpanEnd), + SpanStart=self.combine(lambda impl: impl.node.SpanStart), + TableLookup=self.combine(lambda impl: impl.node.TableLookup), + Unpack=self.combine(lambda impl: impl.node.Unpack), + ) + + def build_transform(self) -> TransformImplementation: + return TransformImplementation( + ID=self.combine(lambda impl: impl.transform.ID), + ToLower=self.combine(lambda impl: impl.transform.ToLower), + ToLowerUnsafe=self.combine(lambda impl: impl.transform.ToLowerUnsafe), + ) + + def combine( + self, gather: Callable[[Implementation], Callable[[T], Wrap[T]]] + ) -> Callable[[T], ContainerWrap[T, Wrap[T]]]: + return partial( + ContainerWrap, map={key: gather(impl) for key, impl in self.map.items()} + ) diff --git a/src/llparse_frontend/container/wrap.py b/src/llparse_frontend/container/wrap.py new file mode 100644 index 0000000..29a4ce7 --- /dev/null +++ b/src/llparse_frontend/container/wrap.py @@ -0,0 +1,24 @@ +from collections.abc import Callable +from typing import Generic, TypeVar + +from ..wrap import T, Wrap + +R = TypeVar("R", bound=Wrap) + +# It can be quite confusing in typescript but +# in python we are simply calling the node implementation +# that we were looking for by calling it's constructor. + + +class ContainerWrap(Wrap[T], Generic[T, R]): + __slots__ = ("map",) + + def __init__(self, ref: T, map: dict[str, Callable[[T], R]]): + super().__init__(ref) + self.map = map + + def get(self, key: str) -> R: + assert key in self.map, f'Unknown implementation key "{key}"' + # Typescript doesn't show it but it's simply storing + # the node inside our implementation. Example: "C" + return self.map[key](self.ref) diff --git a/src/llparse_frontend/enumerator.py b/src/llparse_frontend/enumerator.py new file mode 100644 index 0000000..451f030 --- /dev/null +++ b/src/llparse_frontend/enumerator.py @@ -0,0 +1,21 @@ +from collections import deque + +from .node import Node +from .wrap import Wrap + + +def get_all_nodes(root: Wrap[Node]) -> list[Wrap[Node]]: + nodes: set[Wrap[Node]] = set() + queue = deque([root]) + + while queue: + node = queue.pop() + for slot in node.ref.get_slots(): + slot_node = slot.node + + if slot_node in nodes: + continue + nodes.add(slot_node) + queue.append(slot_node) + + return list(nodes) diff --git a/src/llparse_frontend/errors.py b/src/llparse_frontend/errors.py new file mode 100644 index 0000000..b64ec3b --- /dev/null +++ b/src/llparse_frontend/errors.py @@ -0,0 +1,2 @@ +class Error(Exception): + """Exceptions related to llparse-frontend.""" diff --git a/src/llparse_frontend/frontend.py b/src/llparse_frontend/frontend.py new file mode 100644 index 0000000..c83309b --- /dev/null +++ b/src/llparse_frontend/frontend.py @@ -0,0 +1,547 @@ +from dataclasses import dataclass, field +from logging import getLogger +from typing import TypedDict + +from llparse_builder import builder as source + +from . import enumerator, peephole +from .errors import Error +from .implementation import Implementation +from .namespace import frontend +from .node.single import ReadonlySingleEdge +from .span_field import SpanField +from .trie import Trie, TrieEmpty, TrieNode, TrieSequence, TrieSingle, TrieSingleChild +from .utils import Identifier +from .wrap import Wrap + +logger = getLogger(__name__) +debug = logger.debug + +DEFAULT_MIN_TABLE_SIZE = 32 +DEFAULT_MAX_TABLE_WIDTH = 4 + + +class FrontendLazyOptions(TypedDict, total=False): + max_table_elem_width: int | None = None + min_table_size: int | None = None + + +@dataclass(slots=True, frozen=True) +class FrontendResult: + prefix: str + properties: list[source.Property] + root: Wrap[frontend.node.Node] + spans: list[SpanField] + resumption_targets: list[Wrap[frontend.node.Node]] + + +@dataclass(slots=True) +class TableLookupTarget: + trie: TrieEmpty + no_advance: bool + keys: list[int] = field(default_factory=list) + + +class Frontend: + __slots__ = ( + "code_cache", + "code_id", + "id", + "implementation", + "map", + "options", + "prefix", + "resumption_targets", + "span_map", + ) + + def __init__( + self, + prefix: str, + implementation: Implementation, + max_table_elem_width: int | None = None, + min_table_size: int | None = None, + ) -> None: + self.prefix = prefix + self.id = Identifier(f"{prefix}__n_") + self.code_id = Identifier(f"{prefix}__c_") + self.implementation = implementation + self.resumption_targets = set() + self.code_cache = {} + self.span_map = {} + self.map = {} + + self.options: FrontendLazyOptions = { + "max_table_elem_width": max_table_elem_width or DEFAULT_MAX_TABLE_WIDTH, + "min_table_size": min_table_size or DEFAULT_MIN_TABLE_SIZE, + } + assert self.options["max_table_elem_width"] > 0, ( + "Invalid `max_table_elem_width`, must be positive" + ) + + def compile( + self, root: source.node.Node, properties: list[source.Property] + ) -> FrontendResult: + debug("checking loops") + lc = source.LoopChecker() + lc.check(root) + + debug("allocating spans") + span_allocator = source.SpanAllocator() + source_spans = span_allocator.allocate(root) + + spans: list[SpanField] = [] + for index, concurrent in enumerate(source_spans.concurrency): + span = SpanField( + index, [self.translate_code(c.callback) for c in concurrent] + ) + + for sourceSpan in concurrent: + self.span_map[sourceSpan] = span + + spans.append(span) + + debug("translating") + out = self.translate(root) + + # Enumerate + debug("enumerating") + nodes = enumerator.get_all_nodes(out) + # Peephole optimizations... + debug("peephole optimization") + out = peephole.optimize(out, nodes) + + # Re-Enumerate + debug("re-enumerating") + nodes = enumerator.get_all_nodes(out) + + debug("registering resumption targets") + + # DONT FORGET TO ADD "OUT" TO THE RESUMPTION TARGETS!!! + self.resumption_targets.add(out) + + # Register resumption targets... + for node in nodes: + self.register_node(node) + + return FrontendResult( + prefix=self.prefix, + properties=properties, + resumption_targets=self.resumption_targets, + root=out, + spans=spans, + ) + + def register_node(self, node: Wrap[frontend.node.Node]) -> None: + if isinstance( + node.ref, + ( + frontend.node.Unpack, + frontend.node.Consume, + frontend.node.Empty, + frontend.node.Sequence, + frontend.node.Single, + frontend.node.TableLookup, + frontend.node.SpanStart, + ), + ): + self.resumption_targets.add(node) + elif isinstance(node.ref, (frontend.node.Pause, frontend.node.SpanEnd)): + self.resumption_targets.add(node.ref.otherwise.node) + + def translate_match( + self, node: source.node.Match + ) -> list[Wrap[frontend.node.Node]]: + trie = Trie(node.name) + assert node.get_otherwise_edge() + trieNode = trie.build(list(node)) + + if not trieNode: + return self.implementation.node.Empty( + frontend.node.Empty(self.id.id(node.name)) + ) + + children: list = [] + + self.translate_trie(node, trieNode, children) + assert children + + return children + + def translate(self, node: source.node.Node) -> Wrap[frontend.node.Node]: + if node in self.map: + return self.map[node] + + def id(): + return self.id.id(node.name) + + node_impl = self.implementation.node + + result = None + if isinstance(node, source.node.Error): + result = node_impl.Error(frontend.node.Error(id(), node.code, node.reason)) + + elif isinstance(node, source.node.Pause): + result = node_impl.Pause(frontend.node.Pause(id(), node.code, node.reason)) + + elif isinstance(node, source.node.Consume): + result = node_impl.Consume(frontend.node.Consume(id(), node.field)) + + elif isinstance(node, source.node.SpanStart): + result = node_impl.SpanStart( + frontend.node.SpanStart( + id(), + self.span_map[node.span], + self.translate_span_code(node.span.callback), + ) + ) + + elif isinstance(node, source.node.SpanEnd): + result = node_impl.SpanEnd( + frontend.node.SpanEnd( + id(), + self.span_map[node.span], + self.translate_span_code(node.span.callback), + ) + ) + + elif isinstance(node, source.node.Invoke): + assert node.code.signature in ["match", "value"], ( + "Passing `span` callback to `invoke` is not allowed" + ) + result = node_impl.Invoke( + frontend.node.Invoke(id(), self.translate_code(node.code)) + ) + + elif isinstance(node, source.node.Match): + result = self.translate_match(node) + + elif isinstance(node, source.node.Unpack): + result = self.translate_unpack(node) + + else: + raise Error(f'Unknown Node Type for :"{node.name}" {type(node)}') + + otherwise = node.get_otherwise_edge() + + if isinstance(result, list): + # result:list[WrappedNode] + + assert isinstance(node, (source.node.Match, source.node.Unpack)) + _match = node + + assert otherwise, f'Node "{node.name}" has no ".otherwise()"' + + if isinstance(node, source.node.Match): + for child in result: + if not child.ref.otherwise: + child.ref.set_otherwise( + self.translate(otherwise.node), otherwise.no_advance + ) + transform = self.translate_transform(_match.get_transform()) + for child in result: + # TODO Vizonex : This might break , be sure to make a workaround function here... + child.ref.set_transform(transform) + + else: + result[-1].ref.set_otherwise( + self.translate(otherwise.node), otherwise.no_advance + ) + assert len(result) >= 1 + return result[0] + + else: + single = result + + assert isinstance(single.ref, frontend.node.Node) + + self.map[node] = single + + if otherwise is not None: + single.ref.set_otherwise( + self.translate(otherwise.node), otherwise.no_advance + ) + + else: + assert isinstance(node, source.node.Error), ( + f'Node "{node.name}" has no `.otherwise()' + ) + + if isinstance(single.ref, frontend.node.Invoke): + for edge in node: + single.ref.add_edge( + ord(edge.key) if isinstance(edge.key, str) else edge.key, + self.translate(edge.node), + ) + else: + assert node + + return single + + def translate_unpack( + self, node: source.node.Unpack + ) -> list[Wrap[frontend.node.Unpack]]: + inner = frontend.node.Unpack( + self.Id.id(node.name), + node.field, + node.bits, + node.signed, + node.little_endian, + 0, + ) + result = [self.implementation.node.Unpack(inner)] + # front is to avoid overlapping with python's functions (aka next) + front = self.map[node] = result[0] + + for offset in range(1, node.bits): + unique_name = self.id.id(f"{node.name}_byte{offset + 1}") + inner = frontend.node.Unpack( + unique_name, + node.field, + node.bits, + node.signed, + node.little_endian, + offset, + ) + outer = self.implementation.node.Unpack(inner) + result.append(outer) + # unpack will advance since we are unpacking values... + front.ref.set_otherwise(outer, False) + front = result[-1] + return result + + def maybe_table_lookup( + self, + node: source.code.Match, + trie: TrieSingle, + children: list[Wrap[frontend.node.Node]], + ): + if len(trie.children) < self.options["min_table_size"]: + return None + + targets: dict[source.code.Node, TableLookupTarget] = {} + + def check_child(child: TrieSingleChild): + nonlocal targets + if not isinstance(child.node, TrieEmpty): + debug(f'non-leaf trie child of "{node.name}" prevents table allocation') + return False + empty = child.node + if empty.value is not None: + debug( + f'value passing trie leaf of "{node.name}" prevents table allocation' + ) + return False + + target = empty.node + if target not in targets: + targets[target] = TableLookupTarget( + keys=[child.key], no_advance=child.no_advance, trie=empty + ) + return True + + existing = targets[target] + if existing.no_advance != child.no_advance: + debug( + f'no_advance mismatch in a trie leaf of "{node.name}" prevents ' + "table allocation" + ) + return False + existing.keys.append(child.key) + return True + + if not all(check_child(child) for child in trie.children): + return + + # Weave width limit for optimization... + if len(targets) >= (1 << self.options["max_table_elem_width"]): + debug( + f'too many different trie targets of "{node.name}" for a table allocation' + ) + return + + table = self.implementation.node.TableLookup( + frontend.node.TableLookup(self.id.id(node.name)) + ) + children.append(table) + + # Break Loop + if not self.map.get(node): + self.map[node] = table + + for target in targets.values(): + _next = self.translate_trie(node, target.trie, children) + table.ref.add_edge( + frontend.node.ReadonlyTableEdge( + keys=target.keys, no_advance=target.no_advance, node=_next + ) + ) + + # print('optimized "%s" to a table lookup node' % node.name) + # Node Has been Optimized to a table Lookup , Now return... + return table + + def translate_sequence( + self, + node: source.node.Match, + trie: TrieSequence, + children: list[Wrap[frontend.node.Node]], + ) -> Wrap[frontend.node.Match]: + sequence = self.implementation.node.Sequence( + frontend.node.Sequence(self.id.id(node.name), trie.select) + ) + + children.append(sequence) + + if not self.map.get(node): + self.map[node] = sequence + + child_node = self.translate_trie(node, trie.child, children) + + value = trie.child.value if isinstance(trie.child, TrieEmpty) else None + + sequence.ref.set_edge(child_node, value) + + return sequence + + def translate_trie( + self, + node: source.code.Match, + trie: TrieNode, + children: list[Wrap[frontend.node.Node]], + ): + if isinstance(trie, TrieEmpty): + assert self.map.get(node) + return self.translate(trie.node) + elif isinstance(trie, TrieSingle): + return self.translate_single(node, trie, children) + elif isinstance(trie, TrieSequence): + return self.translate_sequence(node, trie, children) + else: + raise Error("Unknown trie node") + + def translate_single( + self, + node: source.code.Match, + trie: TrieSingle, + children: list[Wrap[frontend.node.Node]], + ): + # Check if Tablelookup could be a valid option to Optimze our code up... + if maybeTable := self.maybe_table_lookup(node, trie, children): + return maybeTable + + single = self.implementation.node.Single( + frontend.node.Single(self.id.id(node.name)) + ) + children.append(single) + + # Break loop... + if node not in self.map: + self.map[node] = single + + for child in trie.children: + child_node = self.translate_trie(node, child.node, children) + + single.ref.add_edge( + ReadonlySingleEdge( + key=child.key, + no_advance=child.no_advance, + node=child_node, + value=child.node.value + if isinstance(child.node, TrieEmpty) + else None, + ) + ) + + if otherwise := trie.otherwise: + single.ref.set_otherwise( + self.translate_trie(node, otherwise, children), True, otherwise.value + ) + return single + + def translate_span_code(self, code: source.code.Span): + return self.translate_code(code) + + def translate_code(self, code: source.code.Code): + """Translates Builder Classes to Frontend Classes...""" + + prefixed = self.code_id.id(code.name).name + code_impl = self.implementation.code + + # res : WrappedCode + if isinstance(code, source.code.IsEqual): + res = code_impl.IsEqual( + frontend.code.IsEqual(prefixed, code.field, code.value) + ) + + elif isinstance(code, source.code.Load): + res = code_impl.Load(frontend.code.Load(prefixed, code.field)) + + elif isinstance(code, source.code.MulAdd): + m = frontend.code.MulAdd( + prefixed, + code.field, + frontend.code.MulAddOptions( + code.options.base, code.options.max, code.options.signed + ), + ) + res = code_impl.MulAdd(m) + + elif isinstance(code, source.code.And): + # NOTE (Vizonex) I did see the frontend on the Typescript Version Using "Or" instead of "And" + # line 460 of llparse-frontend/src/frontend.ts + # So I'm wondering if that was an accident or by design. Might need to Open A Github issue about it... + res = code_impl.And(frontend.code.And(prefixed, code.field, code.value)) + elif isinstance(code, source.code.Or): + res = code_impl.Or(frontend.code.Or(prefixed, code.field, code.value)) + + elif isinstance(code, source.code.Store): + res = code_impl.Store(frontend.code.Store(code.name, code.field)) + + elif isinstance(code, source.code.Test): + res = code_impl.Test(frontend.code.Test(prefixed, code.field, code.value)) + + elif isinstance(code, source.code.Update): + res = code_impl.Update( + frontend.code.Update(prefixed, code.field, code.value) + ) + + # External Callbacks... + + elif isinstance(code, source.code.Span): + res = code_impl.External(frontend.code.Span(code.name)) + + elif isinstance(code, source.code.Match): + res = code_impl.External(frontend.code.Match(code.name)) + + elif isinstance(code, source.code.Value): + res = code_impl.External(frontend.code.Value(code.name)) + + else: + raise Error(f'Un-supported code:"{code.name}" type: "{type(code)}"') + + cache_key = res.ref.cache_key + if _res := self.code_cache.get(cache_key): + return _res + self.code_cache[cache_key] = res + return res + + def translate_transform( + self, transform: source.transform.Transform | None + ) -> Wrap[ + frontend.transform.Transform + | frontend.transform.ID + | frontend.transform.ToLower + | frontend.transform.ToLowerUnsafe + ]: + transform_impl = self.implementation.transform + if not transform or transform.name == "id": + return transform_impl.ID(frontend.transform.ID()) + + match transform.name: + case "to_lower": + return transform_impl.ToLower(frontend.transform.ToLower()) + case "to_lower_unsafe": + return transform_impl.ToLowerUnsafe(frontend.transform.ToLowerUnsafe()) + case _: + raise Error(f"Unexpected transform name named {transform.name!r}") diff --git a/src/llparse_frontend/implementation/__init__.py b/src/llparse_frontend/implementation/__init__.py new file mode 100644 index 0000000..8b24da3 --- /dev/null +++ b/src/llparse_frontend/implementation/__init__.py @@ -0,0 +1,11 @@ +from .code import CodeImplementation +from .full import Implementation +from .node import NodeImplementation +from .transform import TransformImplementation + +__all__ = ( + "CodeImplementation", + "Implementation", + "NodeImplementation", + "TransformImplementation", +) diff --git a/src/llparse_frontend/implementation/code.py b/src/llparse_frontend/implementation/code.py new file mode 100644 index 0000000..c4bb5f3 --- /dev/null +++ b/src/llparse_frontend/implementation/code.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +from .. import code +from ..wrap import Wrap +from .typedefs import WrapCall + + +@dataclass(slots=True, frozen=True) +class CodeImplementation: + And: WrapCall[code.And] = Wrap + IsEqual: WrapCall[code.IsEqual] = Wrap + Load: WrapCall[code.Load] = Wrap + Match: WrapCall[code.Match] = Wrap + MulAdd: WrapCall[code.MulAdd] = Wrap + Or: WrapCall[code.Or] = Wrap + Span: WrapCall[code.Span] = Wrap + Store: WrapCall[code.Store] = Wrap + Test: WrapCall[code.Test] = Wrap + Update: WrapCall[code.Update] = Wrap + Value: WrapCall[code.Value] = Wrap diff --git a/src/llparse_frontend/implementation/full.py b/src/llparse_frontend/implementation/full.py new file mode 100644 index 0000000..c1f3199 --- /dev/null +++ b/src/llparse_frontend/implementation/full.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass, field + +from .code import CodeImplementation +from .node import NodeImplementation +from .transform import TransformImplementation + + +@dataclass(slots=True, frozen=True) +class Implementation: + code: CodeImplementation = field(default_factory=CodeImplementation) + node: NodeImplementation = field(default_factory=NodeImplementation) + transform: TransformImplementation = field(default_factory=TransformImplementation) diff --git a/src/llparse_frontend/implementation/node.py b/src/llparse_frontend/implementation/node.py new file mode 100644 index 0000000..7fe6e90 --- /dev/null +++ b/src/llparse_frontend/implementation/node.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +from .. import node +from ..wrap import Wrap +from .typedefs import WrapCall + + +@dataclass(slots=True, frozen=True) +class NodeImplementation: + Consume: WrapCall[node.Consume] = Wrap + Empty: WrapCall[node.Empty] = Wrap + Error: WrapCall[node.Error] = Wrap + Invoke: WrapCall[node.Invoke] = Wrap + Pause: WrapCall[node.Pause] = Wrap + Sequence: WrapCall[node.Sequence] = Wrap + Single: WrapCall[node.Single] = Wrap + SpanEnd: WrapCall[node.SpanEnd] = Wrap + SpanStart: WrapCall[node.SpanStart] = Wrap + TableLookup: WrapCall[node.TableLookup] = Wrap + Unpack: WrapCall[node.Unpack] = Wrap diff --git a/src/llparse_frontend/implementation/transform.py b/src/llparse_frontend/implementation/transform.py new file mode 100644 index 0000000..5809cba --- /dev/null +++ b/src/llparse_frontend/implementation/transform.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + +from .. import transform +from ..wrap import Wrap +from .typedefs import WrapCall + + +@dataclass(frozen=True, slots=True) +class TransformImplementation: + ID: WrapCall[transform.ID] = Wrap + ToLower: WrapCall[transform.ToLower] = Wrap + ToLowerUnsafe: WrapCall[transform.ToLowerUnsafe] = Wrap diff --git a/src/llparse_frontend/implementation/typedefs.py b/src/llparse_frontend/implementation/typedefs.py new file mode 100644 index 0000000..7031446 --- /dev/null +++ b/src/llparse_frontend/implementation/typedefs.py @@ -0,0 +1,7 @@ +from collections.abc import Callable +from typing import TypeVar + +from ..wrap import Wrap + +T = TypeVar("T") +WrapCall = Callable[[T], Wrap[T]] diff --git a/src/llparse_frontend/namespace/frontend.py b/src/llparse_frontend/namespace/frontend.py new file mode 100644 index 0000000..9d6599b --- /dev/null +++ b/src/llparse_frontend/namespace/frontend.py @@ -0,0 +1,3 @@ +from .. import code, node, transform + +__all__ = ("code", "node", "transform") diff --git a/src/llparse_frontend/node/__init__.py b/src/llparse_frontend/node/__init__.py new file mode 100644 index 0000000..b146875 --- /dev/null +++ b/src/llparse_frontend/node/__init__.py @@ -0,0 +1,36 @@ +from .base import Node, ReadonlyOtherwiseEdge +from .consume import Consume +from .empty import Empty +from .error import Error +from .invoke import Invoke, ReadonlyInvokeEdge +from .match import Match +from .pause import Pause +from .sequence import ReadonlySequenceEdge, Sequence +from .single import ReadonlySingleEdge, Single +from .slot import Slot +from .span_end import SpanEnd +from .span_start import SpanStart +from .table_lookup import ReadonlyTableEdge, TableLookup +from .unpack import Unpack + +__all__ = ( + "Consume", + "Empty", + "Error", + "Invoke", + "Match", + "Node", + "Pause", + "ReadonlyInvokeEdge", + "ReadonlyOtherwiseEdge", + "ReadonlySequenceEdge", + "ReadonlySingleEdge", + "ReadonlyTableEdge", + "Sequence", + "Single", + "Slot", + "SpanEnd", + "SpanStart", + "TableLookup", + "Unpack", +) diff --git a/src/llparse_frontend/node/base.py b/src/llparse_frontend/node/base.py new file mode 100644 index 0000000..95d6356 --- /dev/null +++ b/src/llparse_frontend/node/base.py @@ -0,0 +1,66 @@ +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from functools import partial + +from typing_extensions import Self + +from ..utils import UniqueName +from ..wrap import Wrap +from .slot import Slot + + +@dataclass(slots=True, frozen=True) +class ReadonlyOtherwiseEdge: + node: "Wrap[Node]" + no_advance: bool + value: int | None + + +@dataclass(slots=True, frozen=True) +class OtherwiseEdge: + node: "Wrap[Node]" + no_advance: bool + value: int | None + + +def update_otherwise(otherwise: OtherwiseEdge, value: "Wrap[Node]") -> None: + otherwise.node = value + + +class Node: + __slots__ = ("id", "priv_otherwise", "priv_slots") + priv_otherwise: OtherwiseEdge | None + priv_slots: Sequence["Slot[Node]"] | None + + def __init__(self, id: UniqueName): + self.id = id + self.priv_otherwise = None + self.priv_slots = None + + def set_otherwise( + self, node: "Wrap[Self]", no_advance: bool, value: int | None = None + ): + self.priv_otherwise = OtherwiseEdge(node, no_advance, value) + + @property + def otherwise(self) -> ReadonlyOtherwiseEdge | None: + if self.priv_otherwise: + return ReadonlyOtherwiseEdge( + node=self.priv_otherwise.node, + no_advance=self.priv_otherwise.no_advance, + value=self.priv_otherwise.value, + ) + + def get_slots(self) -> Iterator["Slot[Node]"]: + if self.priv_slots is None: + self.priv_slots = list(self.build_slots()) + yield from self.priv_slots + + def build_slots(self): + otherwise = self.priv_otherwise + if otherwise is not None: + yield Slot(otherwise.node, partial(update_otherwise, otherwise)) + + # incase needed... + def __hash__(self) -> int: + return hash(self.id) diff --git a/src/llparse_frontend/node/consume.py b/src/llparse_frontend/node/consume.py new file mode 100644 index 0000000..3b66ec4 --- /dev/null +++ b/src/llparse_frontend/node/consume.py @@ -0,0 +1,10 @@ +from ..utils import UniqueName +from .base import Node + + +class Consume(Node): + __slots__ = ("field",) + + def __init__(self, id: UniqueName, field: str): + super().__init__(id) + self.field = field diff --git a/src/llparse_frontend/node/empty.py b/src/llparse_frontend/node/empty.py new file mode 100644 index 0000000..ad55fd3 --- /dev/null +++ b/src/llparse_frontend/node/empty.py @@ -0,0 +1,5 @@ +from .base import Node + + +class Empty(Node): + pass diff --git a/src/llparse_frontend/node/error.py b/src/llparse_frontend/node/error.py new file mode 100644 index 0000000..39d2962 --- /dev/null +++ b/src/llparse_frontend/node/error.py @@ -0,0 +1,11 @@ +from ..utils import UniqueName +from .base import Node + + +class Error(Node): + __slots__ = ("code", "reason") + + def __init__(self, id: UniqueName, code: int, reason: str) -> None: + super().__init__(id) + self.code = code + self.reason = reason diff --git a/src/llparse_frontend/node/invoke.py b/src/llparse_frontend/node/invoke.py new file mode 100644 index 0000000..02fa501 --- /dev/null +++ b/src/llparse_frontend/node/invoke.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass +from functools import partial + +from ..code import Code +from ..utils import UniqueName +from ..wrap import Wrap +from .base import Node +from .slot import Slot + + +@dataclass(slots=True) +class InvokeEdge: + code: int + node: Wrap[Node] + + +@dataclass(slots=True, frozen=True) +class ReadonlyInvokeEdge: + code: int + node: Wrap[Node] + + +def update_edge(edge: InvokeEdge, value: Wrap[Node]) -> None: + edge.node = value + + +class Invoke(Node): + __slots__ = ("code", "priv_edges") + + def __init__(self, id: UniqueName, code: Wrap[Code]): + super().__init__(id) + self.code = code + self.priv_edges: list[InvokeEdge] = [] + + def add_edge(self, code: int, node: Wrap[Node]): + self.priv_edges.append(InvokeEdge(code, node)) + + @property + def edges(self) -> list[ReadonlyInvokeEdge]: + return [ReadonlyInvokeEdge(e.code, e.node) for e in self.priv_edges] + + def build_slots(self): + for edge in self.priv_edges: + yield Slot(edge.node, partial(update_edge, edge)) + return super().build_slots() diff --git a/src/llparse_frontend/node/match.py b/src/llparse_frontend/node/match.py new file mode 100644 index 0000000..3e3d2bd --- /dev/null +++ b/src/llparse_frontend/node/match.py @@ -0,0 +1,14 @@ +from ..transform import Transform +from ..wrap import Wrap +from .base import Node + + +class Match(Node): + __slots__ = ("transform",) + + def __init__(self, id): + super().__init__(id) + self.transform: Transform | None = None + + def set_transform(self, transform: Wrap[Transform]): + self.transform = transform diff --git a/src/llparse_frontend/node/pause.py b/src/llparse_frontend/node/pause.py new file mode 100644 index 0000000..c72b17d --- /dev/null +++ b/src/llparse_frontend/node/pause.py @@ -0,0 +1,5 @@ +from .error import Error as ErrorNode + + +class Pause(ErrorNode): + pass diff --git a/src/llparse_frontend/node/sequence.py b/src/llparse_frontend/node/sequence.py new file mode 100644 index 0000000..399774e --- /dev/null +++ b/src/llparse_frontend/node/sequence.py @@ -0,0 +1,50 @@ +from collections.abc import Iterator +from dataclasses import dataclass +from functools import partial + +from ..utils import UniqueName +from ..wrap import Wrap +from .base import Node +from .match import Match +from .slot import Slot + + +@dataclass(slots=True) +class SequenceEdge: + node: Wrap[Node] + value: int | None + + +@dataclass(slots=True, frozen=True) +class ReadonlySequenceEdge: + node: Wrap[Node] + value: int | None + + +def update_edge(edge: SequenceEdge, value: Wrap[Node]) -> None: + edge.node = value + + +class Sequence(Match): + __slots__ = ("priv_edge", "select") + + priv_edge: SequenceEdge | None + + def __init__(self, id: UniqueName, select: bytes): + super().__init__(id) + self.select = select + self.priv_edge = None + + def set_edge(self, node: Wrap[Node], value: int | None): + assert self.priv_edge is None + self.priv_edge = SequenceEdge(node, value) + + @property + def edge(self) -> ReadonlySequenceEdge: + return ReadonlySequenceEdge(self.priv_edge.node, self.priv_edge.value) + + def build_slots(self) -> Iterator[Slot[Node]]: + edge = self.priv_edge + if edge is not None: + yield Slot(edge.node, partial(update_edge, edge)) + yield from super().build_slots() diff --git a/src/llparse_frontend/node/single.py b/src/llparse_frontend/node/single.py new file mode 100644 index 0000000..fe5017e --- /dev/null +++ b/src/llparse_frontend/node/single.py @@ -0,0 +1,66 @@ +from collections.abc import Iterator, MutableSequence +from dataclasses import dataclass +from functools import partial + +from ..utils import UniqueName +from ..wrap import Wrap +from .base import Node +from .match import Match +from .slot import Slot + + +@dataclass(slots=True) +class SingleEdge: + key: int + node: Wrap[Node] + no_advance: bool + value: int | None + + +@dataclass(slots=True, frozen=True) +class ReadonlySingleEdge: + key: int + node: Wrap[Node] + no_advance: bool + value: int | None + + +def update_edge(edge: SingleEdge, value: Wrap[Node]) -> None: + edge.node = value + + +class Single(Match): + __slots__ = ("priv_edges", "select") + + priv_edges: MutableSequence[SingleEdge] + + def __init__(self, id: UniqueName): + super().__init__(id) + self.priv_edges = [] + + def add_edge(self, edge: ReadonlySingleEdge) -> None: + self.priv_edges.append( + SingleEdge( + key=edge.key, + no_advance=edge.no_advance, + node=edge.node, + value=edge.value, + ) + ) + + @property + def edges(self) -> list[ReadonlySingleEdge]: + return [ + ReadonlySingleEdge( + key=edge.key, + no_advance=edge.no_advance, + node=edge.node, + value=edge.value, + ) + for edge in self.priv_edges + ] + + def build_slots(self) -> Iterator[Slot[Node]]: + for edge in self.priv_edges: + yield Slot(edge.node, partial(update_edge, edge)) + yield from super().build_slots() diff --git a/src/llparse_frontend/node/slot.py b/src/llparse_frontend/node/slot.py new file mode 100644 index 0000000..d8c2750 --- /dev/null +++ b/src/llparse_frontend/node/slot.py @@ -0,0 +1,21 @@ +from collections.abc import Callable +from typing import Generic, TypeVar + +from ..wrap import Wrap + +Node = TypeVar("Node") + + +class Slot(Generic[Node]): + def __init__(self, node: Wrap[Node], update: Callable[[Wrap[Node]], None]): + self.priv_node = node + self.priv_update = update + + @property + def node(self) -> Wrap[Node]: + return self.priv_node + + @node.setter + def node(self, value: Wrap[Node]): + self.priv_node = value + self.priv_update(value) diff --git a/src/llparse_frontend/node/span_end.py b/src/llparse_frontend/node/span_end.py new file mode 100644 index 0000000..2c52e0d --- /dev/null +++ b/src/llparse_frontend/node/span_end.py @@ -0,0 +1,14 @@ +from ..code import Span +from ..span_field import SpanField +from ..utils import UniqueName +from ..wrap import Wrap +from .base import Node + + +class SpanEnd(Node): + __slots__ = ("callback", "field") + + def __init__(self, id: UniqueName, field: SpanField, callback: Wrap[Span]): + super().__init__(id) + self.field = field + self.callback = callback diff --git a/src/llparse_frontend/node/span_start.py b/src/llparse_frontend/node/span_start.py new file mode 100644 index 0000000..7b10bb2 --- /dev/null +++ b/src/llparse_frontend/node/span_start.py @@ -0,0 +1,14 @@ +from ..code import Span +from ..span_field import SpanField +from ..utils import UniqueName +from ..wrap import Wrap +from .base import Node + + +class SpanStart(Node): + __slots__ = ("callback", "field") + + def __init__(self, id: UniqueName, field: SpanField, callback: Wrap[Span]): + super().__init__(id) + self.field = field + self.callback = callback diff --git a/src/llparse_frontend/node/table_lookup.py b/src/llparse_frontend/node/table_lookup.py new file mode 100644 index 0000000..5f2a85a --- /dev/null +++ b/src/llparse_frontend/node/table_lookup.py @@ -0,0 +1,62 @@ +from collections.abc import Iterator, MutableSequence, Sequence +from dataclasses import dataclass +from functools import partial + +from ..utils import UniqueName +from ..wrap import Wrap +from .base import Node +from .match import Match +from .slot import Slot + + +@dataclass(slots=True) +class TableEdge: + keys: Sequence[int] + node: Wrap[Node] + no_advance: bool + + +@dataclass(slots=True, frozen=True) +class ReadonlyTableEdge: + keys: Sequence[int] + node: Wrap[Node] + no_advance: bool + + +def update_edge(edge: TableEdge, value: Wrap[Node]) -> None: + edge.node = value + + +class TableLookup(Match): + __slots__ = ("priv_edges",) + + priv_edges: MutableSequence[TableEdge] + + def __init__(self, id: UniqueName): + super().__init__(id) + self.priv_edges = [] + + def add_edge(self, edge: ReadonlyTableEdge) -> None: + self.priv_edges.append( + TableEdge( + keys=edge.keys, + no_advance=edge.no_advance, + node=edge.node, + ) + ) + + @property + def edges(self) -> list[ReadonlyTableEdge]: + return [ + ReadonlyTableEdge( + keys=edge.keys, + no_advance=edge.no_advance, + node=edge.node, + ) + for edge in self.priv_edges + ] + + def build_slots(self) -> Iterator[Slot[Node]]: + for edge in self.priv_edges: + yield Slot(edge.node, partial(update_edge, edge)) + yield from super().build_slots() diff --git a/src/llparse_frontend/node/unpack.py b/src/llparse_frontend/node/unpack.py new file mode 100644 index 0000000..0e0d158 --- /dev/null +++ b/src/llparse_frontend/node/unpack.py @@ -0,0 +1,29 @@ +from ..utils import UniqueName +from .base import Node + + +class Unpack(Node): + __slots__ = ( + "bits", + "byte_offset", + "field", + "little_endian", + "signed", + ) + + def __init__( + self, + id: UniqueName, + field: str, + bits: int, + signed: bool, + little_endian: bool, + byte_offset: int, + ): + super().__init__(id) + + self.field = field + self.bits = bits + self.signed = signed + self.little_endian = little_endian + self.byte_offset = byte_offset diff --git a/src/llparse_frontend/peephole.py b/src/llparse_frontend/peephole.py new file mode 100644 index 0000000..09a425f --- /dev/null +++ b/src/llparse_frontend/peephole.py @@ -0,0 +1,43 @@ +from .node import Empty, Node +from .wrap import Wrap + + +# There's no point to having a Peephole class it's been optimized +# into a single function +def optimize(root: Wrap[Node], nodes: list[Wrap[Node]]) -> Wrap[Node]: + changed = set(nodes) + + while changed: + previous = changed.copy() + changed.clear() + + for node in previous: + # Combined 2 functions from typescript llparse to + # just needing 1, this refactoring change allows for more speed. and less costly calls.. + altered = False + + for slot in node.ref.get_slots(): + if not isinstance(slot.node.ref, Empty) or not slot.node.ref.otherwise: + continue + + otherwise = slot.node.ref.otherwise + + # Node skips so we cannot optimize + if not otherwise.no_advance: + continue + + slot.node.ref = otherwise.node.ref + altered = True + + if altered: + changed.add(node) + + while isinstance(root.ref, Empty): + if not root.ref.otherwise or not root.ref.otherwise.no_advance: + break + root = root.ref.otherwise.node + + return root + + +__all__ = ("optimize",) diff --git a/src/llparse_frontend/span_field.py b/src/llparse_frontend/span_field.py new file mode 100644 index 0000000..57dd970 --- /dev/null +++ b/src/llparse_frontend/span_field.py @@ -0,0 +1,11 @@ +from collections.abc import Sequence +from dataclasses import dataclass + +from .code import Span +from .wrap import Wrap + + +@dataclass(slots=True) +class SpanField: + index: int + callbacks: Sequence[Wrap[Span]] diff --git a/src/llparse_frontend/transform/__init__.py b/src/llparse_frontend/transform/__init__.py new file mode 100644 index 0000000..39642fb --- /dev/null +++ b/src/llparse_frontend/transform/__init__.py @@ -0,0 +1,6 @@ +from .base import Transform +from .id import ID +from .to_lower import ToLower +from .to_lower_unsafe import ToLowerUnsafe + +__all__ = ("ID", "ToLower", "ToLowerUnsafe", "Transform") diff --git a/src/llparse_frontend/transform/base.py b/src/llparse_frontend/transform/base.py new file mode 100644 index 0000000..186f4bc --- /dev/null +++ b/src/llparse_frontend/transform/base.py @@ -0,0 +1,5 @@ +class Transform: + __slots__ = ("name",) + + def __init__(self, name: str): + self.name = name diff --git a/src/llparse_frontend/transform/id.py b/src/llparse_frontend/transform/id.py new file mode 100644 index 0000000..ac52452 --- /dev/null +++ b/src/llparse_frontend/transform/id.py @@ -0,0 +1,6 @@ +from .base import Transform + + +class ID(Transform): + def __init__(self): + super().__init__("id") diff --git a/src/llparse_frontend/transform/to_lower.py b/src/llparse_frontend/transform/to_lower.py new file mode 100644 index 0000000..c108d30 --- /dev/null +++ b/src/llparse_frontend/transform/to_lower.py @@ -0,0 +1,6 @@ +from .base import Transform + + +class ToLower(Transform): + def __init__(self) -> None: + super().__init__("to_lower") diff --git a/src/llparse_frontend/transform/to_lower_unsafe.py b/src/llparse_frontend/transform/to_lower_unsafe.py new file mode 100644 index 0000000..bad0bb0 --- /dev/null +++ b/src/llparse_frontend/transform/to_lower_unsafe.py @@ -0,0 +1,6 @@ +from .base import Transform + + +class ToLowerUnsafe(Transform): + def __init__(self) -> None: + super().__init__("to_lower_unsafe") diff --git a/src/llparse_frontend/trie/__init__.py b/src/llparse_frontend/trie/__init__.py new file mode 100644 index 0000000..300b6fd --- /dev/null +++ b/src/llparse_frontend/trie/__init__.py @@ -0,0 +1,143 @@ +from collections.abc import Sequence +from dataclasses import dataclass +from functools import total_ordering + +from llparse_builder import node as api +from llparse_builder.edge import Edge + +from .empty import TrieEmpty +from .node import TrieNode +from .sequence import TrieSequence +from .single import TrieSingle, TrieSingleChild + +__all__ = ("IEdge", "Trie", "TrieEmpty", "TrieNode", "TrieSequence", "TrieSingle") + + +@total_ordering +@dataclass(slots=True, frozen=True) +class IEdge: + key: bytes + node: api.Node + no_advance: bool + value: int | None = None + + def __lt__(self, object: "IEdge"): + return self.key < object.key + + +class Trie: + __slots__ = ("name",) + + def __init__(self, name: str): + self.name = name + + def build(self, edges: Sequence[Edge]): + if not edges: + return + + internal_edges: list[IEdge] = [] + + for edge in edges: + key = chr(edge.key) if isinstance(edge.key, int) else edge.key + internal_edges.append( + IEdge( + key=key.encode("utf-8") if isinstance(key, str) else key, + no_advance=edge.no_advance, + node=edge.node, + value=edge.value, + ) + ) + + return self.level(internal_edges, []) + + def empty(self, edge: Edge): + return TrieEmpty(edge.node, edge.value) + + def level(self, edges: Sequence[Edge], path: Sequence[bytes]): + first = edges[0].key + last = edges[-1].key + + # Leaf + if len(edges) == 1 and not len(edges[0].key): + return self.empty(edges[0]) + + # Find the longest common sub-string + common = 0 + for common in range(len(first)): + if first[common] != last[common]: + break + + # Sequence + if common > 1: + # We need to slice the common part of the sequence + # where anything is matching + # example: 'abc123', 'abc567' -> 'abc' + return self.sequence(edges, first[: common + 1], path) + + # single + return self.single(edges, path) + + def slice(self, edges: Sequence[IEdge], off: int) -> Sequence[IEdge]: + return sorted( + [ + IEdge( + key=edge.key[off:], + no_advance=edge.no_advance, + node=edge.node, + value=edge.value, + ) + for edge in edges + ], + key=lambda k: k.key, + ) + + def sequence( + self, edges: Sequence[IEdge], prefix: bytes, path: list[bytes] + ) -> TrieSequence: + sliced = self.slice(edges, len(prefix)) + assert not any(edge.no_advance for edge in edges) + child = self.level(sliced, path + [prefix]) + return TrieSequence(prefix, child) + + def single(self, edges: Sequence[IEdge], path: list[bytes]): + if not len(edges[0].key): + assert path, f'Empty root entry at "{self.name}"' + assert len(edges) == 1 or edges[1].key, ( + f'Duplicate entries in "{self.name}" at [ {b"".join(path)} ]' + ) + + keys: dict[int, list[IEdge]] = {} + otherwise = None + for edge in edges: + if not edge.key: + otherwise = TrieEmpty(edge.node, edge.value) + continue + + key = edge.key[0] + + if key in keys: + keys[key].append(edge) + else: + keys[key] = [edge] + + children: list[TrieSingleChild] = [] + for key, sub_edges in keys.items(): + sliced = self.slice(sub_edges, 1) + + sub_path = path + [chr(key).encode("utf-8")] + + no_advance = any(e.no_advance for e in sub_edges) + all_same = all(e.no_advance == no_advance for e in sub_edges) + + if not (all_same or len(sub_edges) == 0): + err = ( + f'Conflicting `.peek` and `.match` entries in "{self.name}" at: [' + + (b", ".join(sub_path).decode("utf-8")) + + "]" + ) + raise TypeError(err) + children.append( + TrieSingleChild(key, no_advance, self.level(sliced, sub_path)) + ) + + return TrieSingle(children, otherwise) diff --git a/src/llparse_frontend/trie/empty.py b/src/llparse_frontend/trie/empty.py new file mode 100644 index 0000000..ab01af8 --- /dev/null +++ b/src/llparse_frontend/trie/empty.py @@ -0,0 +1,13 @@ +from typing import Any + +from llparse_builder import node as api + +from .node import TrieNode + + +class TrieEmpty(TrieNode): + __slots__ = ("node", "value") + + def __init__(self, node: api.Node, value: int | Any): + self.node = node + self.value = value diff --git a/src/llparse_frontend/trie/node.py b/src/llparse_frontend/trie/node.py new file mode 100644 index 0000000..b300145 --- /dev/null +++ b/src/llparse_frontend/trie/node.py @@ -0,0 +1,5 @@ +from abc import ABC + + +class TrieNode(ABC): + __slots__ = () diff --git a/src/llparse_frontend/trie/sequence.py b/src/llparse_frontend/trie/sequence.py new file mode 100644 index 0000000..75e2176 --- /dev/null +++ b/src/llparse_frontend/trie/sequence.py @@ -0,0 +1,9 @@ +from .node import TrieNode + + +class TrieSequence(TrieNode): + __slots__ = ("child", "select") + + def __init__(self, select: bytes, child: TrieNode) -> None: + self.select = select + self.child = child diff --git a/src/llparse_frontend/trie/single.py b/src/llparse_frontend/trie/single.py new file mode 100644 index 0000000..b3b994d --- /dev/null +++ b/src/llparse_frontend/trie/single.py @@ -0,0 +1,22 @@ +from collections.abc import Sequence +from dataclasses import dataclass + +from .empty import TrieEmpty +from .node import TrieNode + + +@dataclass(slots=True) +class TrieSingleChild: + key: int + no_advance: bool + node: TrieNode + + +class TrieSingle(TrieNode): + __slots__ = ("children", "otherwise") + + def __init__( + self, children: Sequence[TrieSingleChild], otherwise: TrieEmpty | None + ) -> None: + self.children = children + self.otherwise = otherwise diff --git a/src/llparse_frontend/utils/__init__.py b/src/llparse_frontend/utils/__init__.py new file mode 100644 index 0000000..227d33e --- /dev/null +++ b/src/llparse_frontend/utils/__init__.py @@ -0,0 +1,14 @@ +from ..errors import Error +from .identifier import Identifier, UniqueName + + +def to_cache_key(value: int | bool) -> str: + if isinstance(value, bool): + return "true" if value else "false" + elif isinstance(value, int): + return f"m{-value}" if value < 0 else f"{value}" + else: + raise Error(f'Unsupported value: "{value}"') + + +__all__ = ("Identifier", "UniqueName") diff --git a/src/llparse_frontend/utils/identifier.py b/src/llparse_frontend/utils/identifier.py new file mode 100644 index 0000000..6cb3e63 --- /dev/null +++ b/src/llparse_frontend/utils/identifier.py @@ -0,0 +1,29 @@ +from dataclasses import dataclass, field + + +# NOTE: Got rid of the naming convention of IUniqueName +# for just UniqueName. It's not an interface in python +@dataclass(slots=True, unsafe_hash=True) +class UniqueName: + name: str + original_name: str + + +@dataclass(slots=True, frozen=True) +class Identifier: + prefix: str = "" + postfix: str = "" + ns: set[str] = field(init=False, default_factory=set) + + def id(self, name: str): + target = self.prefix + name + self.postfix + + if target in self.ns: + for i in range(1, len(self.ns) + 1): + new_target = f"{target}_{i}" + if new_target not in self.ns: + target = new_target + break + + self.ns.add(target) + return UniqueName(name=target, original_name=name) diff --git a/src/llparse_frontend/wrap.py b/src/llparse_frontend/wrap.py new file mode 100644 index 0000000..406d4a5 --- /dev/null +++ b/src/llparse_frontend/wrap.py @@ -0,0 +1,17 @@ +from typing import Generic, TypeVar + +T = TypeVar("T") + +# While it may not make as much sense to have this in python +# due to duck-typing, It was kept to preverse the parody incase +# of unknown edge cases. + + +class Wrap(Generic[T]): + __slots__ = ("ref",) + + def __init__(self, ref: T) -> None: + self.ref = ref + + def __hash__(self) -> int: + return hash(self.ref) diff --git a/tests/builder/conftest.py b/tests/builder/conftest.py new file mode 100644 index 0000000..972e29c --- /dev/null +++ b/tests/builder/conftest.py @@ -0,0 +1,8 @@ +import pytest + +from llparse_builder import Builder + + +@pytest.fixture +def b() -> Builder: + return Builder() diff --git a/tests/builder/test_binary_search.py b/tests/builder/test_binary_search.py new file mode 100644 index 0000000..2600a29 --- /dev/null +++ b/tests/builder/test_binary_search.py @@ -0,0 +1,52 @@ +import pytest + +from llparse_builder.binary_search import binary_search + +arr = [1, 2, 2, 2, 3, 5, 9] + + +# based off test/binary-search.test.ts +def cmp(a: float, b: float, idx, seq) -> float: + return a - b + + +def test_not_a_sequence(): + with pytest.raises(TypeError): + binary_search(None, 3, cmp) + + +def test_invalid_input(): + with pytest.raises(TypeError): + binary_search(arr, 3, None) + + +def test_returning_index_of_sorted_array(): + assert binary_search(arr, 3, cmp) + + +def test_index_of_item_plus_one_or_negate(): + assert binary_search(arr, 4, cmp) == -6 + + +def test_index_if_item_exists_multiple_times(): + assert binary_search(arr, 2, cmp) == 3 + + +def test_with_empty_arrays(): + assert binary_search([], 42, cmp) == -1 + + +def test_arrays_of_floats(): + assert binary_search([0.0, 0.1, 0.2, 0.3, 0.4], 0.25, cmp) == -4 + + +def test_index_and_array_comparator(): + indexes = [] + + def index_cmp(a: int, b: int, i: int | None, array: list[int]): + assert array == arr + indexes.append(i) + return cmp(a, b, None, None) + + binary_search(arr, 3, index_cmp) + assert indexes == [3, 5, 4] diff --git a/tests/builder/test_builder.py b/tests/builder/test_builder.py new file mode 100644 index 0000000..3fea00e --- /dev/null +++ b/tests/builder/test_builder.py @@ -0,0 +1,60 @@ +import pytest + +from llparse_builder import Builder + + +def test_primitive_graph(b: Builder): + start = b.node("start") + end = b.node("end") + + start.peek("e", end).match("a", start).otherwise(b.error(1, "error")) + end.skip_to(start) + edges = start.priv_edges + assert len(edges) == 2 + assert not edges[0].no_advance + assert edges[0].node == start + assert edges[1].no_advance + assert edges[1].node == end + + +def test_disallowing_duplicate_edges(b: Builder): + start = b.node("start") + start.peek("e", start) + + with pytest.raises(AssertionError): + start.peek("e", start) + + +def test_disallowing_select_to_match_invoke(b: Builder): + start = b.node("start") + invoke = b.invoke(b.code.match("something")) + + with pytest.raises(AssertionError): + start.select("a", 1, invoke) + + +def test_disallow_peek_to_value_invoke(b: Builder): + start = b.node("start") + invoke = b.invoke(b.code.value("something")) + with pytest.raises(AssertionError): + start.peek("a", invoke) + + +def test_allow_select_to_value_invoke(b: Builder): + start = b.node("start") + invoke = b.invoke(b.code.value("something")) + start.select("a", 1, invoke) + + +def test_allow_create_edges_for_invoke(b: Builder): + start = b.node("start") + invoke = b.invoke( + b.code.value("something"), + { + -1: start, + 1: start, + 10: start, + }, + ) + edges = {e.key for e in invoke.priv_edges} + assert edges == {-1, 1, 10} diff --git a/tests/test_loop_checker.py b/tests/builder/test_loop_checker.py similarity index 66% rename from tests/test_loop_checker.py rename to tests/builder/test_loop_checker.py index ae0299a..1aea96b 100644 --- a/tests/test_loop_checker.py +++ b/tests/builder/test_loop_checker.py @@ -1,26 +1,26 @@ -from llparse.pybuilder import LoopChecker, Builder -from llparse.errors import Error import pytest +from llparse_builder import Builder, LoopChecker +from llparse_builder.errors import Error + # based off llparse-builder/test/loop-checker.test.ts -# Time Taken: 7 hours if you count the hard amounts of debugging, I went through. -@pytest.fixture() -def loop_checker() -> tuple[LoopChecker, Builder]: - return LoopChecker(), Builder() +@pytest.fixture +def lc() -> LoopChecker: + return LoopChecker() + +def test_detect_shallow_loops(lc: LoopChecker, b: Builder) -> None: -def test_detect_shallow_loops(loop_checker: tuple[LoopChecker, Builder]) -> None: - lc, b = loop_checker start = b.node("start") start.otherwise(start) - with pytest.raises(Error, match=r'Detected loop in "start" through "start"'): + with pytest.raises(Error, match=r'Detected loop in "start".*"start"'): lc.check(start) -def test_detect_loops(loop_checker: tuple[LoopChecker, Builder]) -> None: - lc, b = loop_checker +def test_detect_loops(lc: LoopChecker, b: Builder) -> None: + start = b.node("start") a = b.node("a") invoke = b.invoke( @@ -38,21 +38,21 @@ def test_detect_loops(loop_checker: tuple[LoopChecker, Builder]) -> None: lc.check(start) -def test_detect_shallow_loops_2(loop_checker: tuple[LoopChecker, Builder]) -> None: - lc, b = loop_checker +def test_detect_unreachable_keys(lc: LoopChecker, b: Builder) -> None: + start = b.node("start") loop = b.node("loop") start.peek("a", loop).otherwise(b.error(1, "error")) + loop.match("a", loop).otherwise(loop) - with pytest.raises(Error, match=r'Detected loop in "loop" through "loop"'): + + with pytest.raises(Error, match=r'Detected loop in "loop" through.*"loop"'): lc.check(loop) -def test_ignore_loops_through_peek_to_match( - loop_checker: tuple[LoopChecker, Builder], -) -> None: - lc, b = loop_checker +def test_ignore_loops_through_peek_to_match(lc: LoopChecker, b: Builder) -> None: + start = b.node("start") a = b.node("a") invoke = b.invoke( @@ -69,8 +69,8 @@ def test_ignore_loops_through_peek_to_match( lc.check(start) -def test_ignore_irrelevant_peeks(loop_checker: tuple[LoopChecker, Builder]) -> None: - lc, b = loop_checker +def test_ignore_irrelevant_peeks(lc: LoopChecker, b: Builder) -> None: + start = b.node("start") a = b.node("a") @@ -80,10 +80,8 @@ def test_ignore_irrelevant_peeks(loop_checker: tuple[LoopChecker, Builder]) -> N lc.check(start) -def test_ignore_loops_with_multi_peek_match( - loop_checker: tuple[LoopChecker, Builder], -) -> None: - lc, b = loop_checker +def test_ignore_loops_with_multi_peek_match(lc: LoopChecker, b: Builder) -> None: + start = b.node("start") another = b.node("another") @@ -154,7 +152,7 @@ def test_ignore_loops_with_multi_peek_match( "Y", "Z", ] - start.match(ALPHA, start).peek(NUM, another).skipTo(start) + start.match(ALPHA, start).peek(NUM, another).skip_to(start) another.match(NUM, another).otherwise(start) lc.check(start) diff --git a/tests/test_span_allocator.py b/tests/builder/test_span_allocator.py similarity index 89% rename from tests/test_span_allocator.py rename to tests/builder/test_span_allocator.py index 1b76a7a..a044c70 100644 --- a/tests/test_span_allocator.py +++ b/tests/builder/test_span_allocator.py @@ -1,9 +1,8 @@ -from llparse.spanalloc import SpanAllocator -from llparse.pybuilder import Builder -from llparse.errors import Error - import pytest +from llparse_builder.builder import Builder, SpanAllocator +from llparse_builder.errors import Error + # Brought over and translated from llparse-builder/tests/span-allocator.ts @@ -20,7 +19,7 @@ def test_allocate_single_span(span_alloc: tuple[SpanAllocator, Builder]) -> None start.otherwise(span.start(body)) - body.skipTo(span.end(start)) + body.skip_to(span.end(start)) res = sa.allocate(start) @@ -46,7 +45,7 @@ def test_allocate_overlapping_spans(span_alloc: tuple[SpanAllocator, Builder]) - body1.otherwise(span2.start(body2)) - body2.skipTo(span2.end(span1.end(start))) + body2.skip_to(span2.end(span1.end(start))) res = sa.allocate(start) @@ -76,9 +75,9 @@ def test_allocate_non_overlapping_spans( body2 = b.node("body2") start.match("a", span1.start(body1)).otherwise(span2.start(body2)) - body1.skipTo(span1.end(start)) + body1.skip_to(span1.end(start)) - body2.skipTo(span2.end(start)) + body2.skip_to(span2.end(start)) res = sa.allocate(start) @@ -118,9 +117,9 @@ def test_propagate_through_invoke_map(span_alloc: tuple[SpanAllocator, Builder]) b.invoke( b.code.load("custom"), { - 0: span.end().skipTo(start), + 0: span.end().skip_to(start), }, - span.end().skipTo(start), + span.end().skip_to(start), ) ) diff --git a/tests/builder/test_unpack.py b/tests/builder/test_unpack.py new file mode 100644 index 0000000..ee8ac67 --- /dev/null +++ b/tests/builder/test_unpack.py @@ -0,0 +1,150 @@ +from typing import NamedTuple, cast + +import pytest + +from llparse_builder import Builder +from llparse_builder.errors import Error + +# NOTE: These sections were not part of the original typescript llparse. +# It was added by the python author due to a desire +# for those trying to implement protocol-like parsers +# realted to http2/3 and socks4/5 protocols. Although +# the pakets are meant to be read at a single time. +# this at the very least prevents needing very large buffers +# when parsing these small packets. It also prevents +# packet loss with sockets. + + +class Format(NamedTuple): + value: str + bit_len: int + signed: bool + + def __str__(self): + return self.value + + +@pytest.fixture( + params=( + Format("b", 1, True), + Format("c", 1, True), + Format("?", 1, True), + Format("h", 2, True), + Format("i", 4, True), + Format("l", 4, True), + Format("q", 8, True), + Format("B", 1, False), + Format("H", 2, False), + Format("I", 4, False), + Format("L", 4, False), + Format("Q", 8, False), + Format("y", 3, True), + Format("Y", 3, False), + ), + ids=str, +) +def fmt(request: pytest.FixtureRequest) -> Format: + return cast(Format, request.param) + + +@pytest.fixture +def le(fmt) -> Format: + return Format(f"<{fmt.value}", fmt.bit_len, fmt.signed) + + +# NOTE: '=' is not tested due to it's questionableness +@pytest.fixture(params=(">", "!", "@")) +def be(request: pytest.FixtureRequest, fmt) -> Format: + return Format(f"{request.param}{fmt.value}", fmt.bit_len, fmt.signed) + + +def test_unpack_classic_method_le(b: Builder): + unpack = b.intLE("prop", 3) + assert unpack.bits == 3 + assert unpack.signed == True + assert unpack.little_endian == True + assert unpack.name == "prop_int_24_le" + + +def test_unpack_class_method_be(b: Builder): + unpack = b.intBE("prop", 3) + assert unpack.bits == 3 + assert unpack.signed == True + assert unpack.little_endian == False + assert unpack.name == "prop_int_24_be" + + +def test_unpack_class_method_ule(b: Builder): + unpack = b.uintLE("prop", 3) + assert unpack.bits == 3 + assert unpack.signed == False + assert unpack.little_endian == True + assert unpack.name == "prop_uint_24_le" + + +def test_unpack_class_method_ube(b: Builder): + unpack = b.uintBE("prop", 3) + assert unpack.bits == 3 + assert unpack.signed == False + assert unpack.little_endian == False + assert unpack.name == "prop_uint_24_be" + + +def test_unpack_bounds(b: Builder): + with pytest.raises(ValueError, match="bits should be a positive integer"): + b.intLE("bwah", -666) + + with pytest.raises(ValueError, match=r"bits too large, .*"): + b.intLE("bwah", 666) + + +def test_unpack_should_always_advance(b: Builder): + prop = b.uintBE("prop", 3) + + with pytest.raises(Error, match=r"Otherwise .*"): + prop.otherwise() + + +def test_unpack_format_option_single(b: Builder, fmt: Format): + up = b.unpack("prop", fmt.value) + assert up.bits == fmt.bit_len + assert up.signed == fmt.signed + # Name not really required for testing but will test it as a + # safety measure. + if up.signed: + assert up.name == f"prop_int_{fmt.bit_len * 8}_be" + + else: + assert up.name == f"prop_uint_{fmt.bit_len * 8}_be" + + +def test_unpack_format_option_multiple(b: Builder, fmt: Format): + up = b.unpack("prop", fmt.value) + assert up.bits == fmt.bit_len + assert up.signed == fmt.signed + # Name not really required for testing but will test it as a + # safety measure. + if up.signed: + assert up.name == f"prop_int_{fmt.bit_len * 8}_be" + + else: + assert up.name == f"prop_uint_{fmt.bit_len * 8}_be" + + +def test_unpack_format_option_le(b: Builder, le: Format): + up = b.unpack("prop", le.value) + assert up.bits == le.bit_len + assert up.signed == le.signed + # Name not really required for testing but will test it as a + # safety measure. + if up.signed: + assert up.name == f"prop_int_{le.bit_len * 8}_le" + + else: + assert up.name == f"prop_uint_{le.bit_len * 8}_le" + + +def test_unpack_format_option_be(b: Builder, be: Format): + # it's the same thing as big endian only with extra context + # as to what is wanted. + test_unpack_format_option_multiple(b, be) diff --git a/tests/builder/test_unpack_creator.py b/tests/builder/test_unpack_creator.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/builder/test_unpack_creator.py @@ -0,0 +1 @@ + diff --git a/tests/frontend/__init__.py b/tests/frontend/__init__.py new file mode 100644 index 0000000..3479934 --- /dev/null +++ b/tests/frontend/__init__.py @@ -0,0 +1 @@ +# Needed don't remove... diff --git a/tests/frontend/conftest.py b/tests/frontend/conftest.py new file mode 100644 index 0000000..972e29c --- /dev/null +++ b/tests/frontend/conftest.py @@ -0,0 +1,8 @@ +import pytest + +from llparse_builder import Builder + + +@pytest.fixture +def b() -> Builder: + return Builder() diff --git a/tests/frontend/fixtures/__init__.py b/tests/frontend/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/frontend/fixtures/implementation/__init__.py b/tests/frontend/fixtures/implementation/__init__.py new file mode 100644 index 0000000..0f71949 --- /dev/null +++ b/tests/frontend/fixtures/implementation/__init__.py @@ -0,0 +1,5 @@ +from . import code as code +from . import node as node +from . import transform as transform + +__all__ = ("code", "node", "transform") diff --git a/tests/frontend/fixtures/implementation/code/__init__.py b/tests/frontend/fixtures/implementation/code/__init__.py new file mode 100644 index 0000000..f3343ef --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/__init__.py @@ -0,0 +1,36 @@ +# This code was autogenerated from tools/generate_impl.py +# It is used to handle code generation for new language implementations +# And gives a useful template for the list of nodes and code objects needed. +# Editing is fine but use tools/generate_impl.py at your own risk. + +from .and_ import And +from .base import Code +from .external import External +from .field import Field +from .is_equal import IsEqual +from .load import Load +from .match import Match +from .mul_add import MulAdd +from .or_ import Or +from .span import Span +from .store import Store +from .test import Test +from .update import Update +from .value import Value + +__all__ = ( + "And", + "Code", + "External", + "Field", + "IsEqual", + "Load", + "Match", + "MulAdd", + "Or", + "Span", + "Store", + "Test", + "Update", + "Value", +) diff --git a/tests/frontend/fixtures/implementation/code/and_.py b/tests/frontend/fixtures/implementation/code/and_.py new file mode 100644 index 0000000..6d143d4 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/and_.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .base import Code + + +class And(Code[frontend.And]): + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/code/base.py b/tests/frontend/fixtures/implementation/code/base.py new file mode 100644 index 0000000..8ba7611 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/base.py @@ -0,0 +1,14 @@ +from abc import ABC +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class Code(ABC, Generic[T]): + __slots__ = ("ref",) + + def __init__(self, ref: T) -> None: + self.ref = ref + + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/code/external.py b/tests/frontend/fixtures/implementation/code/external.py new file mode 100644 index 0000000..d0e34da --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/external.py @@ -0,0 +1,7 @@ +from llparse_frontend import code as frontend + +from .base import Code + + +class External(Code[frontend.External]): + pass diff --git a/tests/frontend/fixtures/implementation/code/field.py b/tests/frontend/fixtures/implementation/code/field.py new file mode 100644 index 0000000..0bdbbab --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/field.py @@ -0,0 +1,7 @@ +from .base import Code, TypeVar + +T = TypeVar("T", bound=Code) + + +class Field(Code[T]): + pass diff --git a/tests/frontend/fixtures/implementation/code/is_equal.py b/tests/frontend/fixtures/implementation/code/is_equal.py new file mode 100644 index 0000000..912c846 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/is_equal.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .field import Field + + +class IsEqual(Field[frontend.IsEqual]): + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/code/load.py b/tests/frontend/fixtures/implementation/code/load.py new file mode 100644 index 0000000..125d784 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/load.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .field import Field + + +class Load(Field[frontend.Load]): + def build(self): + return "" diff --git a/tests/frontend/fixtures/implementation/code/match.py b/tests/frontend/fixtures/implementation/code/match.py new file mode 100644 index 0000000..30033d8 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/match.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .base import Code + + +class Match(Code[frontend.Match]): + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/code/mul_add.py b/tests/frontend/fixtures/implementation/code/mul_add.py new file mode 100644 index 0000000..db0de77 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/mul_add.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .field import Field + + +class MulAdd(Field[frontend.MulAdd]): + def build(self): + return "" diff --git a/tests/frontend/fixtures/implementation/code/or_.py b/tests/frontend/fixtures/implementation/code/or_.py new file mode 100644 index 0000000..2c2356a --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/or_.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .field import Field + + +class Or(Field[frontend.Or]): + def build(self): + return super().build() diff --git a/tests/frontend/fixtures/implementation/code/span.py b/tests/frontend/fixtures/implementation/code/span.py new file mode 100644 index 0000000..6949701 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/span.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .base import Code + + +class Span(Code[frontend.Span]): + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/code/store.py b/tests/frontend/fixtures/implementation/code/store.py new file mode 100644 index 0000000..5b5810a --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/store.py @@ -0,0 +1,9 @@ +from llparse.ext import Writer +from llparse_frontend import code as frontend + +from .field import Field + + +class Store(Field[frontend.Store]): + def do_build(self, out: Writer) -> None: + super().do_build(out) diff --git a/tests/frontend/fixtures/implementation/code/test.py b/tests/frontend/fixtures/implementation/code/test.py new file mode 100644 index 0000000..5922383 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/test.py @@ -0,0 +1,9 @@ +from llparse.ext import Writer +from llparse_frontend import code as frontend + +from .field import Field + + +class Test(Field[frontend.Or]): + def do_build(self, out: Writer) -> None: + super().do_build(out) diff --git a/tests/frontend/fixtures/implementation/code/update.py b/tests/frontend/fixtures/implementation/code/update.py new file mode 100644 index 0000000..7a44c60 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/update.py @@ -0,0 +1,9 @@ +from llparse.ext import Writer +from llparse_frontend import code as frontend + +from .field import Field + + +class Update(Field[frontend.Update]): + def do_build(self, out: Writer) -> None: + super().do_build(out) diff --git a/tests/frontend/fixtures/implementation/code/value.py b/tests/frontend/fixtures/implementation/code/value.py new file mode 100644 index 0000000..8aa0ad8 --- /dev/null +++ b/tests/frontend/fixtures/implementation/code/value.py @@ -0,0 +1,8 @@ +from llparse_frontend import code as frontend + +from .base import Code + + +class Value(Code[frontend.Value]): + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/node/__init__.py b/tests/frontend/fixtures/implementation/node/__init__.py new file mode 100644 index 0000000..7aee9e8 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/__init__.py @@ -0,0 +1,32 @@ +# This code was autogenerated from tools/generate_impl.py +# It is used to handle code generation for new language implementations +# And gives a useful template for the list of nodes and code objects needed. +# Editing is fine but use tools/generate_impl.py at your own risk. + +from .base import Node +from .consume import Consume +from .empty import Empty +from .error import ErrorNode as Error +from .invoke import Invoke +from .pause import Pause +from .sequence import Sequence +from .single import Single +from .span_end import SpanEnd +from .span_start import SpanStart +from .table_lookup import TableLookup +from .unpack import Unpack + +__all__ = ( + "Consume", + "Empty", + "Error", + "Invoke", + "Node", + "Pause", + "Sequence", + "Single", + "SpanEnd", + "SpanStart", + "TableLookup", + "Unpack", +) diff --git a/tests/frontend/fixtures/implementation/node/base.py b/tests/frontend/fixtures/implementation/node/base.py new file mode 100644 index 0000000..c750558 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/base.py @@ -0,0 +1,42 @@ +from abc import ABC, abstractmethod +from typing import Generic, TypeVar, cast + +from llparse_frontend import node + +T = TypeVar("T", bound="node.Node") + + +class Node(ABC, Generic[T]): + __slots__ = ("built", "ref") + + def __init__(self, ref: T) -> None: + self.ref = ref + self.built = False + + @abstractmethod + def do_build(self, out: list[str]) -> None: ... + + def build(self, out: list[str]) -> None: + if self.built: + return + self.built = True + self.do_build(out) + if self.ref.otherwise is not None: + return cast(Node[T], self.ref.otherwise.node).build(out) + + def format(self, value: str) -> str: + otherwise = "" + if self.ref.otherwise is not None: + otherwise_ref = cast(Node[T], self.ref.otherwise.node).ref + otherwise = ( + " otherwise=" + + ("-no_adv" if self.ref.otherwise.no_advance else "") + + otherwise_ref.id.name + ) + if self.ref.otherwise.value is not None: + otherwise += f":{self.ref.otherwise.value}" + + return ( + f"<{self.__class__.__name__} name={self.ref.id.name} " + + f"{value}{otherwise}/>" + ) diff --git a/tests/frontend/fixtures/implementation/node/consume.py b/tests/frontend/fixtures/implementation/node/consume.py new file mode 100644 index 0000000..ebeaa53 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/consume.py @@ -0,0 +1,8 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class Consume(Node[frontend.Consume]): + def do_build(self, out: list[str]) -> None: + out.append(self.format(f"field={self.ref.field}")) diff --git a/tests/frontend/fixtures/implementation/node/empty.py b/tests/frontend/fixtures/implementation/node/empty.py new file mode 100644 index 0000000..02d3d0a --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/empty.py @@ -0,0 +1,9 @@ +from llparse.ext import Writer +from llparse_frontend import node as frontend + +from .base import Node + + +class Empty(Node[frontend.Empty]): + def do_build(self, out: Writer) -> None: + out.append(self.format("")) diff --git a/tests/frontend/fixtures/implementation/node/error.py b/tests/frontend/fixtures/implementation/node/error.py new file mode 100644 index 0000000..8cc2d37 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/error.py @@ -0,0 +1,8 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class ErrorNode(Node[frontend.Error]): + def do_build(self, out: list[str]) -> None: + out.append(self.format("")) diff --git a/tests/frontend/fixtures/implementation/node/invoke.py b/tests/frontend/fixtures/implementation/node/invoke.py new file mode 100644 index 0000000..9ac9367 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/invoke.py @@ -0,0 +1,8 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class Invoke(Node[frontend.Invoke]): + def do_build(self, out: list[str]) -> None: + out.append(self.format("")) diff --git a/tests/frontend/fixtures/implementation/node/pause.py b/tests/frontend/fixtures/implementation/node/pause.py new file mode 100644 index 0000000..9070431 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/pause.py @@ -0,0 +1,6 @@ +from .error import ErrorNode + + +class Pause(ErrorNode): + def do_build(self, out: list[str]) -> None: + out.append(self.format("")) diff --git a/tests/frontend/fixtures/implementation/node/sequence.py b/tests/frontend/fixtures/implementation/node/sequence.py new file mode 100644 index 0000000..61531e2 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/sequence.py @@ -0,0 +1,17 @@ +from binascii import hexlify +from typing import cast + +from llparse_frontend import node as frontend + +from .base import Node + + +class Sequence(Node[frontend.Sequence]): + def do_build(self, out: list[str]) -> None: + code = f'select="{hexlify(self.ref.select).decode("utf-8")}" ' + code += f'edge="{self.ref.edge.node.ref.id.name}"' + if self.ref.edge.value is not None: + code += f":{self.ref.edge.value}" + out.append(self.format(code)) + edge_node = cast(Node, self.ref.edge.node) + edge_node.build(out) diff --git a/tests/frontend/fixtures/implementation/node/single.py b/tests/frontend/fixtures/implementation/node/single.py new file mode 100644 index 0000000..19e216b --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/single.py @@ -0,0 +1,23 @@ +from typing import cast + +from llparse_frontend import node as frontend + +from .base import Node + + +class Single(Node[frontend.Single]): + def do_build(self, out: list[str]) -> None: + edges = [] + for edge in self.ref.edges: + code = ( + f"k{edge.key}{'-no_adv-' if edge.no_advance else ''}=" + + edge.node.ref.id.name + ) + if edge.value is not None: + code += f":{edge.value}" + edges.append(code) + out.append(self.format(" ".join(edges))) + + for edge in self.ref.edges: + edge_node = cast(Node, edge.node) + edge_node.build(out) diff --git a/tests/frontend/fixtures/implementation/node/span_end.py b/tests/frontend/fixtures/implementation/node/span_end.py new file mode 100644 index 0000000..4e81882 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/span_end.py @@ -0,0 +1,8 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class SpanEnd(Node[frontend.SpanEnd]): + def do_build(self, out: list[str]) -> None: + out.append(self.format("")) diff --git a/tests/frontend/fixtures/implementation/node/span_start.py b/tests/frontend/fixtures/implementation/node/span_start.py new file mode 100644 index 0000000..be80fcc --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/span_start.py @@ -0,0 +1,8 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class SpanStart(Node[frontend.SpanStart]): + def do_build(self, out: list[str]) -> None: + out.append(self.format("")) diff --git a/tests/frontend/fixtures/implementation/node/table_lookup.py b/tests/frontend/fixtures/implementation/node/table_lookup.py new file mode 100644 index 0000000..204a660 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/table_lookup.py @@ -0,0 +1,8 @@ +from llparse_frontend import node as frontend + +from .base import Node + + +class TableLookup(Node[frontend.TableLookup]): + def do_build(self, out: list[str]) -> None: + out.append(self.format("")) diff --git a/tests/frontend/fixtures/implementation/node/unpack.py b/tests/frontend/fixtures/implementation/node/unpack.py new file mode 100644 index 0000000..ae49f43 --- /dev/null +++ b/tests/frontend/fixtures/implementation/node/unpack.py @@ -0,0 +1,14 @@ +from llparse_frontend import node as frontend + +from .base import Node + +# Unused but defined incase needed in a later update for testing. + + +class Unpack(Node[frontend.Unpack]): + def do_build(self, out: list[str]) -> None: + out.append( + self.format( + f'bits="{self.ref.bits}" name="{self.ref.id.name}" signed={self.ref.signed}' + ) + ) diff --git a/tests/frontend/fixtures/implementation/transform/__init__.py b/tests/frontend/fixtures/implementation/transform/__init__.py new file mode 100644 index 0000000..288dc6f --- /dev/null +++ b/tests/frontend/fixtures/implementation/transform/__init__.py @@ -0,0 +1,16 @@ +# This code was autogenerated from tools/generate_impl.py +# It is used to handle code generation for new language implementations +# And gives a useful template for the list of nodes and code objects needed. +# Editing is fine but use tools/generate_impl.py at your own risk. + +from .base import Transform +from .id import ID +from .to_lower import ToLower +from .to_lower_unsafe import ToLowerUnsafe + +__all__ = ( + "ID", + "ToLower", + "ToLowerUnsafe", + "Transform", +) diff --git a/tests/frontend/fixtures/implementation/transform/base.py b/tests/frontend/fixtures/implementation/transform/base.py new file mode 100644 index 0000000..6d037be --- /dev/null +++ b/tests/frontend/fixtures/implementation/transform/base.py @@ -0,0 +1,14 @@ +from abc import ABC, abstractmethod +from typing import Generic, TypeVar + +T = TypeVar("T") + + +class Transform(ABC, Generic[T]): + __slots__ = ("ref",) + + def __init__(self, ref: T) -> None: + self.ref = ref + + @abstractmethod + def build(self) -> str: ... diff --git a/tests/frontend/fixtures/implementation/transform/id.py b/tests/frontend/fixtures/implementation/transform/id.py new file mode 100644 index 0000000..a431cc6 --- /dev/null +++ b/tests/frontend/fixtures/implementation/transform/id.py @@ -0,0 +1,8 @@ +from llparse_frontend import transform as frontend + +from .base import Transform + + +class ID(Transform[frontend.ID]): + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/transform/to_lower.py b/tests/frontend/fixtures/implementation/transform/to_lower.py new file mode 100644 index 0000000..fb1f804 --- /dev/null +++ b/tests/frontend/fixtures/implementation/transform/to_lower.py @@ -0,0 +1,8 @@ +from llparse_frontend import transform as frontend + +from .base import Transform + + +class ToLower(Transform[frontend.ToLower]): + def build(self) -> str: + return "" diff --git a/tests/frontend/fixtures/implementation/transform/to_lower_unsafe.py b/tests/frontend/fixtures/implementation/transform/to_lower_unsafe.py new file mode 100644 index 0000000..2c38d15 --- /dev/null +++ b/tests/frontend/fixtures/implementation/transform/to_lower_unsafe.py @@ -0,0 +1,8 @@ +from llparse_frontend import transform as frontend + +from .base import Transform + + +class ToLowerUnsafe(Transform[frontend.ToLowerUnsafe]): + def build(self) -> str: + return "" diff --git a/tests/frontend/test_container.py b/tests/frontend/test_container.py new file mode 100644 index 0000000..1dcfd56 --- /dev/null +++ b/tests/frontend/test_container.py @@ -0,0 +1,32 @@ +# Work in progress do not use... +# from llparse_builder.builder import Builder +# from llparse_frontend.container import Container +# from llparse_frontend.frontend import Frontend + +# from .fixtures import implementation + + +# def test_node_translation(b: Builder): +# comb = Container() +# comb.add('a', implementation) +# comb.add('b', implementation) + +# f = Frontend('llparse', comb.build()) + +# root = b.node('root') + +# root.match('ab', root) +# root.match('acd', root) +# root.match('efg', root) +# root.otherwise(b.error(123, 'hello')) + +# f_root = f.compile(root, []).root + +# out: list[str] = [] +# f_root.get('a').build(out) + + +# assert '' in out + diff --git a/tests/frontend/test_frontend.py b/tests/frontend/test_frontend.py new file mode 100644 index 0000000..1695bb1 --- /dev/null +++ b/tests/frontend/test_frontend.py @@ -0,0 +1,142 @@ +from typing import cast + +import pytest + +from llparse_builder import builder as source +from llparse_frontend import node +from llparse_frontend.frontend import Frontend + +from .fixtures import implementation +from .fixtures.implementation.node.base import Node + + +@pytest.fixture +def f() -> Frontend: + return Frontend("llparse", implementation) + + +def check_nodes(f: Frontend, root: source.node.Node, expected: list[str]): + f_root = cast(Node[node.Node], f.compile(root, []).root) + out = [] + + f_root.build(out) + + # XXX: we don't expect the python parody to lay these + # out exactly as typescript does we just care about + # the functionality and how it works. Ordering is not a concern. + + for line in out: + print(line) + assert line in expected + + +def check_resumption_targets(f: Frontend, expected: list[str]): + targets = [t.ref.id.name for t in f.resumption_targets] + + for line in targets: + assert line in expected + + +def test_node_implementation_translation(f: Frontend, b: source.Builder) -> None: + root = b.node("root") + + root.match("ab", root) + root.match("acd", root) + root.match("efg", root) + root.otherwise(b.error(123, "hello")) + + check_nodes( + f, + root, + [ + "", + "", + "", + "", + "", + "", + ], + ) + + check_resumption_targets( + f, + # any order can be done as long as all of them are here... + [ + "llparse__n_root", + "llparse__n_root_1", + "llparse__n_root_2", + "llparse__n_root_3", + ], + ) + + +def test_peephole_optimization(f: Frontend, b: source.Builder) -> None: + root = b.node("root") + root1 = b.node("a") + root2 = b.node("b") + node1 = b.node("c") + node2 = b.node("d") + + root.otherwise(root1) + root1.otherwise(root2) + root2.skip_to(node1) + node1.otherwise(node2) + node2.otherwise(root) + + check_nodes( + f, + root, + [ + "", + ], + ) + + check_resumption_targets( + f, + [ + "llparse__n_b", + ], + ) + + +def test_resumption_targets(b: source.Builder, f: Frontend): + b.property("i64", "counter") + root = b.node("root") + end = b.node("end") + store = b.invoke(b.code.store("counter")) + + root.select({"a": 1, "b": 2}, store) + root.otherwise(b.error(1, "okay")) + + store.otherwise(end) + + end.match("ohai", root) + end.match( + "paus", b.pause(1, "paused").otherwise(b.pause(2, "paused").otherwise(root)) + ) + end.otherwise(b.error(2, "ohai")) + + check_nodes( + f, + root, + [ + "", + "", + "", + '', + "", + "", + '', + ], + ) + + check_resumption_targets( + f, + [ + "llparse__n_root", + "llparse__n_end", + "llparse__n_end_1", + "llparse__n_end_2", + "llparse__n_pause_1", + ], + ) diff --git a/tests/test_capi.py b/tests/test_capi.py deleted file mode 100644 index ed52b9b..0000000 --- a/tests/test_capi.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Tests tools for writing C-API Wrappers -""" -from llparse import LLParse -import re - -import pytest - -DUMMY_HEADER = """#ifndef LLPARSE_CAPI_INCLUDE -#define LLPARSE_CAPI_INCLUDE -#ifdef __cplusplus -extern "C" { -#endif -#include -#include - -#if defined(__wasm__) -#define LLPARSE_EXPORT __attribute__((visibility("default"))) -#elif defined(_WIN32) -#define LLPARSE_EXPORT __declspec(dllexport) -#else -#define LLPARSE_EXPORT -#endif - -typedef llparse_internal_t llparse_t; -typedef struct llparse_settings_s llparse_settings_t; -typedef int (*llparse_data_cb)(llparse_t*, const char* at, size_t length); -typedef int (*llparse_cb)(llparse_t*); - -struct llparse_settings_s { - /* Spans */ - llparse_data_cb llparse_on_span; - /* Callbacks */ - llparse_cb llparse_on_test; -}; - -LLPARSE_EXPORT -void llparse_settings_init(llparse_settings_t* settings); - -LLPARSE_EXPORT -int llparse_execute(llparse_t* parser, const char* data, size_t len); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* LLPARSE_CAPI_INCLUDE */ -""" - - - -@pytest.fixture() -def llparse() -> LLParse: - return LLParse("llparse_internal") - -def test_collecting_spans(llparse:LLParse): - lc = llparse.capi("llparse") - span = llparse.span(llparse.code.span("span")) - start = llparse.node("start") - body = llparse.node("body") - - start.otherwise(span.start(body)) - - body.skipTo(span.end(start)) - - lc.use("span") - result = lc.filter(start) - assert result.use.spans, "No spans found" - - -def test_collecting_matches(llparse:LLParse): - lc = llparse.capi("lc") - span = llparse.span(llparse.code.span("llparse_on_span")) - on_test = llparse.code.match("llparse_on_test") - - start = llparse.node("start") - body = llparse.node("body") - - start.otherwise( - span.start(body) - ) - - body.skipTo( - span.end( - llparse.invoke(on_test, {0:start}, llparse.error(-1, "error")) - ) - ) - lc.use("llparse_") - result = lc.filter(start) - assert result.use.spans, "No spans found" - assert result.use.matches, "No matches found" - - -def test_write_capi(llparse:LLParse): - lc = llparse.capi("llparse") - span = llparse.span(llparse.code.span("llparse_on_span")) - on_test = llparse.code.match("llparse_on_test") - - start = llparse.node("start") - body = llparse.node("body") - - start.otherwise( - span.start(body) - ) - - body.skipTo( - span.end( - llparse.invoke(on_test, {0:start}, llparse.error(-1, "error")) - ) - ) - - lc.use("span") - lc.use_regex(r"llparse_([^\s]+)") - - - result = lc.build(start) - assert result.header.strip() == DUMMY_HEADER.strip() - - - - diff --git a/tests/test_compilator.py b/tests/test_compilator.py deleted file mode 100644 index 863970e..0000000 --- a/tests/test_compilator.py +++ /dev/null @@ -1,53 +0,0 @@ -# The good old http_parser was borrowed from llparse.org to demonstrate this for you :) -from llparse import LLParse - - -def test_http_parser_example(): - p = LLParse("http_parser") - method = p.node("method") - beforeUrl = p.node("before_url") - urlSpan = p.span(p.code.span("on_url")) - url = p.node("url") - http = p.node("http") - - # Add custom uint8_t property to the state - p.property("i8", "method") - - # Store method inside a custom property - onMethod = p.invoke(p.code.store("method"), beforeUrl) - - # Invoke custom C function - complete = p.invoke( - p.code.match("on_complete"), - { - # Restart - 0: method - }, - p.error(4, "`on_complete` error"), - ) - - method.select( - { - "HEAD": 0, - "GET": 1, - "POST": 2, - "PUT": 3, - "DELETE": 4, - "OPTIONS": 5, - "CONNECT": 6, - "TRACE": 7, - "PATCH": 8, - }, - onMethod, - ).otherwise(p.error(5, "Expected method")) - - beforeUrl.match(" ", beforeUrl).otherwise(urlSpan.start(url)) - - url.peek(" ", urlSpan.end(http)).skipTo(url) - - http.match(" HTTP/1.1\r\n\r\n", complete).otherwise( - p.error(6, "Expected HTTP/1.1 and two newlines") - ) - - # if this build fails in any way then we have failed... - c = p.build(method) diff --git a/tests/test_frontend.py b/tests/test_frontend.py deleted file mode 100644 index f6265fa..0000000 --- a/tests/test_frontend.py +++ /dev/null @@ -1,180 +0,0 @@ -from llparse import LLParse -from llparse.pybuilder.main_code import Operator - -import pytest - - -@pytest.fixture(params=[">", "<", ">=", "<="]) -def op(request: pytest.FixtureRequest) -> str: - return request.param - - -def test_build_tables(): - # There was a bug with 1.2 of our version that doesn't affect the node-js one where - # it wouldn't building tables, this attempts to simulate the problem Currenlty this - # bug is patched now :) - p = LLParse("lltable") - start = p.node("start") - loop = p.node("loop") - loop.skipTo(start) - start.match( - [ - 48, - 49, - 50, - 51, - 52, - 53, - 54, - 55, - 56, - 57, - 97, - 98, - 99, - 100, - 101, - 102, - 103, - 104, - 105, - 106, - 107, - 108, - 109, - 110, - 111, - 112, - 113, - 114, - 115, - 116, - 117, - 118, - 119, - 120, - 121, - 122, - 65, - 66, - 67, - 68, - 69, - 70, - 71, - 72, - 73, - 74, - 75, - 76, - 77, - 78, - 79, - 80, - 81, - 82, - 83, - 84, - 85, - 86, - 87, - 88, - 89, - 90, - 33, - 34, - 35, - 36, - 37, - 38, - 39, - 40, - 41, - 42, - 43, - 44, - 45, - 46, - 47, - 58, - 59, - 60, - 61, - 62, - 63, - 64, - 91, - 92, - 93, - 94, - 95, - 96, - 123, - 124, - 125, - 126, - 32, - 9, - 10, - 13, - 11, - 12, - ], - loop, - ).otherwise(p.error(0, "im a little teapot")) - - # If there is not a lookup_table this then it has failed me ;-; - assert "lookup_table" in p.build(start).c - - -def test_pausing(): - # Ensure frotentend LoopChecker does not mark off against Pausing - p = LLParse("lltest") - s = p.node("start") - s2 = p.node("start2") - s.match("p", p.pause(1, "parser was asked to pause").otherwise(s2)).skipTo(s) - s2.match("p", p.pause(2, "parser was asked to pause again").otherwise(s)).skipTo(s2) - p.build(s) - - -def test_intnodes(): - # IntNodes should be registered to llparse_state_t and should be a resumption target - # Otherwise we crash during dynamic streams of data this is a bug in 0.1.6 that is fixed in 0.1.7 - p = LLParse("lltest") - p.property("i16", "value") - start = p.node("start") - value = p.intBE("value", bits=2).skipTo(start) - start.skipTo(value) - - artifacts = p.build(start) - code = artifacts.c.splitlines() - assert " case s_n_lltest__n_value_int_16be_byte2:" in code, ( - "Int Node should be a resumption target" - ) - - -def test_operators(op: str): - test = LLParse("test") - - test.property("i16", "a") - - node = test.node("node_1") - # WARNING: Don't try and do this normally need operator exposed for making fixtures a bit easier. - node.match( - "1", - test.invoke( - Operator(op, "a", 10), {1: node}, test.error(1, "a is not greater than 10") - ), - ).otherwise(test.error(2, "lol")) - - t = test.build(node) - code = t.c.splitlines(keepends=False) - assert f" return state->a {op} 10;" in code - if op == ">": - assert "int test__c_gt_a_10 (" in code - elif op == "<": - assert "int test__c_lt_a_10 (" in code - elif op == ">=": - assert "int test__c_ge_a_10 (" in code - elif op == "<=": - assert "int test__c_le_a_10 (" in code