diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7ae51b1..3a9d353 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,6 +28,9 @@ jobs: sudo apt-get update -y sudo apt-get install -y build-essential llvm clang cmake python3-pip + - name: Install Python Dependencies + run: python3 -m pip install -r requirements-dev.txt + - name: Build zpiler run: | mkdir -p build && cd build @@ -35,7 +38,7 @@ jobs: make -j - name: Run Tests - run: python3 test_runner.py + run: python3 -m pytest -q test-windows: runs-on: windows-latest @@ -58,7 +61,7 @@ jobs: uses: actions/cache@v4 with: path: C:\Users\runneradmin\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} restore-keys: | ${{ runner.os }}-pip- @@ -68,6 +71,9 @@ jobs: - name: Install CMake run: choco install --no-progress -y cmake + - name: Install Python Dependencies + run: python -m pip install -r requirements-dev.txt + - name: Build zpiler shell: cmd run: | @@ -80,4 +86,4 @@ jobs: shell: cmd run: | chcp 65001 - python test_runner.py + python -m pytest -q diff --git a/README.md b/README.md index b2523ee..4a00354 100644 --- a/README.md +++ b/README.md @@ -145,18 +145,24 @@ Generates: `program` (executable) ### Automated Testing -Run all test cases for a specific target: +Install test dependencies: ```bash -# Test native platform (auto-detected) -python test_runner.py +python3 -m pip install -r requirements-dev.txt +``` + +Run strict golden-output tests with `pytest`: +```bash +# Auto-detect native target when TARGET is not set +pytest -q -# Test specific targets -TARGET=linux python test_runner.py # Linux -TARGET=windows python test_runner.py # Windows (if ml64 available) -TARGET=llvm python test_runner.py # LLVM +# Linux + LLVM +TARGET=linux,llvm pytest -q -# Test multiple targets -TARGET=linux,llvm python test_runner.py +# Windows +TARGET=windows pytest -q + +# Update/create runtime goldens explicitly +TARGET=linux pytest -q --bless ``` --- @@ -195,7 +201,13 @@ fn main() { ## 🧪 Testing -You can add `.zz` programs inside a `tests/` folder and run them individually through the pipeline. Type errors, parsing errors, and codegen output are printed to the terminal. +Test suites are directory-based: + +- `tests/runtime`: program must compile, run, and match exact `stdout`/`stderr`/exit code +- `tests/runtime_fail`: program must compile, run, and match exact failing output + nonzero exit code +- `tests/compile_fail`: program must fail compilation with expected exit code and required stderr substrings + +Expected files live under `tests/expected//...` with mirrored paths. --- diff --git a/__pycache__/test_runner.cpython-312-pytest-7.4.4.pyc b/__pycache__/test_runner.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..e112eae Binary files /dev/null and b/__pycache__/test_runner.cpython-312-pytest-7.4.4.pyc differ diff --git a/main.cpp b/main.cpp index d7022c9..e9942db 100644 --- a/main.cpp +++ b/main.cpp @@ -130,6 +130,7 @@ int main(int argc, char *argv[]) { cg->generate(std::move(program)); } catch (std::exception const &exc) { std::cerr << "ERROR: " << exc.what() << "\n"; + return 1; } return 0; diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..40543aa --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +pytest==8.3.3 diff --git a/src/parser/Parser.cpp b/src/parser/Parser.cpp index 1686647..b9f6413 100644 --- a/src/parser/Parser.cpp +++ b/src/parser/Parser.cpp @@ -200,7 +200,7 @@ namespace zust errMsg + " at line " + std::to_string(currentToken.line) + ", column " + std::to_string(currentToken.column)}); shouldTypecheck = false; - exit(0); + exit(1); return; } } diff --git a/tests/__pycache__/conftest.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/conftest.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..e9f967c Binary files /dev/null and b/tests/__pycache__/conftest.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/__pycache__/conftest.cpython-312.pyc b/tests/__pycache__/conftest.cpython-312.pyc new file mode 100644 index 0000000..2c43f3b Binary files /dev/null and b/tests/__pycache__/conftest.cpython-312.pyc differ diff --git a/tests/__pycache__/test_pipeline.cpython-312-pytest-7.4.4.pyc b/tests/__pycache__/test_pipeline.cpython-312-pytest-7.4.4.pyc new file mode 100644 index 0000000..4b31f17 Binary files /dev/null and b/tests/__pycache__/test_pipeline.cpython-312-pytest-7.4.4.pyc differ diff --git a/tests/__pycache__/test_pipeline.cpython-312.pyc b/tests/__pycache__/test_pipeline.cpython-312.pyc new file mode 100644 index 0000000..11d450f Binary files /dev/null and b/tests/__pycache__/test_pipeline.cpython-312.pyc differ diff --git a/tests/compile_fail/syntax/missing_semicolon.zz b/tests/compile_fail/syntax/missing_semicolon.zz new file mode 100644 index 0000000..de4bb19 --- /dev/null +++ b/tests/compile_fail/syntax/missing_semicolon.zz @@ -0,0 +1,3 @@ +fn main() { + let a: int64_t = 10 +} diff --git a/tests/compile_fail/type/bad_assignment.zz b/tests/compile_fail/type/bad_assignment.zz new file mode 100644 index 0000000..cc0fabb --- /dev/null +++ b/tests/compile_fail/type/bad_assignment.zz @@ -0,0 +1,4 @@ +extern fn foo(x: not_a_type) -> none; + +fn main() { +} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..552a1e5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import os +import sys +from dataclasses import dataclass +from pathlib import Path + +import pytest + + +@dataclass(frozen=True) +class TargetConfig: + name: str + zpiler_format: str + asm_ext: str + + +TARGETS: dict[str, TargetConfig] = { + "linux": TargetConfig(name="linux", zpiler_format="x86_64-linux", asm_ext=".s"), + "windows": TargetConfig( + name="windows", zpiler_format="x86_64-mswin", asm_ext=".asm" + ), + "llvm": TargetConfig(name="llvm", zpiler_format="llvm-ir", asm_ext=".ll"), +} + + +def detect_native_target() -> str: + plat = sys.platform + if plat.startswith("linux"): + return "linux" + if plat.startswith("win32") or plat.startswith("cygwin"): + return "windows" + return "llvm" + + +def parse_target_env() -> list[str]: + env = os.getenv("TARGET") + if env: + wanted = [t.strip() for t in env.split(",") if t.strip() in TARGETS] + if not wanted: + raise pytest.UsageError( + f"TARGET={env} contains no known backends. " + f"Known targets: {', '.join(TARGETS)}" + ) + return wanted + return [detect_native_target()] + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--bless", + action="store_true", + default=False, + help="Create or update runtime golden output files.", + ) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "target_name" in metafunc.fixturenames: + targets = parse_target_env() + metafunc.parametrize( + "target_name", targets, ids=[f"target={target}" for target in targets] + ) + + +@pytest.fixture(scope="session") +def bless(request: pytest.FixtureRequest) -> bool: + return bool(request.config.getoption("--bless")) + + +@pytest.fixture(scope="session") +def project_root() -> Path: + return Path(__file__).resolve().parents[1] + + +@pytest.fixture(scope="session") +def zpiler_path(project_root: Path) -> Path: + candidates = [ + project_root / "build" / "zpiler", + project_root / "build" / "Debug" / "zpiler.exe", + project_root / "build" / "zpiler.exe", + ] + for candidate in candidates: + if candidate.exists(): + return candidate + + candidate_text = "\n".join(str(path) for path in candidates) + pytest.exit( + "Could not find zpiler executable. Build project first.\n" + f"Searched:\n{candidate_text}", + returncode=2, + ) + + +@pytest.fixture(scope="session") +def artifacts_root(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("pipeline_artifacts") + + +@pytest.fixture +def target_config(target_name: str) -> TargetConfig: + return TARGETS[target_name] diff --git a/tests/expected/compile_fail/syntax/missing_semicolon.stderr.contains b/tests/expected/compile_fail/syntax/missing_semicolon.stderr.contains new file mode 100644 index 0000000..5ec650b --- /dev/null +++ b/tests/expected/compile_fail/syntax/missing_semicolon.stderr.contains @@ -0,0 +1,3 @@ +# Required syntax-failure markers +[Syntax Error] +Expected ';' after declaration diff --git a/tests/expected/compile_fail/type/bad_assignment.exitcode b/tests/expected/compile_fail/type/bad_assignment.exitcode new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/tests/expected/compile_fail/type/bad_assignment.exitcode @@ -0,0 +1 @@ +1 diff --git a/tests/expected/compile_fail/type/bad_assignment.stderr.contains b/tests/expected/compile_fail/type/bad_assignment.stderr.contains new file mode 100644 index 0000000..16d4c26 --- /dev/null +++ b/tests/expected/compile_fail/type/bad_assignment.stderr.contains @@ -0,0 +1,3 @@ +# Required semantic/type-failure markers +[Type Error] +Undefined type 'not_a_type' diff --git a/tests/expected/runtime/conditionals/if-elif-else.exitcode b/tests/expected/runtime/conditionals/if-elif-else.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/conditionals/if-elif-else.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/conditionals/if-elif-else.stderr b/tests/expected/runtime/conditionals/if-elif-else.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/conditionals/if-elif-else.stdout b/tests/expected/runtime/conditionals/if-elif-else.stdout new file mode 100644 index 0000000..b4de394 --- /dev/null +++ b/tests/expected/runtime/conditionals/if-elif-else.stdout @@ -0,0 +1 @@ +11 diff --git a/tests/expected/runtime/conditionals/if-elif.exitcode b/tests/expected/runtime/conditionals/if-elif.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/conditionals/if-elif.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/conditionals/if-elif.stderr b/tests/expected/runtime/conditionals/if-elif.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/conditionals/if-elif.stdout b/tests/expected/runtime/conditionals/if-elif.stdout new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/conditionals/if-else.exitcode b/tests/expected/runtime/conditionals/if-else.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/conditionals/if-else.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/conditionals/if-else.stderr b/tests/expected/runtime/conditionals/if-else.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/conditionals/if-else.stdout b/tests/expected/runtime/conditionals/if-else.stdout new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/conditionals/if.exitcode b/tests/expected/runtime/conditionals/if.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/conditionals/if.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/conditionals/if.stderr b/tests/expected/runtime/conditionals/if.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/conditionals/if.stdout b/tests/expected/runtime/conditionals/if.stdout new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/functions/basics.exitcode b/tests/expected/runtime/functions/basics.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/functions/basics.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/functions/basics.stderr b/tests/expected/runtime/functions/basics.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/functions/basics.stdout b/tests/expected/runtime/functions/basics.stdout new file mode 100644 index 0000000..d9a9da2 --- /dev/null +++ b/tests/expected/runtime/functions/basics.stdout @@ -0,0 +1,3 @@ +42 +9.000000 +true diff --git a/tests/expected/runtime/functions/recursive.exitcode b/tests/expected/runtime/functions/recursive.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/functions/recursive.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/functions/recursive.stderr b/tests/expected/runtime/functions/recursive.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/functions/recursive.stdout b/tests/expected/runtime/functions/recursive.stdout new file mode 100644 index 0000000..39af040 --- /dev/null +++ b/tests/expected/runtime/functions/recursive.stdout @@ -0,0 +1,3 @@ +120 +true +false diff --git a/tests/expected/runtime/functions/scopes.exitcode b/tests/expected/runtime/functions/scopes.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/functions/scopes.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/functions/scopes.stderr b/tests/expected/runtime/functions/scopes.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/functions/scopes.stdout b/tests/expected/runtime/functions/scopes.stdout new file mode 100644 index 0000000..2bd5a0a --- /dev/null +++ b/tests/expected/runtime/functions/scopes.stdout @@ -0,0 +1 @@ +22 diff --git a/tests/expected/runtime/loops/control.exitcode b/tests/expected/runtime/loops/control.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/loops/control.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/loops/control.stderr b/tests/expected/runtime/loops/control.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/loops/control.stdout b/tests/expected/runtime/loops/control.stdout new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/operations/binary.exitcode b/tests/expected/runtime/operations/binary.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/operations/binary.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/operations/binary.stderr b/tests/expected/runtime/operations/binary.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/operations/binary.stdout b/tests/expected/runtime/operations/binary.stdout new file mode 100644 index 0000000..b6ac604 --- /dev/null +++ b/tests/expected/runtime/operations/binary.stdout @@ -0,0 +1,24 @@ +30 +-10 +200 +2 +12.500000 +-7.500000 +25.000000 +4.000000 +5.859500 +0.423500 +8.538597 +1.155813 +3 +true +true +false +false +false +true +false +false +false +false +true diff --git a/tests/expected/runtime/operations/unary.exitcode b/tests/expected/runtime/operations/unary.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/operations/unary.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/operations/unary.stderr b/tests/expected/runtime/operations/unary.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/operations/unary.stdout b/tests/expected/runtime/operations/unary.stdout new file mode 100644 index 0000000..553e21a --- /dev/null +++ b/tests/expected/runtime/operations/unary.stdout @@ -0,0 +1,5 @@ +5 +255 +100 +0 +1 diff --git a/tests/expected/runtime/strings/escapes.exitcode b/tests/expected/runtime/strings/escapes.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/strings/escapes.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/strings/escapes.stderr b/tests/expected/runtime/strings/escapes.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/strings/escapes.stdout b/tests/expected/runtime/strings/escapes.stdout new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/types.exitcode b/tests/expected/runtime/types.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/types.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/types.stderr b/tests/expected/runtime/types.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/types.stdout b/tests/expected/runtime/types.stdout new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/types.stdout @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/variables.exitcode b/tests/expected/runtime/variables.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/tests/expected/runtime/variables.exitcode @@ -0,0 +1 @@ +0 diff --git a/tests/expected/runtime/variables.linux.stdout b/tests/expected/runtime/variables.linux.stdout new file mode 100644 index 0000000..ebce988 --- /dev/null +++ b/tests/expected/runtime/variables.linux.stdout @@ -0,0 +1 @@ +It is what it is \ No newline at end of file diff --git a/tests/expected/runtime/variables.llvm.stdout b/tests/expected/runtime/variables.llvm.stdout new file mode 100644 index 0000000..ebce988 --- /dev/null +++ b/tests/expected/runtime/variables.llvm.stdout @@ -0,0 +1 @@ +It is what it is \ No newline at end of file diff --git a/tests/expected/runtime/variables.stderr b/tests/expected/runtime/variables.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime/variables.stdout b/tests/expected/runtime/variables.stdout new file mode 100644 index 0000000..89c8485 --- /dev/null +++ b/tests/expected/runtime/variables.stdout @@ -0,0 +1 @@ +DO_NOT_USE_SHARED_VARIABLES_STDOUT diff --git a/tests/expected/runtime/variables.windows.stdout b/tests/expected/runtime/variables.windows.stdout new file mode 100644 index 0000000..ebce988 --- /dev/null +++ b/tests/expected/runtime/variables.windows.stdout @@ -0,0 +1 @@ +It is what it is \ No newline at end of file diff --git a/tests/expected/runtime_fail/exit/nonzero.exitcode b/tests/expected/runtime_fail/exit/nonzero.exitcode new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/tests/expected/runtime_fail/exit/nonzero.exitcode @@ -0,0 +1 @@ +7 diff --git a/tests/expected/runtime_fail/exit/nonzero.stderr b/tests/expected/runtime_fail/exit/nonzero.stderr new file mode 100644 index 0000000..e69de29 diff --git a/tests/expected/runtime_fail/exit/nonzero.stdout b/tests/expected/runtime_fail/exit/nonzero.stdout new file mode 100644 index 0000000..050fd9d --- /dev/null +++ b/tests/expected/runtime_fail/exit/nonzero.stdout @@ -0,0 +1 @@ +about-to-fail diff --git a/tests/zz/conditionals/if-elif-else.zz b/tests/runtime/conditionals/if-elif-else.zz similarity index 100% rename from tests/zz/conditionals/if-elif-else.zz rename to tests/runtime/conditionals/if-elif-else.zz diff --git a/tests/zz/conditionals/if-elif.zz b/tests/runtime/conditionals/if-elif.zz similarity index 100% rename from tests/zz/conditionals/if-elif.zz rename to tests/runtime/conditionals/if-elif.zz diff --git a/tests/zz/conditionals/if-else.zz b/tests/runtime/conditionals/if-else.zz similarity index 100% rename from tests/zz/conditionals/if-else.zz rename to tests/runtime/conditionals/if-else.zz diff --git a/tests/zz/conditionals/if.zz b/tests/runtime/conditionals/if.zz similarity index 100% rename from tests/zz/conditionals/if.zz rename to tests/runtime/conditionals/if.zz diff --git a/tests/zz/functions/basics.zz b/tests/runtime/functions/basics.zz similarity index 100% rename from tests/zz/functions/basics.zz rename to tests/runtime/functions/basics.zz diff --git a/tests/zz/functions/recursive.zz b/tests/runtime/functions/recursive.zz similarity index 100% rename from tests/zz/functions/recursive.zz rename to tests/runtime/functions/recursive.zz diff --git a/tests/zz/functions/scopes.zz b/tests/runtime/functions/scopes.zz similarity index 100% rename from tests/zz/functions/scopes.zz rename to tests/runtime/functions/scopes.zz diff --git a/tests/zz/loops/control.zz b/tests/runtime/loops/control.zz similarity index 100% rename from tests/zz/loops/control.zz rename to tests/runtime/loops/control.zz diff --git a/tests/zz/operations/binary.zz b/tests/runtime/operations/binary.zz similarity index 100% rename from tests/zz/operations/binary.zz rename to tests/runtime/operations/binary.zz diff --git a/tests/zz/operations/unary.zz b/tests/runtime/operations/unary.zz similarity index 100% rename from tests/zz/operations/unary.zz rename to tests/runtime/operations/unary.zz diff --git a/tests/zz/strings/escapes.zz b/tests/runtime/strings/escapes.zz similarity index 100% rename from tests/zz/strings/escapes.zz rename to tests/runtime/strings/escapes.zz diff --git a/tests/zz/types.zz b/tests/runtime/types.zz similarity index 100% rename from tests/zz/types.zz rename to tests/runtime/types.zz diff --git a/tests/zz/variables.zz b/tests/runtime/variables.zz similarity index 100% rename from tests/zz/variables.zz rename to tests/runtime/variables.zz diff --git a/tests/runtime_fail/exit/nonzero.zz b/tests/runtime_fail/exit/nonzero.zz new file mode 100644 index 0000000..301d201 --- /dev/null +++ b/tests/runtime_fail/exit/nonzero.zz @@ -0,0 +1,7 @@ +extern fn printf(fmt: string, ...) -> int32_t; +extern fn exit(code: int32_t) -> none; + +fn main() { + printf("about-to-fail\n"); + exit(7); +} diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..eb74446 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,568 @@ +from __future__ import annotations + +import difflib +import shutil +import subprocess +from pathlib import Path + +import pytest + + +TESTS_DIR = Path(__file__).resolve().parent +RUNTIME_DIR = TESTS_DIR / "runtime" +RUNTIME_FAIL_DIR = TESTS_DIR / "runtime_fail" +COMPILE_FAIL_DIR = TESTS_DIR / "compile_fail" +EXPECTED_DIR = TESTS_DIR / "expected" + + +def discover_cases(root: Path) -> list[Path]: + if not root.exists(): + return [] + return sorted(path.relative_to(root) for path in root.rglob("*.zz")) + + +RUNTIME_CASES = discover_cases(RUNTIME_DIR) +RUNTIME_FAIL_CASES = discover_cases(RUNTIME_FAIL_DIR) +COMPILE_FAIL_CASES = discover_cases(COMPILE_FAIL_DIR) + + +def normalize_line_endings(data: bytes) -> bytes: + return data.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + + +def decode_text(data: bytes) -> str: + return data.decode("utf-8", errors="replace") + + +def diff_text(expected: bytes, actual: bytes, label: str) -> str: + expected_text = decode_text(expected) + actual_text = decode_text(actual) + diff = "".join( + difflib.unified_diff( + expected_text.splitlines(keepends=True), + actual_text.splitlines(keepends=True), + fromfile=f"expected/{label}", + tofile=f"actual/{label}", + ) + ) + if diff: + return diff + return ( + f"Byte mismatch for {label}, but textual diff is empty.\n" + f"expected={expected!r}\nactual={actual!r}\n" + ) + + +def run_process( + cmd: list[str], *, cwd: Path | None = None +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + cmd, + cwd=str(cwd) if cwd is not None else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def format_process_result( + step: str, cmd: list[str], proc: subprocess.CompletedProcess[bytes] +) -> str: + stdout = decode_text(normalize_line_endings(proc.stdout)) + stderr = decode_text(normalize_line_endings(proc.stderr)) + return ( + f"{step} failed.\n" + f"Command: {' '.join(cmd)}\n" + f"Exit code: {proc.returncode}\n" + f"--- stdout ---\n{stdout}\n" + f"--- stderr ---\n{stderr}" + ) + + +def compile_source( + *, + source: Path, + asm_out: Path, + target_config, + zpiler_path: Path, +) -> tuple[list[str], subprocess.CompletedProcess[bytes]]: + cmd = [ + str(zpiler_path), + "--format", + target_config.zpiler_format, + "-o", + str(asm_out), + str(source), + ] + return cmd, run_process(cmd) + + +def assemble_source( + *, + target_name: str, + asm_out: Path, + obj_out: Path, + work_dir: Path, +) -> tuple[list[str], subprocess.CompletedProcess[bytes]]: + if target_name == "linux": + cmd = ["as", str(asm_out), "-o", str(obj_out)] + return cmd, run_process(cmd) + + if target_name == "llvm": + cmd = ["llc", "-filetype=obj", str(asm_out), "-o", str(obj_out)] + return cmd, run_process(cmd) + + if target_name == "windows": + cmd = ["ml64", "/nologo", "/c", str(asm_out)] + proc = run_process(cmd, cwd=work_dir) + if proc.returncode != 0: + return cmd, proc + + obj_temp = work_dir / f"{asm_out.stem}.obj" + if not obj_temp.exists(): + return ( + cmd, + subprocess.CompletedProcess( + cmd, + returncode=1, + stdout=proc.stdout, + stderr=proc.stderr + + f"\nExpected object file not found: {obj_temp}\n".encode(), + ), + ) + + obj_out.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(obj_temp), str(obj_out)) + return cmd, proc + + raise ValueError(f"Unknown target: {target_name}") + + +def link_object( + *, + target_name: str, + obj_out: Path, + exe_out: Path, +) -> tuple[list[str], subprocess.CompletedProcess[bytes]]: + cmd = ["gcc", str(obj_out), "-o", str(exe_out)] + if target_name == "llvm": + cmd.append("-no-pie") + return cmd, run_process(cmd) + + +def run_executable(exe_out: Path, work_dir: Path) -> subprocess.CompletedProcess[bytes]: + return run_process([str(exe_out)], cwd=work_dir) + + +def expectation_candidates(base: Path, target_name: str, ext: str) -> tuple[Path, Path]: + specific = Path(f"{base}.{target_name}.{ext}") + shared = Path(f"{base}.{ext}") + return specific, shared + + +def resolve_existing_expectation(base: Path, target_name: str, ext: str) -> Path: + specific, shared = expectation_candidates(base, target_name, ext) + if specific.exists(): + return specific + return shared + + +def resolve_write_expectation(base: Path, target_name: str, ext: str) -> Path: + specific, shared = expectation_candidates(base, target_name, ext) + if specific.exists(): + return specific + if shared.exists(): + return shared + return shared + + +def expected_base(mode: str, rel_path: Path) -> Path: + return EXPECTED_DIR / mode / rel_path.with_suffix("") + + +def parse_exit_code(path: Path) -> int: + text = decode_text(normalize_line_endings(path.read_bytes())).strip() + if not text: + pytest.fail(f"Expected exit code file is empty: {path}", pytrace=False) + try: + return int(text) + except ValueError as exc: + pytest.fail(f"Invalid exit code in {path}: {text!r}", pytrace=False) + raise exc + + +def load_or_bless_output( + *, + base: Path, + target_name: str, + ext: str, + actual: bytes, + bless: bool, +) -> tuple[bytes, Path]: + actual_norm = normalize_line_endings(actual) + read_path = resolve_existing_expectation(base, target_name, ext) + + if bless: + write_path = resolve_write_expectation(base, target_name, ext) + write_path.parent.mkdir(parents=True, exist_ok=True) + if (not write_path.exists()) or ( + normalize_line_endings(write_path.read_bytes()) != actual_norm + ): + write_path.write_bytes(actual_norm) + return actual_norm, write_path + + if not read_path.exists(): + pytest.fail( + f"Missing expected file: {read_path}\n" + "Re-run with --bless to create runtime expectations.", + pytrace=False, + ) + + return normalize_line_endings(read_path.read_bytes()), read_path + + +def load_or_bless_exit_code( + *, + base: Path, + target_name: str, + actual: int, + bless: bool, +) -> tuple[int, Path]: + read_path = resolve_existing_expectation(base, target_name, "exitcode") + + if bless: + write_path = resolve_write_expectation(base, target_name, "exitcode") + write_path.parent.mkdir(parents=True, exist_ok=True) + rendered = f"{actual}\n".encode() + if (not write_path.exists()) or (write_path.read_bytes() != rendered): + write_path.write_bytes(rendered) + return actual, write_path + + if not read_path.exists(): + pytest.fail( + f"Missing expected file: {read_path}\n" + "Re-run with --bless to create runtime expectations.", + pytrace=False, + ) + return parse_exit_code(read_path), read_path + + +def assert_bytes_equal( + *, + label: str, + expected: bytes, + actual: bytes, + expected_path: Path, + source: Path, + target_name: str, +) -> None: + if expected == actual: + return + diff = diff_text(expected, actual, label) + raise AssertionError( + f"{source} on target={target_name} mismatched {label}.\n" + f"Expectation file: {expected_path}\n{diff}" + ) + + +def build_and_run( + *, + mode: str, + rel_path: Path, + source_root: Path, + target_name: str, + target_config, + zpiler_path: Path, + artifacts_root: Path, +) -> subprocess.CompletedProcess[bytes]: + source = source_root / rel_path + case_root = artifacts_root / mode / target_name / rel_path.with_suffix("") + case_root.mkdir(parents=True, exist_ok=True) + + asm_out = case_root / f"{source.stem}{target_config.asm_ext}" + obj_out = case_root / f"{source.stem}.obj" + exe_out = case_root / f"{source.stem}.exe" + + compile_cmd, compile_proc = compile_source( + source=source, + asm_out=asm_out, + target_config=target_config, + zpiler_path=zpiler_path, + ) + if compile_proc.returncode != 0: + pytest.fail( + format_process_result("Compile", compile_cmd, compile_proc), pytrace=False + ) + + assemble_cmd, assemble_proc = assemble_source( + target_name=target_name, + asm_out=asm_out, + obj_out=obj_out, + work_dir=case_root, + ) + if assemble_proc.returncode != 0: + pytest.fail( + format_process_result("Assemble", assemble_cmd, assemble_proc), pytrace=False + ) + + link_cmd, link_proc = link_object( + target_name=target_name, + obj_out=obj_out, + exe_out=exe_out, + ) + if link_proc.returncode != 0: + pytest.fail(format_process_result("Link", link_cmd, link_proc), pytrace=False) + + return run_executable(exe_out, case_root) + + +def run_compile_only( + *, + rel_path: Path, + source_root: Path, + target_config, + zpiler_path: Path, + artifacts_root: Path, + target_name: str, +) -> subprocess.CompletedProcess[bytes]: + source = source_root / rel_path + case_root = artifacts_root / "compile_fail" / target_name / rel_path.with_suffix("") + case_root.mkdir(parents=True, exist_ok=True) + asm_out = case_root / f"{source.stem}{target_config.asm_ext}" + _, compile_proc = compile_source( + source=source, + asm_out=asm_out, + target_config=target_config, + zpiler_path=zpiler_path, + ) + return compile_proc + + +def load_required_substrings(base: Path, target_name: str) -> tuple[list[str], Path]: + contains_path = resolve_existing_expectation(base, target_name, "stderr.contains") + if not contains_path.exists(): + pytest.fail( + f"Missing required compile_fail expectation: {contains_path}", pytrace=False + ) + + lines = decode_text(normalize_line_endings(contains_path.read_bytes())).splitlines() + required = [] + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + required.append(stripped) + + if not required: + pytest.fail( + f"No required substrings found in {contains_path}. " + "Add at least one non-comment line.", + pytrace=False, + ) + return required, contains_path + + +def load_compile_fail_expected_exit(base: Path, target_name: str) -> tuple[int, Path | None]: + exit_path = resolve_existing_expectation(base, target_name, "exitcode") + if not exit_path.exists(): + return 1, None + return parse_exit_code(exit_path), exit_path + + +@pytest.mark.parametrize("rel_path", RUNTIME_CASES, ids=lambda p: p.as_posix()) +def test_runtime( + rel_path: Path, + target_name: str, + target_config, + project_root: Path, + zpiler_path: Path, + artifacts_root: Path, + bless: bool, +) -> None: + proc = build_and_run( + mode="runtime", + rel_path=rel_path, + source_root=project_root / "tests" / "runtime", + target_name=target_name, + target_config=target_config, + zpiler_path=zpiler_path, + artifacts_root=artifacts_root, + ) + + base = expected_base("runtime", rel_path) + + expected_stdout, stdout_path = load_or_bless_output( + base=base, + target_name=target_name, + ext="stdout", + actual=proc.stdout, + bless=bless, + ) + expected_stderr, stderr_path = load_or_bless_output( + base=base, + target_name=target_name, + ext="stderr", + actual=proc.stderr, + bless=bless, + ) + expected_exit, exit_path = load_or_bless_exit_code( + base=base, + target_name=target_name, + actual=proc.returncode, + bless=bless, + ) + + actual_stdout = normalize_line_endings(proc.stdout) + actual_stderr = normalize_line_endings(proc.stderr) + assert_bytes_equal( + label="stdout", + expected=expected_stdout, + actual=actual_stdout, + expected_path=stdout_path, + source=rel_path, + target_name=target_name, + ) + assert_bytes_equal( + label="stderr", + expected=expected_stderr, + actual=actual_stderr, + expected_path=stderr_path, + source=rel_path, + target_name=target_name, + ) + if proc.returncode != expected_exit: + raise AssertionError( + f"{rel_path} on target={target_name} mismatched exit code.\n" + f"Expectation file: {exit_path}\n" + f"Expected: {expected_exit}\nActual: {proc.returncode}" + ) + + +@pytest.mark.parametrize("rel_path", RUNTIME_FAIL_CASES, ids=lambda p: p.as_posix()) +def test_runtime_fail( + rel_path: Path, + target_name: str, + target_config, + project_root: Path, + zpiler_path: Path, + artifacts_root: Path, + bless: bool, +) -> None: + proc = build_and_run( + mode="runtime_fail", + rel_path=rel_path, + source_root=project_root / "tests" / "runtime_fail", + target_name=target_name, + target_config=target_config, + zpiler_path=zpiler_path, + artifacts_root=artifacts_root, + ) + + base = expected_base("runtime_fail", rel_path) + + expected_stdout, stdout_path = load_or_bless_output( + base=base, + target_name=target_name, + ext="stdout", + actual=proc.stdout, + bless=bless, + ) + expected_stderr, stderr_path = load_or_bless_output( + base=base, + target_name=target_name, + ext="stderr", + actual=proc.stderr, + bless=bless, + ) + expected_exit, exit_path = load_or_bless_exit_code( + base=base, + target_name=target_name, + actual=proc.returncode, + bless=bless, + ) + + if expected_exit == 0: + pytest.fail( + f"runtime_fail expectation cannot use exit code 0: {exit_path}", + pytrace=False, + ) + + actual_stdout = normalize_line_endings(proc.stdout) + actual_stderr = normalize_line_endings(proc.stderr) + assert_bytes_equal( + label="stdout", + expected=expected_stdout, + actual=actual_stdout, + expected_path=stdout_path, + source=rel_path, + target_name=target_name, + ) + assert_bytes_equal( + label="stderr", + expected=expected_stderr, + actual=actual_stderr, + expected_path=stderr_path, + source=rel_path, + target_name=target_name, + ) + if proc.returncode != expected_exit: + raise AssertionError( + f"{rel_path} on target={target_name} mismatched exit code.\n" + f"Expectation file: {exit_path}\n" + f"Expected: {expected_exit}\nActual: {proc.returncode}" + ) + + +@pytest.mark.parametrize("rel_path", COMPILE_FAIL_CASES, ids=lambda p: p.as_posix()) +def test_compile_fail( + rel_path: Path, + target_name: str, + target_config, + project_root: Path, + zpiler_path: Path, + artifacts_root: Path, +) -> None: + proc = run_compile_only( + rel_path=rel_path, + source_root=project_root / "tests" / "compile_fail", + target_config=target_config, + zpiler_path=zpiler_path, + artifacts_root=artifacts_root, + target_name=target_name, + ) + + base = expected_base("compile_fail", rel_path) + expected_exit, exit_path = load_compile_fail_expected_exit(base, target_name) + if proc.returncode != expected_exit: + exit_source = str(exit_path) if exit_path is not None else "default(1)" + pytest.fail( + "compile_fail exit code mismatch.\n" + f"Source: {rel_path}\n" + f"Target: {target_name}\n" + f"Expected ({exit_source}): {expected_exit}\n" + f"Actual: {proc.returncode}\n" + f"--- stdout ---\n{decode_text(normalize_line_endings(proc.stdout))}\n" + f"--- stderr ---\n{decode_text(normalize_line_endings(proc.stderr))}", + pytrace=False, + ) + + if proc.returncode == 0: + pytest.fail( + "compile_fail test unexpectedly succeeded.\n" + f"Source: {rel_path}\nTarget: {target_name}", + pytrace=False, + ) + + required_substrings, contains_path = load_required_substrings(base, target_name) + stderr_text = decode_text(normalize_line_endings(proc.stderr)) + missing = [needle for needle in required_substrings if needle not in stderr_text] + if missing: + joined = "\n".join(f"- {needle}" for needle in missing) + pytest.fail( + "compile_fail stderr missing required substrings.\n" + f"Source: {rel_path}\n" + f"Target: {target_name}\n" + f"Expectation file: {contains_path}\n" + f"Missing:\n{joined}\n" + f"--- stderr ---\n{stderr_text}", + pytrace=False, + )