Lets see if the new testing framework works

.github/workflows/test.yml

@@ -28,14 +28,17 @@ jobs:
           sudo apt-get update -y
           sudo apt-get install -y build-essential llvm clang cmake python3-pip
 
+      - name: Install Python Dependencies
+        run: python3 -m pip install -r requirements-dev.txt
+
       - name: Build zpiler
         run: |
           mkdir -p build && cd build
           cmake .. -DCMAKE_BUILD_TYPE=Release
           make -j
 
       - name: Run Tests
-        run: python3 test_runner.py
+        run: python3 -m pytest -q
 
   test-windows:
     runs-on: windows-latest
@@ -58,7 +61,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: C:\Users\runneradmin\AppData\Local\pip\Cache
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
@@ -68,6 +71,9 @@ jobs:
       - name: Install CMake
         run: choco install --no-progress -y cmake
 
+      - name: Install Python Dependencies
+        run: python -m pip install -r requirements-dev.txt
+
       - name: Build zpiler
         shell: cmd
         run: |
@@ -80,4 +86,4 @@ jobs:
         shell: cmd
         run: |
           chcp 65001
-          python test_runner.py
+          python -m pytest -q

README.md

@@ -145,18 +145,24 @@ Generates: `program` (executable)
 
 ### Automated Testing
 
-Run all test cases for a specific target:
+Install test dependencies:
 ```bash
-# Test native platform (auto-detected)
-python test_runner.py
+python3 -m pip install -r requirements-dev.txt
+```
+
+Run strict golden-output tests with `pytest`:
+```bash
+# Auto-detect native target when TARGET is not set
+pytest -q
 
-# Test specific targets
-TARGET=linux python test_runner.py      # Linux
-TARGET=windows python test_runner.py    # Windows (if ml64 available)
-TARGET=llvm python test_runner.py       # LLVM
+# Linux + LLVM
+TARGET=linux,llvm pytest -q
 
-# Test multiple targets
-TARGET=linux,llvm python test_runner.py
+# Windows
+TARGET=windows pytest -q
+
+# Update/create runtime goldens explicitly
+TARGET=linux pytest -q --bless
 ```
 
 ---
@@ -195,7 +201,13 @@ fn main() {
 
 ## 🧪 Testing
 
-You can add `.zz` programs inside a `tests/` folder and run them individually through the pipeline. Type errors, parsing errors, and codegen output are printed to the terminal.
+Test suites are directory-based:
+
+- `tests/runtime`: program must compile, run, and match exact `stdout`/`stderr`/exit code
+- `tests/runtime_fail`: program must compile, run, and match exact failing output + nonzero exit code
+- `tests/compile_fail`: program must fail compilation with expected exit code and required stderr substrings
+
+Expected files live under `tests/expected/<mode>/...` with mirrored paths.
 
 ---
 

main.cpp

@@ -130,6 +130,7 @@ int main(int argc, char *argv[]) {
         cg->generate(std::move(program));
     } catch (std::exception const &exc) {
         std::cerr << "ERROR: " << exc.what() << "\n";
+        return 1;
     }
 
     return 0;

requirements-dev.txt

@@ -0,0 +1 @@
+pytest==8.3.3

src/parser/Parser.cpp

@@ -200,7 +200,7 @@ namespace zust
                       errMsg + " at line " + std::to_string(currentToken.line) +
                           ", column " + std::to_string(currentToken.column)});
             shouldTypecheck = false;
-            exit(0);
+            exit(1);
             return;
         }
     }

tests/compile_fail/syntax/missing_semicolon.zz

@@ -0,0 +1,3 @@
+fn main() {
+    let a: int64_t = 10
+}

tests/compile_fail/type/bad_assignment.zz

@@ -0,0 +1,4 @@
+extern fn foo(x: not_a_type) -> none;
+
+fn main() {
+}

tests/conftest.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import pytest
+
+
+@dataclass(frozen=True)
+class TargetConfig:
+    name: str
+    zpiler_format: str
+    asm_ext: str
+
+
+TARGETS: dict[str, TargetConfig] = {
+    "linux": TargetConfig(name="linux", zpiler_format="x86_64-linux", asm_ext=".s"),
+    "windows": TargetConfig(
+        name="windows", zpiler_format="x86_64-mswin", asm_ext=".asm"
+    ),
+    "llvm": TargetConfig(name="llvm", zpiler_format="llvm-ir", asm_ext=".ll"),
+}
+
+
+def detect_native_target() -> str:
+    plat = sys.platform
+    if plat.startswith("linux"):
+        return "linux"
+    if plat.startswith("win32") or plat.startswith("cygwin"):
+        return "windows"
+    return "llvm"
+
+
+def parse_target_env() -> list[str]:
+    env = os.getenv("TARGET")
+    if env:
+        wanted = [t.strip() for t in env.split(",") if t.strip() in TARGETS]
+        if not wanted:
+            raise pytest.UsageError(
+                f"TARGET={env} contains no known backends. "
+                f"Known targets: {', '.join(TARGETS)}"
+            )
+        return wanted
+    return [detect_native_target()]
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--bless",
+        action="store_true",
+        default=False,
+        help="Create or update runtime golden output files.",
+    )
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
+    if "target_name" in metafunc.fixturenames:
+        targets = parse_target_env()
+        metafunc.parametrize(
+            "target_name", targets, ids=[f"target={target}" for target in targets]
+        )
+
+
+@pytest.fixture(scope="session")
+def bless(request: pytest.FixtureRequest) -> bool:
+    return bool(request.config.getoption("--bless"))
+
+
+@pytest.fixture(scope="session")
+def project_root() -> Path:
+    return Path(__file__).resolve().parents[1]
+
+
+@pytest.fixture(scope="session")
+def zpiler_path(project_root: Path) -> Path:
+    candidates = [
+        project_root / "build" / "zpiler",
+        project_root / "build" / "Debug" / "zpiler.exe",
+        project_root / "build" / "zpiler.exe",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+
+    candidate_text = "\n".join(str(path) for path in candidates)
+    pytest.exit(
+        "Could not find zpiler executable. Build project first.\n"
+        f"Searched:\n{candidate_text}",
+        returncode=2,
+    )
+
+
+@pytest.fixture(scope="session")
+def artifacts_root(tmp_path_factory: pytest.TempPathFactory) -> Path:
+    return tmp_path_factory.mktemp("pipeline_artifacts")
+
+
+@pytest.fixture
+def target_config(target_name: str) -> TargetConfig:
+    return TARGETS[target_name]

tests/expected/compile_fail/syntax/missing_semicolon.stderr.contains

@@ -0,0 +1,3 @@
+# Required syntax-failure markers
+[Syntax Error]
+Expected ';' after declaration

tests/expected/compile_fail/type/bad_assignment.exitcode

@@ -0,0 +1 @@
+1

tests/expected/compile_fail/type/bad_assignment.stderr.contains

@@ -0,0 +1,3 @@
+# Required semantic/type-failure markers
+[Type Error]
+Undefined type 'not_a_type'

tests/expected/runtime/conditionals/if-elif-else.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/conditionals/if-elif-else.stdout

@@ -0,0 +1 @@
+11

tests/expected/runtime/conditionals/if-elif.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/conditionals/if-else.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/conditionals/if.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/functions/basics.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/functions/basics.stdout

@@ -0,0 +1,3 @@
+42
+9.000000
+true

tests/expected/runtime/functions/recursive.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/functions/recursive.stdout

@@ -0,0 +1,3 @@
+120
+true
+false

tests/expected/runtime/functions/scopes.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/functions/scopes.stdout

@@ -0,0 +1 @@
+22

tests/expected/runtime/loops/control.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/operations/binary.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/operations/binary.stdout

@@ -0,0 +1,24 @@
+30
+-10
+200
+2
+12.500000
+-7.500000
+25.000000
+4.000000
+5.859500
+0.423500
+8.538597
+1.155813
+3
+true
+true
+false
+false
+false
+true
+false
+false
+false
+false
+true

tests/expected/runtime/operations/unary.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/operations/unary.stdout

@@ -0,0 +1,5 @@
+5
+255
+100
+0
+1

tests/expected/runtime/strings/escapes.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/types.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/types.stdout

@@ -0,0 +1 @@
+0

tests/expected/runtime/variables.exitcode

@@ -0,0 +1 @@
+0

tests/expected/runtime/variables.linux.stdout

@@ -0,0 +1 @@
+It is what it is

tests/expected/runtime/variables.llvm.stdout

@@ -0,0 +1 @@
+It is what it is

tests/expected/runtime/variables.stdout

@@ -0,0 +1 @@
+DO_NOT_USE_SHARED_VARIABLES_STDOUT

tests/expected/runtime/variables.windows.stdout

@@ -0,0 +1 @@
+It is what it is

tests/expected/runtime_fail/exit/nonzero.exitcode

@@ -0,0 +1 @@
+7

tests/expected/runtime_fail/exit/nonzero.stdout

@@ -0,0 +1 @@
+about-to-fail

tests/runtime_fail/exit/nonzero.zz

@@ -0,0 +1,7 @@
+extern fn printf(fmt: string, ...) -> int32_t;
+extern fn exit(code: int32_t) -> none;
+
+fn main() {
+    printf("about-to-fail\n");
+    exit(7);
+}