Test coverage for manifest/baseline and CI hardening

.github/workflows/ci.yml

@@ -9,10 +9,15 @@ concurrency:
   group: ci-${{ github.ref }}
   cancel-in-progress: true
 
+# Least privilege: nothing in CI needs to write to the repository.
+permissions:
+  contents: read
+
 jobs:
   static:
     name: lint + types + unit (Python ${{ matrix.python }})
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
@@ -36,6 +41,7 @@ jobs:
   web:
     name: web (types, tests, build)
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     defaults:
       run:
         working-directory: web
@@ -57,6 +63,7 @@ jobs:
   package:
     name: package (wheel installs and works outside the repo)
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v7
       - uses: astral-sh/setup-uv@v7
@@ -91,6 +98,7 @@ jobs:
   integration:
     name: integration (docker)
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     needs: static
     services:
       postgres:
@@ -126,6 +134,7 @@ jobs:
   e2e:
     name: e2e (run, replay, gate)
     runs-on: ubuntu-latest
+    timeout-minutes: 30
     needs: integration
     if: github.ref == 'refs/heads/main' || github.event_name == 'pull_request'
     steps:

src/meridian/grading/graders/json_state.py

@@ -13,7 +13,7 @@
 from pathlib import Path
 from typing import Any
 
-from jsonpath_ng import parse as parse_jsonpath
+from jsonpath_ng.parser import parse as parse_jsonpath
 
 from meridian.grading.graders.base import GraderError, failed, passed, resolve
 from meridian.models.run import AssertionResult

tests/unit/test_gate_baseline.py

@@ -0,0 +1,61 @@
+"""Baseline resolution against a real throwaway git repository."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from meridian.gate.baseline import BaselineError, merge_base, worktree_at
+
+pytestmark = pytest.mark.unit
+
+
+def _git(repo: Path, *args: str) -> str:
+    return subprocess.run(
+        ["git", "-c", "user.name=t", "-c", "user.email=t@example.com", *args],
+        cwd=repo,
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+
+
+@pytest.fixture
+def repo(tmp_path: Path) -> Path:
+    _git(tmp_path, "init", "-q", "-b", "main")
+    (tmp_path / "f.txt").write_text("base")
+    _git(tmp_path, "add", ".")
+    _git(tmp_path, "commit", "-q", "-m", "base")
+    return tmp_path
+
+
+def test_merge_base_finds_the_divergence_point(repo: Path) -> None:
+    base = _git(repo, "rev-parse", "HEAD")
+    _git(repo, "checkout", "-q", "-b", "feature")
+    (repo / "f.txt").write_text("head")
+    _git(repo, "commit", "-q", "-am", "head")
+    assert merge_base(repo, "main") == base
+
+
+def test_merge_base_is_none_for_an_unknown_ref(repo: Path) -> None:
+    assert merge_base(repo, "does-not-exist") is None
+
+
+def test_worktree_checks_out_the_sha_and_cleans_up(repo: Path) -> None:
+    base = _git(repo, "rev-parse", "HEAD")
+    (repo / "f.txt").write_text("changed")
+    _git(repo, "commit", "-q", "-am", "next")
+
+    with worktree_at(repo, base) as tree:
+        assert (tree / "f.txt").read_text() == "base"
+        parent = tree.parent
+    assert not parent.exists()
+    assert (repo / "f.txt").read_text() == "changed"
+    assert "meridian-baseline" not in _git(repo, "worktree", "list")
+
+
+def test_worktree_on_a_missing_sha_is_a_harness_error(repo: Path) -> None:
+    with pytest.raises(BaselineError, match="fetch-depth"), worktree_at(repo, "0" * 40):
+        pass

tests/unit/test_manifest_build.py

@@ -0,0 +1,109 @@
+"""Manifest assembly: the reproducibility contract depends on these hashes."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from meridian.config import MeridianConfig
+from meridian.manifest.build import (
+    build_manifest,
+    directory_hash,
+    environment_facts,
+    file_hash,
+    git_sha,
+)
+from meridian.suites.loader import load_suite
+from tests.conftest import CHECKOUT_SUITE
+
+pytestmark = pytest.mark.unit
+
+
+def test_directory_hash_is_stable_across_calls(tmp_path: Path) -> None:
+    (tmp_path / "a.txt").write_text("one")
+    assert directory_hash(tmp_path) == directory_hash(tmp_path)
+    assert directory_hash(tmp_path).startswith("sha256:")
+
+
+def test_directory_hash_changes_when_content_changes(tmp_path: Path) -> None:
+    (tmp_path / "prompt.md").write_text("be helpful")
+    before = directory_hash(tmp_path)
+    (tmp_path / "prompt.md").write_text("be terse")
+    assert directory_hash(tmp_path) != before
+
+
+def test_directory_hash_changes_when_a_file_is_renamed(tmp_path: Path) -> None:
+    (tmp_path / "a.md").write_text("same")
+    before = directory_hash(tmp_path)
+    (tmp_path / "a.md").rename(tmp_path / "b.md")
+    assert directory_hash(tmp_path) != before
+
+
+def test_directory_hash_ignores_pycache(tmp_path: Path) -> None:
+    (tmp_path / "a.py").write_text("x = 1")
+    before = directory_hash(tmp_path)
+    cache = tmp_path / "__pycache__"
+    cache.mkdir()
+    (cache / "a.cpython-312.pyc").write_bytes(b"\x00junk")
+    assert directory_hash(tmp_path) == before
+
+
+def test_file_hash_matches_known_sha256(tmp_path: Path) -> None:
+    path = tmp_path / "f"
+    path.write_bytes(b"abc")
+    assert file_hash(path) == (
+        "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
+    )
+
+
+def test_environment_facts_includes_lock_hash_only_when_lock_exists(tmp_path: Path) -> None:
+    assert "uv_lock" not in environment_facts(tmp_path)
+    (tmp_path / "uv.lock").write_text("lock")
+    assert environment_facts(tmp_path)["uv_lock"] == file_hash(tmp_path / "uv.lock")
+    assert "uv_lock" not in environment_facts(None)
+
+
+def test_git_sha_is_none_outside_a_repository(tmp_path: Path) -> None:
+    assert git_sha(tmp_path) is None
+
+
+def test_build_manifest_is_deterministic_and_covers_every_trial() -> None:
+    suite = load_suite(CHECKOUT_SUITE)
+    config = MeridianConfig()
+
+    def build() -> object:
+        return build_manifest(
+            suite=suite, config=config, run_id="run-1", created_unix_ms=1_700_000_000_000
+        )
+
+    first = build_manifest(
+        suite=suite, config=config, run_id="run-1", created_unix_ms=1_700_000_000_000
+    )
+    assert first == build()
+    assert len(first.tasks) == len(suite.tasks)
+    assert len(first.trials) == len(suite.tasks) * config.execution.n_trials
+    assert first.n_trials == config.execution.n_trials
+    assert first.sut_content_hash is None
+
+
+def test_build_manifest_seeds_depend_on_run_id() -> None:
+    suite = load_suite(CHECKOUT_SUITE)
+    config = MeridianConfig()
+    a = build_manifest(suite=suite, config=config, run_id="run-a", created_unix_ms=1)
+    b = build_manifest(suite=suite, config=config, run_id="run-b", created_unix_ms=1)
+    assert [t.seed for t in a.trials] != [t.seed for t in b.trials]
+
+
+def test_build_manifest_records_docker_version_and_sut_hash(tmp_path: Path) -> None:
+    (tmp_path / "agent.py").write_text("print(1)")
+    manifest = build_manifest(
+        suite=load_suite(CHECKOUT_SUITE),
+        config=MeridianConfig(),
+        run_id="r",
+        created_unix_ms=1,
+        sut_root=tmp_path,
+        docker_version="27.0.1",
+    )
+    assert manifest.env["docker"] == "27.0.1"
+    assert manifest.sut_content_hash == directory_hash(tmp_path)