From 161417ff69293122c10fbdb0b47994eefeef6d4e Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 16:02:59 +0100 Subject: [PATCH 01/31] ci(backend): replace shards with exact-custody semantic lanes Co-authored-by: Konan --- .../STATUS.md | 12 +- .github/workflows/backend.yml | 300 +++----- backend/scripts/ci_test_shards.py | 703 ------------------ backend/scripts/run_isolated_tests.py | 246 +++++- backend/scripts/run_test_lanes.py | 624 ++++++++++++++++ .../scripts/validate_test_lane_evidence.py | 377 ++++++++++ backend/tests/test_ci_test_lanes.py | 248 ++++++ backend/tests/test_ci_test_shards.py | 448 ----------- .../tests/test_isolated_database_runner.py | 122 +++ backend/tests/test_test_lane_evidence.py | 296 ++++++++ docs/operations_backend_testing.md | 113 +-- scripts/test_agent_gates.py | 154 ++-- 12 files changed, 2126 insertions(+), 1517 deletions(-) delete mode 100644 backend/scripts/ci_test_shards.py create mode 100644 backend/scripts/run_test_lanes.py create mode 100644 backend/scripts/validate_test_lane_evidence.py create mode 100644 backend/tests/test_ci_test_lanes.py delete mode 100644 backend/tests/test_ci_test_shards.py create mode 100644 backend/tests/test_test_lane_evidence.py diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md index 1b918dee..bff75000 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md @@ -2,13 +2,13 @@ - Phase: signed implementation - Active planning chunk: none -- Active implementation chunk: `WS-CI-001-02A` -- Proposed implementation successor: `WS-CI-001-02B` -- Later proposed chunk: `WS-CI-001-02B` (cannot start before 02A evidence) +- Active implementation chunk: `WS-CI-001-02B` +- Proposed implementation successor: none +- Later proposed chunk: `WS-CI-001-03` planning only; not a declared successor - Human direction: preserve Konan's authorship and measured work from PR #180, but adopt it only under prospective zero-trust scope and evidence -- Current gate: runtime schema recovery reviewed at `cf91bb81`; full Backend suite, - canonical fingerprint, and all coverage gates remain before user-owned merge - decision +- Current gate: exact-custody semantic-lane implementation is in deterministic + proof; hosted full Backend execution, exact node/coverage/resource evidence, + internal review, external review, and user-owned merge decision remain - Deferred option: routing/cache/timing reassessment is future `WS-CI-001-03`, with no start or successor declaration diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 05595ae4..6a1c46fb 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -13,11 +13,9 @@ env: MINIO_IMAGE: quay.io/minio/minio:latest@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e jobs: - preflight: + test: runs-on: ubuntu-latest - timeout-minutes: 30 - outputs: - tree_sha: ${{ steps.identity.outputs.tree_sha }} + timeout-minutes: 45 services: postgres: @@ -45,24 +43,27 @@ jobs: python-version: "3.12" - id: identity - name: Bind checked-out tree + name: Bind exact checked-out tree shell: bash run: | set -euo pipefail - tree_sha=$(git rev-parse HEAD) - if [[ ! "${tree_sha}" =~ ^[0-9a-f]{40}$ ]]; then - exit 1 - fi + tree_sha="$(git rev-parse HEAD)" + test "${tree_sha}" = "${GITHUB_SHA}" + test -z "$(git status --porcelain)" + test "$(git rev-parse "${tree_sha}^{tree}")" = "$(git rev-parse HEAD^{tree})" echo "tree_sha=${tree_sha}" >> "${GITHUB_OUTPUT}" - name: Internal review evidence gate run: python3 scripts/check_internal_review_evidence.py - - name: Install backend + - name: Install backend and exact Ruff working-directory: backend run: | + set -euo pipefail python -m pip install --upgrade pip python -m pip install -e ".[dev]" + python -m pip install ruff==0.15.22 + test "$(ruff --version)" = "ruff 0.15.22" - name: Lint working-directory: backend @@ -72,77 +73,6 @@ jobs: working-directory: backend run: docstr-coverage --config .docstr.yaml - - name: Isolated database runner test - working-directory: backend - env: - WORKSTREAM_TEST_ADMIN_DATABASE_URL: postgresql+asyncpg://workstream:workstream@localhost:5433/postgres - run: python -m pytest -q tests/test_isolated_database_runner.py - - - name: Collect and plan exact test inventory - shell: bash - run: | - set -euo pipefail - mkdir -p .ci/plan - python3 backend/scripts/ci_test_shards.py plan \ - --repository-root . \ - --tree-sha "${{ steps.identity.outputs.tree_sha }}" \ - --shards 4 \ - --output .ci/plan/shard-manifest.json - - - name: Upload immutable shard plan - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: backend-shard-plan-${{ steps.identity.outputs.tree_sha }} - path: .ci/plan/shard-manifest.json - if-no-files-found: error - retention-days: 7 - - shards: - needs: preflight - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4] - - services: - postgres: - image: public.ecr.aws/docker/library/postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 - env: - POSTGRES_DB: workstream_test - POSTGRES_USER: workstream - POSTGRES_PASSWORD: workstream - ports: - - 5433:5432 - options: >- - --health-cmd "pg_isready -U workstream -d workstream_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 - with: - python-version: "3.12" - - - name: Install backend - working-directory: backend - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev]" - - - name: Download exact shard plan - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - name: backend-shard-plan-${{ needs.preflight.outputs.tree_sha }} - path: .ci/plan - - name: Start real MinIO artifact provider shell: bash run: | @@ -161,65 +91,71 @@ jobs: docker logs workstream-minio exit 1 - - name: Run isolated shard ${{ matrix.shard }} + - name: Collect canonical semantic-lane inventory + working-directory: backend env: - WORKSTREAM_TEST_ADMIN_DATABASE_URL: postgresql+asyncpg://workstream:workstream@localhost:5433/postgres - WORKSTREAM_TEST_MINIO_ENDPOINT: http://127.0.0.1:9000 + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" run: >- - python3 backend/scripts/ci_test_shards.py run-shard - --repository-root . - --manifest .ci/plan/shard-manifest.json - --shard ${{ matrix.shard }} - --bundle-dir .ci/shard-bundle - --database-metadata "${RUNNER_TEMP}/database-${{ matrix.shard }}.json" - - - name: Upload authenticated shard ${{ matrix.shard }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: backend-coverage-${{ needs.preflight.outputs.tree_sha }}-${{ matrix.shard }} - path: .ci/shard-bundle - if-no-files-found: error - retention-days: 7 + python scripts/run_test_lanes.py + --collect-only + --metadata-dir .ci/test-lanes/collect + --summary-json .ci/test-lanes/collect-summary.json - api_e2e: - needs: preflight - runs-on: ubuntu-latest - timeout-minutes: 30 + - name: Validate canonical semantic-lane inventory + working-directory: backend + run: >- + python scripts/validate_test_lane_evidence.py + --metadata-dir .ci/test-lanes/collect + --summary-json .ci/test-lanes/collect-summary.json - services: - postgres: - image: public.ecr.aws/docker/library/postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 + - name: Execute four semantic lanes + working-directory: backend env: - POSTGRES_DB: workstream_test - POSTGRES_USER: workstream - POSTGRES_PASSWORD: workstream - ports: - - 5433:5432 - options: >- - --health-cmd "pg_isready -U workstream -d workstream_test" - --health-interval 5s - --health-timeout 5s - --health-retries 10 - - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 - with: - python-version: "3.12" + COVERAGE_FILE: .coverage + PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" + WORKSTREAM_TEST_ADMIN_DATABASE_URL: postgresql+asyncpg://workstream:workstream@localhost:5433/postgres + WORKSTREAM_TEST_MINIO_ENDPOINT: http://127.0.0.1:9000 + shell: bash + run: | + set -uo pipefail + rm -f .coverage .coverage.* + lane_exit=0 + python scripts/run_test_lanes.py \ + --metadata-dir .ci/test-lanes/run \ + --summary-json .ci/test-lanes/run-summary.json \ + --timeout-seconds 1200 || lane_exit=$? + printf '%s\n' "${lane_exit}" > .ci/test-lanes/run.exit + + - name: Independently validate completed-node and coverage custody + working-directory: backend + run: >- + python scripts/validate_test_lane_evidence.py + --metadata-dir .ci/test-lanes/run + --summary-json .ci/test-lanes/run-summary.json - - name: Verify checked-out tree + - name: Require semantic-lane execution success shell: bash - run: test "$(git rev-parse HEAD)" = "${{ needs.preflight.outputs.tree_sha }}" + working-directory: backend + run: test "$(cat .ci/test-lanes/run.exit)" = 0 - - name: Install backend + - name: Combine semantic-lane coverage exactly once working-directory: backend + shell: bash run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev]" + set -euo pipefail + shopt -s nullglob + coverage_files=(.ci/test-lanes/run/.coverage.*) + test "${#coverage_files[@]}" -eq 4 + for source in "${coverage_files[@]}"; do + test -f "${source}" + test ! -L "${source}" + destination="$(basename "${source}")" + test ! -e "${destination}" + cp -- "${source}" "${destination}" + test "$(sha256sum "${source}" | cut -d' ' -f1)" = \ + "$(sha256sum "${destination}" | cut -d' ' -f1)" + done + coverage combine - name: API contract real API e2e working-directory: backend @@ -231,93 +167,6 @@ jobs: --timeout-seconds 1500 -- python scripts/api_contract_e2e.py - test: - if: ${{ always() }} - needs: [preflight, shards, api_e2e] - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Require every upstream proof - env: - PREFLIGHT_RESULT: ${{ needs.preflight.result }} - SHARDS_RESULT: ${{ needs.shards.result }} - API_E2E_RESULT: ${{ needs.api_e2e.result }} - shell: bash - run: | - set -euo pipefail - test "${PREFLIGHT_RESULT}" = success - test "${SHARDS_RESULT}" = success - test "${API_E2E_RESULT}" = success - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 - with: - python-version: "3.12" - - - name: Install backend - working-directory: backend - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev]" - - - name: Download exact shard plan - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - name: backend-shard-plan-${{ needs.preflight.outputs.tree_sha }} - path: .ci/plan - - - name: Download shard 1 - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - name: backend-coverage-${{ needs.preflight.outputs.tree_sha }}-1 - path: .ci/bundles/shard-1 - - - name: Download shard 2 - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - name: backend-coverage-${{ needs.preflight.outputs.tree_sha }}-2 - path: .ci/bundles/shard-2 - - - name: Download shard 3 - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - name: backend-coverage-${{ needs.preflight.outputs.tree_sha }}-3 - path: .ci/bundles/shard-3 - - - name: Download shard 4 - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 - with: - name: backend-coverage-${{ needs.preflight.outputs.tree_sha }}-4 - path: .ci/bundles/shard-4 - - - name: Validate exact fan-in and combine coverage - shell: bash - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "${{ needs.preflight.outputs.tree_sha }}" - python3 backend/scripts/ci_test_shards.py fan-in \ - --manifest .ci/plan/shard-manifest.json \ - --tree-sha "${{ needs.preflight.outputs.tree_sha }}" \ - --bundles-root .ci/bundles \ - --output-dir .ci/combined-coverage \ - --summary-output .ci/fan-in-summary.json - python3 -m json.tool .ci/fan-in-summary.json - cd backend - coverage combine ../.ci/combined-coverage - - - name: Upload authenticated timing and balance report - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: backend-fan-in-${{ needs.preflight.outputs.tree_sha }} - path: .ci/fan-in-summary.json - if-no-files-found: error - retention-days: 7 - - name: Backend full-suite coverage working-directory: backend run: coverage report --precision=2 --fail-under=78 @@ -389,3 +238,20 @@ jobs: --include='app/interfaces/auth.py,app/core/auth.py,app/adapters/auth/dev.py,app/adapters/auth/flow.py' --precision=2 --fail-under=90 + + - name: Reassert exact tree custody + if: ${{ always() }} + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${{ steps.identity.outputs.tree_sha }}" + test "$(git rev-parse HEAD^{tree})" = "$(git rev-parse "${{ steps.identity.outputs.tree_sha }}^{tree}")" + + - name: Upload semantic-lane evidence and timing + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: backend-semantic-lane-evidence-${{ steps.identity.outputs.tree_sha }} + path: backend/.ci/test-lanes/** + if-no-files-found: warn + retention-days: 7 diff --git a/backend/scripts/ci_test_shards.py b/backend/scripts/ci_test_shards.py deleted file mode 100644 index 601f5c61..00000000 --- a/backend/scripts/ci_test_shards.py +++ /dev/null @@ -1,703 +0,0 @@ -#!/usr/bin/env python3 -"""Plan, execute, and validate isolated backend CI test shards.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path, PurePosixPath -import re -import shutil -import subprocess -import sys -import tempfile -import time -from typing import Any - -SCHEMA_VERSION = 2 -SHARD_COUNT = 4 -EXCLUDED_MODULE = "backend/tests/test_isolated_database_runner.py" -MODULE_RE = re.compile(r"backend/tests/test_[a-z0-9_]+\.py") -TREE_RE = re.compile(r"[0-9a-f]{40}") -BUNDLE_FILES = {"coverage.data", "result.json"} -COLLECTED_NODES_ENV = "WORKSTREAM_CI_COLLECTED_NODES" -COMPLETED_NODES_ENV = "WORKSTREAM_CI_COMPLETED_NODES" -PYTEST_PLUGINS = ( - "-p", - "pytest_asyncio.plugin", - "-p", - "pytest_cov.plugin", - "-p", - "scripts.ci_test_shards", -) - - -class ShardError(RuntimeError): - """A stable CI shard planning or evidence failure.""" - - -def _append_node(destination: str, node_id: str) -> None: - data = (json.dumps(node_id) + "\n").encode() - flags = os.O_WRONLY | os.O_APPEND - if hasattr(os, "O_NOFOLLOW"): - flags |= os.O_NOFOLLOW - descriptor = os.open(destination, flags) - try: - os.write(descriptor, data) - finally: - os.close(descriptor) - - -def pytest_collection_finish(session: Any) -> None: - """Record the final selected inventory from the executing pytest process.""" - destination = os.environ.get(COLLECTED_NODES_ENV) - if not destination: - return - for item in session.items: - _append_node(destination, item.nodeid) - - -def pytest_runtest_logfinish(nodeid: str, location: tuple[str, int | None, str]) -> None: - """Record a node only after its actual pytest lifecycle has finished.""" - del location - destination = os.environ.get(COMPLETED_NODES_ENV) - if not destination: - return - _append_node(destination, nodeid) - - -def _json_bytes(value: Any) -> bytes: - return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() - - -def _sha256(data: bytes) -> str: - return hashlib.sha256(data).hexdigest() - - -def _regular_file(path: Path) -> bool: - return not path.is_symlink() and path.is_file() - - -def _tree_sha(repository_root: Path) -> str: - result = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=repository_root, - check=True, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ).stdout.strip() - if TREE_RE.fullmatch(result) is None: - raise ShardError("invalid_tree_sha") - return result - - -def _validate_tree_sha(value: str) -> str: - if TREE_RE.fullmatch(value) is None: - raise ShardError("invalid_tree_sha") - return value - - -def discover_modules(repository_root: Path) -> list[str]: - """Return the canonical symlink-free backend test module inventory.""" - tests_root = repository_root / "backend/tests" - if tests_root.is_symlink() or not tests_root.is_dir(): - raise ShardError("invalid_tests_root") - modules: list[str] = [] - for path in sorted(tests_root.glob("test_*.py")): - relative = path.relative_to(repository_root).as_posix() - if MODULE_RE.fullmatch(relative) is None or not _regular_file(path): - raise ShardError("invalid_test_module") - if relative != EXCLUDED_MODULE: - modules.append(relative) - if not modules or len(modules) != len(set(modules)): - raise ShardError("invalid_module_inventory") - return modules - - -def _module_from_node(node_id: str) -> str: - if "::" not in node_id: - raise ShardError("invalid_node_id") - module_part = node_id.split("::", 1)[0] - candidate = f"backend/{module_part}" - if MODULE_RE.fullmatch(candidate) is None: - raise ShardError("invalid_node_module") - return candidate - - -def _node_base(node_id: str) -> str: - """Remove only pytest's final parameter display value from a node ID.""" - _module_from_node(node_id) - if node_id.endswith("]") and "[" in node_id: - return node_id.split("[", 1)[0] - return node_id - - -def _node_signature(nodes: list[str]) -> list[dict[str, Any]]: - counts: dict[str, int] = {} - for node in nodes: - base = _node_base(node) - counts[base] = counts.get(base, 0) + 1 - return [{"base": base, "count": counts[base]} for base in sorted(counts)] - - -def _read_node_log(path: Path) -> list[str]: - try: - nodes = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ShardError("invalid_runtime_nodes") from exc - if not nodes or not all(isinstance(node, str) for node in nodes): - raise ShardError("invalid_runtime_nodes") - return sorted(nodes) - - -def collect_nodes(repository_root: Path, modules: list[str]) -> dict[str, list[str]]: - """Collect canonical pytest node IDs for exactly the supplied modules.""" - backend_root = repository_root / "backend" - relative_modules = [str(PurePosixPath(path).relative_to("backend")) for path in modules] - collection_env = os.environ.copy() - collection_env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" - result = subprocess.run( - [ - sys.executable, - "-m", - "pytest", - "--collect-only", - "-q", - *PYTEST_PLUGINS, - *relative_modules, - ], - cwd=backend_root, - env=collection_env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=False, - ) - if result.returncode != 0: - raise ShardError("pytest_collection_failed") - expected = set(modules) - collected: dict[str, list[str]] = {module: [] for module in modules} - seen: set[str] = set() - for raw_line in result.stdout.splitlines(): - line = raw_line.strip() - if not line or "::" not in line: - continue - module = _module_from_node(line) - if module not in expected or line in seen: - raise ShardError("foreign_or_duplicate_node") - collected[module].append(line) - seen.add(line) - if not seen or any(not nodes for nodes in collected.values()): - raise ShardError("zero_test_module") - return {module: sorted(nodes) for module, nodes in sorted(collected.items())} - - -def _manifest_body( - tree_sha: str, module_rows: list[dict[str, Any]], shard_count: int -) -> dict[str, Any]: - _validate_tree_sha(tree_sha) - if shard_count != SHARD_COUNT: - raise ShardError("invalid_shard_count") - ordered_modules = sorted(row["path"] for row in module_rows) - if not ordered_modules or len(ordered_modules) != len(set(ordered_modules)): - raise ShardError("invalid_module_inventory") - rows_by_module = {row["path"]: row for row in module_rows} - if len(rows_by_module) != len(module_rows): - raise ShardError("invalid_module_inventory") - for module, row in rows_by_module.items(): - if MODULE_RE.fullmatch(module) is None or module == EXCLUDED_MODULE: - raise ShardError("invalid_test_module") - signature, weight = row.get("node_signature"), row.get("weight") - if not isinstance(signature, list) or not signature or not isinstance(weight, int): - raise ShardError("invalid_node_inventory") - if any( - not isinstance(item, dict) - or set(item) != {"base", "count"} - or not isinstance(item["base"], str) - or not isinstance(item["count"], int) - or isinstance(item["count"], bool) - or item["count"] < 1 - or _module_from_node(item["base"]) != module - for item in signature - ): - raise ShardError("node_module_mismatch") - if signature != sorted(signature, key=lambda item: item["base"]): - raise ShardError("invalid_node_inventory") - if len({item["base"] for item in signature}) != len(signature) or weight != sum( - item["count"] for item in signature - ): - raise ShardError("invalid_node_inventory") - - bins: list[dict[str, Any]] = [ - {"id": shard_id, "modules": [], "weight": 0} for shard_id in range(1, shard_count + 1) - ] - weighted = sorted( - ((rows_by_module[module]["weight"], module) for module in ordered_modules), - key=lambda item: (-item[0], item[1]), - ) - for weight, module in weighted: - target = min(bins, key=lambda item: (item["weight"], item["id"])) - target["modules"].append(module) - target["weight"] += weight - for shard in bins: - shard["modules"].sort() - if not shard["modules"]: - raise ShardError("empty_shard") - - canonical_rows = [rows_by_module[module] for module in ordered_modules] - return { - "excluded_modules": [EXCLUDED_MODULE], - "modules": canonical_rows, - "schema_version": SCHEMA_VERSION, - "shard_count": shard_count, - "shards": bins, - "tree_sha": tree_sha, - } - - -def build_manifest( - tree_sha: str, modules_to_nodes: dict[str, list[str]], shard_count: int -) -> dict[str, Any]: - """Build a canonical manifest with a digest over its executable body.""" - for module, nodes in modules_to_nodes.items(): - if MODULE_RE.fullmatch(module) is None or module == EXCLUDED_MODULE: - raise ShardError("invalid_test_module") - if not nodes or nodes != sorted(set(nodes)): - raise ShardError("invalid_node_inventory") - if any(_module_from_node(node) != module for node in nodes): - raise ShardError("node_module_mismatch") - rows = [ - {"node_signature": _node_signature(nodes), "path": module, "weight": len(nodes)} - for module, nodes in sorted(modules_to_nodes.items()) - ] - body = _manifest_body(tree_sha, rows, shard_count) - return {**body, "manifest_sha256": _sha256(_json_bytes(body))} - - -def validate_manifest(manifest: dict[str, Any]) -> dict[str, Any]: - """Validate and canonically reproduce one shard manifest.""" - required = { - "excluded_modules", - "manifest_sha256", - "modules", - "schema_version", - "shard_count", - "shards", - "tree_sha", - } - if not isinstance(manifest, dict) or set(manifest) != required: - raise ShardError("invalid_manifest") - digest = manifest.get("manifest_sha256") - modules = manifest.get("modules") - shard_count = manifest.get("shard_count") - if ( - manifest.get("schema_version") != SCHEMA_VERSION - or manifest.get("excluded_modules") != [EXCLUDED_MODULE] - or not isinstance(digest, str) - or re.fullmatch(r"[0-9a-f]{64}", digest) is None - or not isinstance(modules, list) - or not isinstance(shard_count, int) - or isinstance(shard_count, bool) - or not isinstance(manifest.get("shards"), list) - ): - raise ShardError("invalid_manifest") - rows: list[dict[str, Any]] = [] - for row in modules: - if not isinstance(row, dict) or set(row) != {"node_signature", "path", "weight"}: - raise ShardError("invalid_manifest") - if not isinstance(row["path"], str): - raise ShardError("invalid_manifest") - rows.append(row) - body = _manifest_body(str(manifest.get("tree_sha", "")), rows, shard_count) - reproduced = {**body, "manifest_sha256": _sha256(_json_bytes(body))} - if manifest != reproduced or digest != reproduced["manifest_sha256"]: - raise ShardError("manifest_digest_mismatch") - return reproduced - - -def load_manifest(path: Path) -> dict[str, Any]: - if not _regular_file(path): - raise ShardError("invalid_manifest_file") - try: - value = json.loads(path.read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ShardError("invalid_manifest_file") from exc - return validate_manifest(value) - - -def _shard(manifest: dict[str, Any], shard_id: int) -> dict[str, Any]: - matches = [row for row in manifest["shards"] if row["id"] == shard_id] - if len(matches) != 1: - raise ShardError("invalid_shard_id") - return matches[0] - - -def _assert_checked_out_tree(repository_root: Path, expected: str) -> None: - if _tree_sha(repository_root) != _validate_tree_sha(expected): - raise ShardError("checked_out_tree_mismatch") - - -def _safe_empty_directory(path: Path) -> None: - if path.is_symlink() or (path.exists() and (not path.is_dir() or any(path.iterdir()))): - raise ShardError("invalid_output_directory") - path.mkdir(parents=True, exist_ok=True) - - -def _write_json(path: Path, value: Any) -> None: - if path.exists() or path.is_symlink(): - raise ShardError("output_exists") - path.write_bytes(_json_bytes(value)) - - -def run_shard( - repository_root: Path, - manifest_path: Path, - shard_id: int, - bundle_dir: Path, - database_metadata: Path, -) -> None: - """Run one manifest shard and emit a fixed, non-secret evidence bundle.""" - manifest = load_manifest(manifest_path) - _assert_checked_out_tree(repository_root, manifest["tree_sha"]) - shard = _shard(manifest, shard_id) - expected_rows = [row for row in manifest["modules"] if row["path"] in shard["modules"]] - - _safe_empty_directory(bundle_dir) - coverage_path = bundle_dir / "coverage.data" - collected_path = database_metadata.with_name(f"collected-nodes-{shard_id}.jsonl") - completed_path = database_metadata.with_name(f"completed-nodes-{shard_id}.jsonl") - for path in (collected_path, completed_path): - if path.exists() or path.is_symlink(): - raise ShardError("runtime_nodes_path_exists") - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - os.close(descriptor) - env = os.environ.copy() - env["COVERAGE_FILE"] = str(coverage_path.resolve()) - env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" - env[COLLECTED_NODES_ENV] = str(collected_path.resolve()) - env[COMPLETED_NODES_ENV] = str(completed_path.resolve()) - started = time.monotonic() - command = [ - sys.executable, - "scripts/run_isolated_tests.py", - "--metadata-json", - str(database_metadata), - "--timeout-seconds", - "4800", - "--", - sys.executable, - "-m", - "pytest", - "-q", - *PYTEST_PLUGINS, - *(str(PurePosixPath(module).relative_to("backend")) for module in shard["modules"]), - "--cov=app", - "--cov-report=", - ] - result = subprocess.run(command, cwd=repository_root / "backend", env=env, check=False) - if result.returncode != 0: - raise ShardError("shard_tests_failed") - try: - collected_nodes = _read_node_log(collected_path) - completed_nodes = _read_node_log(completed_path) - except ShardError: - raise - if collected_nodes != completed_nodes or len(collected_nodes) != len(set(collected_nodes)): - raise ShardError("shard_execution_mismatch") - runtime_by_module = {module: [] for module in shard["modules"]} - for node in collected_nodes: - module = _module_from_node(node) - if module not in runtime_by_module: - raise ShardError("shard_execution_mismatch") - runtime_by_module[module].append(node) - runtime_signatures = [ - { - "node_signature": _node_signature(runtime_by_module[row["path"]]), - "path": row["path"], - "weight": len(runtime_by_module[row["path"]]), - } - for row in expected_rows - ] - if runtime_signatures != expected_rows: - raise ShardError("shard_signature_mismatch") - if not _regular_file(coverage_path): - raise ShardError("missing_coverage") - coverage_digest = _sha256(coverage_path.read_bytes()) - metadata = { - "coverage_file": "coverage.data", - "coverage_sha256": coverage_digest, - "duration_seconds": round(time.monotonic() - started, 3), - "manifest_sha256": manifest["manifest_sha256"], - "modules": shard["modules"], - "collected_node_ids": collected_nodes, - "completed_node_ids": completed_nodes, - "schema_version": SCHEMA_VERSION, - "shard_id": shard_id, - "tree_sha": manifest["tree_sha"], - } - _write_json(bundle_dir / "result.json", metadata) - - -def _safe_bundle_files(bundle: Path) -> dict[str, Path]: - if bundle.is_symlink() or not bundle.is_dir(): - raise ShardError("invalid_bundle_directory") - found: dict[str, Path] = {} - for path in bundle.iterdir(): - if path.name not in BUNDLE_FILES or not _regular_file(path): - raise ShardError("unexpected_bundle_path") - found[path.name] = path - if set(found) != BUNDLE_FILES: - raise ShardError("incomplete_bundle") - return found - - -def validate_fan_in( - manifest: dict[str, Any], - bundles_root: Path, - output_dir: Path, - summary_output: Path | None = None, -) -> list[Path]: - """Validate the exact shard set and copy authenticated coverage for combine.""" - if bundles_root.is_symlink() or not bundles_root.is_dir(): - raise ShardError("invalid_bundles_root") - expected_dirs = {f"shard-{index}" for index in range(1, SHARD_COUNT + 1)} - entries = {path.name for path in bundles_root.iterdir()} - if entries != expected_dirs: - raise ShardError("unexpected_bundle_set") - - expected_node_count = sum(module["weight"] for module in manifest["modules"]) - observed_modules: list[str] = [] - coverage_sources: list[Path] = [] - durations: list[float] = [] - for shard_id in range(1, SHARD_COUNT + 1): - files = _safe_bundle_files(bundles_root / f"shard-{shard_id}") - try: - result = json.loads(files["result.json"].read_text(encoding="utf-8")) - except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: - raise ShardError("invalid_result_metadata") from exc - required = { - "coverage_file", - "coverage_sha256", - "duration_seconds", - "manifest_sha256", - "modules", - "collected_node_ids", - "completed_node_ids", - "schema_version", - "shard_id", - "tree_sha", - } - if not isinstance(result, dict) or set(result) != required: - raise ShardError("invalid_result_metadata") - expected_shard = _shard(manifest, shard_id) - if ( - result["schema_version"] != SCHEMA_VERSION - or result["shard_id"] != shard_id - or result["tree_sha"] != manifest["tree_sha"] - or result["manifest_sha256"] != manifest["manifest_sha256"] - or result["modules"] != expected_shard["modules"] - or result["coverage_file"] != "coverage.data" - or not isinstance(result["duration_seconds"], (int, float)) - or isinstance(result["duration_seconds"], bool) - or result["duration_seconds"] < 0 - ): - raise ShardError("result_provenance_mismatch") - collected = result["collected_node_ids"] - completed = result["completed_node_ids"] - if ( - not isinstance(collected, list) - or not all(isinstance(node, str) for node in collected) - or collected != completed - or collected != sorted(set(collected)) - ): - raise ShardError("shard_node_mismatch") - runtime_by_module = {module: [] for module in expected_shard["modules"]} - for node in collected: - module = _module_from_node(node) - if module not in runtime_by_module: - raise ShardError("shard_node_mismatch") - runtime_by_module[module].append(node) - expected_rows = [ - row for row in manifest["modules"] if row["path"] in expected_shard["modules"] - ] - runtime_rows = [ - { - "node_signature": _node_signature(runtime_by_module[row["path"]]), - "path": row["path"], - "weight": len(runtime_by_module[row["path"]]), - } - for row in expected_rows - ] - if runtime_rows != expected_rows: - raise ShardError("shard_node_mismatch") - coverage = files["coverage.data"] - if result["coverage_sha256"] != _sha256(coverage.read_bytes()): - raise ShardError("coverage_digest_mismatch") - observed_modules.extend(result["modules"]) - coverage_sources.append(coverage) - durations.append(float(result["duration_seconds"])) - expected_modules = sorted(module["path"] for module in manifest["modules"]) - if sorted(observed_modules) != expected_modules or len(observed_modules) != len( - set(observed_modules) - ): - raise ShardError("fan_in_module_mismatch") - - _safe_empty_directory(output_dir) - outputs: list[Path] = [] - for shard_id, source in enumerate(coverage_sources, 1): - target = output_dir / f".coverage.shard-{shard_id}" - shutil.copyfile(source, target, follow_symlinks=False) - outputs.append(target) - if summary_output is not None: - _write_json( - summary_output, - { - "manifest_sha256": manifest["manifest_sha256"], - "module_count": len(expected_modules), - "node_count": expected_node_count, - "schema_version": SCHEMA_VERSION, - "timing": { - "imbalance_seconds": round(max(durations) - min(durations), 3), - "maximum_seconds": max(durations), - "total_runner_seconds": round(sum(durations), 3), - }, - "shards": [ - { - "duration_seconds": durations[index - 1], - "id": index, - "module_count": len(_shard(manifest, index)["modules"]), - "node_count": _shard(manifest, index)["weight"], - } - for index in range(1, SHARD_COUNT + 1) - ], - "tree_sha": manifest["tree_sha"], - }, - ) - return outputs - - -def _plan(repository_root: Path, tree_sha: str, shard_count: int) -> dict[str, Any]: - modules = discover_modules(repository_root) - return build_manifest(tree_sha, collect_nodes(repository_root, modules), shard_count) - - -def _dry_run(repository_root: Path, shard_count: int) -> None: - manifest = _plan(repository_root, _tree_sha(repository_root), shard_count) - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - bundles = root / "bundles" - bundles.mkdir() - for shard in manifest["shards"]: - bundle = bundles / f"shard-{shard['id']}" - bundle.mkdir() - coverage = bundle / "coverage.data" - coverage.write_bytes(f"dry-run-{shard['id']}".encode()) - nodes = [ - f"{item['base']}[{index}]" if item["count"] > 1 else item["base"] - for module in manifest["modules"] - if module["path"] in shard["modules"] - for item in module["node_signature"] - for index in range(item["count"]) - ] - nodes.sort() - _write_json( - bundle / "result.json", - { - "coverage_file": "coverage.data", - "coverage_sha256": _sha256(coverage.read_bytes()), - "duration_seconds": 0, - "manifest_sha256": manifest["manifest_sha256"], - "modules": shard["modules"], - "collected_node_ids": nodes, - "completed_node_ids": nodes, - "schema_version": SCHEMA_VERSION, - "shard_id": shard["id"], - "tree_sha": manifest["tree_sha"], - }, - ) - validate_fan_in(manifest, bundles, root / "combined") - print( - json.dumps( - { - "manifest_sha256": manifest["manifest_sha256"], - "modules": len(manifest["modules"]), - "nodes": sum(row["weight"] for row in manifest["modules"]), - "shard_weights": [row["weight"] for row in manifest["shards"]], - "tree_sha": manifest["tree_sha"], - }, - sort_keys=True, - ) - ) - - -def main() -> int: - """Run the requested shard lifecycle command.""" - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(dest="command", required=True) - - plan = subparsers.add_parser("plan") - plan.add_argument("--repository-root", type=Path, required=True) - plan.add_argument("--tree-sha", required=True) - plan.add_argument("--shards", type=int, default=SHARD_COUNT) - plan.add_argument("--output", type=Path, required=True) - - run = subparsers.add_parser("run-shard") - run.add_argument("--repository-root", type=Path, required=True) - run.add_argument("--manifest", type=Path, required=True) - run.add_argument("--shard", type=int, required=True) - run.add_argument("--bundle-dir", type=Path, required=True) - run.add_argument("--database-metadata", type=Path, required=True) - - fan_in = subparsers.add_parser("fan-in") - fan_in.add_argument("--manifest", type=Path, required=True) - fan_in.add_argument("--tree-sha", required=True) - fan_in.add_argument("--bundles-root", type=Path, required=True) - fan_in.add_argument("--output-dir", type=Path, required=True) - fan_in.add_argument("--summary-output", type=Path, required=True) - - dry_run = subparsers.add_parser("dry-run") - dry_run.add_argument("--repository-root", type=Path, required=True) - dry_run.add_argument("--shards", type=int, default=SHARD_COUNT) - - args = parser.parse_args() - try: - if args.command == "plan": - repository_root = args.repository_root.resolve() - manifest = _plan(repository_root, args.tree_sha, args.shards) - if args.output.exists() or args.output.is_symlink(): - raise ShardError("output_exists") - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_bytes(_json_bytes(manifest)) - print(manifest["manifest_sha256"]) - elif args.command == "run-shard": - run_shard( - args.repository_root.resolve(), - args.manifest, - args.shard, - args.bundle_dir, - args.database_metadata, - ) - elif args.command == "fan-in": - manifest = load_manifest(args.manifest) - if manifest["tree_sha"] != _validate_tree_sha(args.tree_sha): - raise ShardError("checked_out_tree_mismatch") - validate_fan_in( - manifest, - args.bundles_root, - args.output_dir, - args.summary_output, - ) - else: - _dry_run(args.repository_root.resolve(), args.shards) - except (OSError, subprocess.SubprocessError, ShardError) as exc: - code = exc.args[0] if isinstance(exc, ShardError) else "ci_shard_operation_failed" - print(f"backend CI shard operation failed: {code}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/backend/scripts/run_isolated_tests.py b/backend/scripts/run_isolated_tests.py index 2d980589..c77eacbc 100644 --- a/backend/scripts/run_isolated_tests.py +++ b/backend/scripts/run_isolated_tests.py @@ -13,7 +13,9 @@ import secrets import signal import subprocess +from subprocess import TimeoutExpired import sys +import time from urllib.parse import quote, urlsplit, urlunsplit import asyncpg @@ -28,6 +30,13 @@ OVERRIDE_ENV = "WORKSTREAM_ALLOW_NONLOCAL_E2E_DATABASE" INTERRUPTED = False TERMINATION_GRACE_SECONDS = 2.0 +HEARTBEAT_SECONDS = 60.0 +MINIO_ENDPOINT_ENV = "WORKSTREAM_TEST_MINIO_ENDPOINT" +MINIO_ACCESS_KEY = "workstream-minio" +MINIO_SECRET_KEY = "workstream-minio-secret-key" +S3_TRAFFIC_LANE = "no_postgres" +S3_TRAFFIC_BUCKET = "workstream-artifacts" +LANE_RE = re.compile(r"[a-z][a-z0-9_]{0,62}") class RunnerError(RuntimeError): @@ -130,6 +139,14 @@ async def _drop(admin_url: str, name: str, role: str) -> None: await connection.execute(f'DROP DATABASE "{name}"') if await connection.fetchval("SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", role): await connection.execute(f'DROP ROLE "{role}"') + database_exists = await connection.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1)", name + ) + role_exists = await connection.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", role + ) + if database_exists or role_exists: + raise RunnerError("cleanup_verification_failed") finally: await connection.close() @@ -146,15 +163,136 @@ async def _observed_head(target_url: str) -> str: return expected -def _child_env(target_url: str) -> dict[str, str]: +def _child_env( + target_url: str, *, minio_bucket: str = "", minio_prefix: str = "" +) -> dict[str, str]: env = os.environ.copy() env.pop(ADMIN_ENV, None) env.pop(OVERRIDE_ENV, None) env["WORKSTREAM_TEST_DATABASE_URL"] = target_url env["WORKSTREAM_DATABASE_URL"] = target_url + if minio_bucket: + env["WORKSTREAM_TEST_MINIO_BUCKET"] = minio_bucket + env["WORKSTREAM_TEST_MINIO_PREFIX"] = minio_prefix return env +def _tree_sha() -> str: + """Return the immutable commit checked out under the runner's exact tree.""" + value = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, stderr=subprocess.DEVNULL + ).strip() + if re.fullmatch(r"[a-f0-9]{40}", value) is None: + raise RunnerError("invalid_tree_sha") + return value + + +def _minio_namespace(lane: str, suffix: str) -> tuple[str, str]: + """Return a lane-bound bucket and prefix without accepting arbitrary names.""" + if LANE_RE.fullmatch(lane) is None: + raise RunnerError("invalid_lane") + bucket = S3_TRAFFIC_BUCKET if lane == S3_TRAFFIC_LANE else f"workstream-ci-{lane}-{suffix}" + return bucket, f"ci/{lane}/{suffix}" + + +def _minio_client(endpoint: str): + parsed = urlsplit(endpoint) + try: + parsed_port = parsed.port + except ValueError as exc: + raise RunnerError("unsafe_minio_endpoint") from exc + if ( + parsed.scheme != "http" + or parsed.hostname not in LOOPBACK + or parsed.username is not None + or parsed.password is not None + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + or parsed_port is None + ): + raise RunnerError("unsafe_minio_endpoint") + from aiobotocore.config import AioConfig + from aiobotocore.session import AioSession + + session = AioSession() + session.set_credentials(MINIO_ACCESS_KEY, MINIO_SECRET_KEY) + return session.create_client( + "s3", + endpoint_url=endpoint, + region_name="us-east-1", + config=AioConfig(s3={"addressing_style": "path"}), + ) + + +async def _create_minio(endpoint: str, bucket: str, prefix: str) -> None: + """Claim and probe one fresh runner-owned MinIO namespace.""" + async with _minio_client(endpoint) as client: + try: + await client.create_bucket(Bucket=bucket) + except BaseException as exc: + raise RunnerError("minio_namespace_collision") from exc + try: + key = f"{prefix}/runner-probe" + await client.put_object(Bucket=bucket, Key=key, Body=b"workstream-ci-probe") + response = await client.get_object(Bucket=bucket, Key=key) + async with response["Body"] as body: + if await body.read() != b"workstream-ci-probe": + raise RunnerError("minio_probe_failed") + await client.delete_object(Bucket=bucket, Key=key) + except BaseException as exc: + try: + await client.delete_object(Bucket=bucket, Key=f"{prefix}/runner-probe") + await client.delete_bucket(Bucket=bucket) + except BaseException as cleanup_error: + raise RunnerError("minio_provisioning_cleanup_failed") from cleanup_error + if isinstance(exc, (KeyboardInterrupt, SystemExit, RunnerError)): + raise + raise RunnerError("minio_probe_failed") from exc + + +async def _drop_minio(endpoint: str, bucket: str, prefix: str) -> None: + """Delete every object in the exact owned bucket, then prove its removal.""" + async with _minio_client(endpoint) as client: + continuation = None + while True: + request = {"Bucket": bucket} + if continuation: + request["ContinuationToken"] = continuation + response = await client.list_objects_v2(**request) + objects = response.get("Contents", []) + if objects: + await client.delete_objects( + Bucket=bucket, + Delete={"Objects": [{"Key": item["Key"]} for item in objects]}, + ) + if not response.get("IsTruncated"): + break + continuation = response.get("NextContinuationToken") + if not continuation: + raise RunnerError("minio_cleanup_failed") + await client.delete_bucket(Bucket=bucket) + buckets = await client.list_buckets() + if bucket in {item["Name"] for item in buckets.get("Buckets", [])}: + raise RunnerError("minio_cleanup_failed") + + +def _write_metadata(path: Path, metadata: dict[str, object]) -> None: + """Create or replace private metadata without following a destination symlink.""" + data = (json.dumps(metadata, indent=2, sort_keys=True) + "\n").encode() + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + try: + os.fchmod(descriptor, 0o600) + view = memoryview(data) + while view: + view = view[os.write(descriptor, view) :] + finally: + os.close(descriptor) + + def _emit(data: bytes, stream, secrets_to_hide: tuple[str, ...]) -> None: for value in secrets_to_hide: data = data.replace(value.encode(), b"[REDACTED_DATABASE_URL]") @@ -173,18 +311,35 @@ def _run( stderr=subprocess.PIPE, start_new_session=True, ) + started_at = time.monotonic() try: - stdout, stderr = process.communicate(timeout=timeout) - return process.returncode, stdout, stderr - except subprocess.TimeoutExpired: - _signal_group(process, signal.SIGKILL) - stdout, stderr = process.communicate() - return 124, stdout, stderr + while True: + elapsed = time.monotonic() - started_at + if timeout is not None and elapsed >= timeout: + _signal_group(process, signal.SIGTERM) + try: + stdout, stderr = process.communicate(timeout=TERMINATION_GRACE_SECONDS) + except TimeoutExpired: + _signal_group(process, signal.SIGKILL) + stdout, stderr = process.communicate() + return 124, stdout, stderr + wait_seconds = HEARTBEAT_SECONDS + if timeout is not None: + wait_seconds = min(wait_seconds, max(0.001, timeout - elapsed)) + try: + stdout, stderr = process.communicate(timeout=wait_seconds) + return process.returncode, stdout, stderr + except TimeoutExpired: + print( + "isolated-test child active: " + f"elapsed_seconds={time.monotonic() - started_at:.0f}", + flush=True, + ) except KeyboardInterrupt: _signal_group(process, signal.SIGTERM) try: stdout, stderr = process.communicate(timeout=TERMINATION_GRACE_SECONDS) - except subprocess.TimeoutExpired: + except TimeoutExpired: _signal_group(process, signal.SIGKILL) stdout, stderr = process.communicate() return 130, stdout, stderr @@ -213,6 +368,8 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--metadata-json", required=True, type=Path) parser.add_argument("--timeout-seconds", type=float) + parser.add_argument("--lane", default="isolated") + parser.add_argument("--tree-sha") parser.add_argument("command", nargs=argparse.REMAINDER) args = parser.parse_args() command = args.command[1:] if args.command[:1] == ["--"] else args.command @@ -220,6 +377,10 @@ def main() -> int: INTERRUPTED = False admin_url = os.environ.pop(ADMIN_ENV, "") owned = False + minio_owned = False + name = role = bucket = prefix = "" + endpoint = os.environ.get(MINIO_ENDPOINT_ENV, "") + metadata: dict[str, object] | None = None result = 2 previous_sigint = signal.signal(signal.SIGINT, _defer_interrupt) previous_sigterm = signal.signal(signal.SIGTERM, _defer_interrupt) @@ -228,21 +389,35 @@ def main() -> int: not command or args.metadata_json.exists() or args.metadata_json.is_symlink() + or args.metadata_json.parent.is_symlink() or not args.metadata_json.parent.is_dir() ): raise RunnerError("invalid_runner_arguments") + if args.timeout_seconds is not None and args.timeout_seconds <= 0: + raise RunnerError("invalid_runner_arguments") + actual_tree_sha = _tree_sha() + if args.tree_sha is not None and args.tree_sha != actual_tree_sha: + raise RunnerError("tree_sha_mismatch") name, role, password, target_url = _urls(admin_url) asyncio.run(_create(admin_url, name, role, password)) owned = True + if endpoint: + suffix = name.removeprefix("workstream_test_") + bucket, prefix = _minio_namespace(args.lane, suffix) + asyncio.run(_create_minio(endpoint, bucket, prefix)) + minio_owned = True signal.signal(signal.SIGINT, _interrupt) signal.signal(signal.SIGTERM, _interrupt) if INTERRUPTED: raise KeyboardInterrupt migration = _run( - [sys.executable, "-m", "alembic", "upgrade", "head"], _child_env(target_url), None + [sys.executable, "-m", "alembic", "upgrade", "head"], + _child_env(target_url, minio_bucket=bucket, minio_prefix=prefix), + None, ) - _emit(migration[1], sys.stdout, (admin_url, target_url)) - _emit(migration[2], sys.stderr, (admin_url, target_url)) + hidden = (admin_url, target_url, MINIO_ACCESS_KEY, MINIO_SECRET_KEY) + _emit(migration[1], sys.stdout, hidden) + _emit(migration[2], sys.stderr, hidden) if migration[0] == 130 and INTERRUPTED: raise KeyboardInterrupt if migration[0] != 0: @@ -251,17 +426,26 @@ def main() -> int: metadata = { "alembic_head": head, "database_name": name, - "schema_version": 1, - "tree_sha": subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True - ).strip(), + "database_cleanup_complete": False, + "database_provisioned": True, + "database_role": role, + "lane": args.lane, + "minio_bucket": bucket or None, + "minio_prefix": prefix or None, + "minio_cleanup_complete": False, + "minio_probe_complete": minio_owned, + "minio_provisioned": minio_owned, + "schema_version": 2, + "tree_sha": actual_tree_sha, } - args.metadata_json.write_text( - json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8" + _write_metadata(args.metadata_json, metadata) + code, stdout, stderr = _run( + command, + _child_env(target_url, minio_bucket=bucket, minio_prefix=prefix), + args.timeout_seconds, ) - code, stdout, stderr = _run(command, _child_env(target_url), args.timeout_seconds) - _emit(stdout, sys.stdout, (admin_url, target_url)) - _emit(stderr, sys.stderr, (admin_url, target_url)) + _emit(stdout, sys.stdout, hidden) + _emit(stderr, sys.stderr, hidden) result = code except (RunnerError, OSError, asyncpg.PostgresError) as exc: code = exc.args[0] if isinstance(exc, RunnerError) else "database_operation_failed" @@ -272,12 +456,34 @@ def main() -> int: finally: signal.signal(signal.SIGINT, _defer_interrupt) signal.signal(signal.SIGTERM, _defer_interrupt) + cleanup_ok = True + minio_cleanup_ok = not minio_owned + if minio_owned: + try: + asyncio.run(_drop_minio(endpoint, bucket, prefix)) + minio_cleanup_ok = True + except BaseException: + print("isolated-test runner failed: minio_cleanup_failed", file=sys.stderr) + result = 2 + cleanup_ok = False + database_cleanup_ok = not owned if owned: try: asyncio.run(_drop(admin_url, name, role)) + database_cleanup_ok = True except BaseException: print("isolated-test runner failed: cleanup_failed", file=sys.stderr) result = 2 + cleanup_ok = False + if metadata is not None: + metadata["database_cleanup_complete"] = database_cleanup_ok + metadata["minio_cleanup_complete"] = minio_cleanup_ok + metadata["cleanup_complete"] = cleanup_ok + try: + _write_metadata(args.metadata_json, metadata) + except OSError: + print("isolated-test runner failed: metadata_write_failed", file=sys.stderr) + result = 2 signal.signal(signal.SIGINT, previous_sigint) signal.signal(signal.SIGTERM, previous_sigterm) return result diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py new file mode 100644 index 00000000..7b44c08a --- /dev/null +++ b/backend/scripts/run_test_lanes.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Collect and run four exact-custody backend test lanes.""" + +from __future__ import annotations + +import argparse +from collections import Counter +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import signal +import subprocess +import sys +import time +from typing import Any, TextIO + +ROOT = Path(__file__).resolve().parents[1] +TESTS_DIR = ROOT / "tests" +ISOLATED_RUNNER = ROOT / "scripts" / "run_isolated_tests.py" +ADMIN_ENV = "WORKSTREAM_TEST_ADMIN_DATABASE_URL" +COLLECTED_ENV = "WORKSTREAM_LANE_COLLECTED_NODES" +COMPLETED_ENV = "WORKSTREAM_LANE_COMPLETED_NODES" +SKIPPED_ENV = "WORKSTREAM_LANE_SKIPPED_NODES" +DESELECTED_ENV = "WORKSTREAM_LANE_DESELECTED_NODES" +SCHEMA_VERSION = 1 +ADMIN_RUNNER_MODULE = "tests/test_isolated_database_runner.py" +ORDINARY_KIND = "ordinary_isolated" +ADMIN_KIND = "admin_runner_self_test" +HEARTBEAT_SECONDS = 60.0 +CLEANUP_GRACE_SECONDS = 10.0 +POLL_SECONDS = 0.05 +SHA_RE = re.compile(r"[0-9a-f]{40}") +LANE_RE = re.compile(r"[a-z][a-z0-9_]*") +INTERRUPTED = False + + +class LaneError(RuntimeError): + """A stable semantic-lane contract failure.""" + + +@dataclass(frozen=True) +class TestLane: + """One dependency-oriented test process.""" + + name: str + modules: tuple[str, ...] + requires_postgres: bool = True + + +LANES = ( + TestLane( + "no_postgres", + ( + "tests/test_actor_legacy_classification.py", + "tests/test_actor_migration_tools.py", + "tests/test_agent_runtime.py", + "tests/test_api_contract_e2e.py", + "tests/test_api_controls.py", + "tests/test_app.py", + "tests/test_artifact_architecture.py", + "tests/test_artifact_authorization.py", + "tests/test_artifact_cleanup_wiring.py", + "tests/test_artifact_preparation.py", + "tests/test_artifact_store_conformance.py", + "tests/test_artifact_verification.py", + "tests/test_artifacts.py", + "tests/test_assertion_helpers.py", + "tests/test_aws_credential_isolation.py", + "tests/test_ci_test_lanes.py", + "tests/test_config.py", + "tests/test_coverage_contract.py", + "tests/test_external_service_adapters.py", + "tests/test_local_artifact_store.py", + "tests/test_s3_artifact_store.py", + "tests/test_test_lane_evidence.py", + ), + ), + TestLane( + "schema_contracts", + ( + "tests/test_alembic.py", + "tests/test_database_reset.py", + ADMIN_RUNNER_MODULE, + ), + ), + TestLane( + "control_plane", + ( + "tests/test_actors.py", + "tests/test_api_rate_controls.py", + "tests/test_audit.py", + "tests/test_auth.py", + "tests/test_authorization.py", + "tests/test_projects.py", + ), + ), + TestLane( + "execution_plane", + ( + "tests/test_artifact_admission.py", + "tests/test_artifact_operator_api.py", + "tests/test_artifact_recovery.py", + "tests/test_checkers.py", + "tests/test_db_session.py", + "tests/test_outbox.py", + "tests/test_tasks.py", + ), + ), +) +@dataclass +class ActiveLane: + key: str + lane: TestLane + execution_kind: str + expected_nodes: tuple[str, ...] + process: subprocess.Popen[bytes] + log: TextIO + log_path: Path + evidence_path: Path + coverage_path: Path + started_at: float + interrupted_at: float | None = None + timed_out: bool = False + + +def _json_bytes(value: Any) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True) + "\n").encode() + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _exclusive_file(path: Path) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(path, flags, 0o600) + os.close(descriptor) + + +def _append_node(destination: str, node_id: str) -> None: + flags = os.O_WRONLY | os.O_APPEND + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(destination, flags) + try: + os.write(descriptor, (json.dumps(node_id) + "\n").encode()) + finally: + os.close(descriptor) + + +def pytest_collection_finish(session: Any) -> None: + """Record exact selected node IDs in the collecting pytest process.""" + destination = os.environ.get(COLLECTED_ENV) + if destination: + for item in session.items: + _append_node(destination, item.nodeid) + + +def pytest_runtest_logfinish(nodeid: str, location: tuple[str, int | None, str]) -> None: + """Record completion only after the node's full pytest lifecycle.""" + del location + destination = os.environ.get(COMPLETED_ENV) + if destination: + _append_node(destination, nodeid) + + +def pytest_runtest_logreport(report: Any) -> None: + """Record every skip observed during setup, call, or teardown.""" + destination = os.environ.get(SKIPPED_ENV) + if destination and report.skipped: + _append_node(destination, report.nodeid) + + +def pytest_deselected(items: list[Any]) -> None: + """Record nodes removed after initial collection.""" + destination = os.environ.get(DESELECTED_ENV) + if destination: + for item in items: + _append_node(destination, item.nodeid) + + +def discover_test_modules(tests_dir: Path = TESTS_DIR, root: Path = ROOT) -> tuple[str, ...]: + """Recursively discover regular test modules without following symlinks.""" + if tests_dir.is_symlink() or not tests_dir.is_dir(): + raise LaneError("invalid_tests_root") + modules: list[str] = [] + for directory, names, files in os.walk(tests_dir, followlinks=False): + current = Path(directory) + if current.is_symlink(): + raise LaneError("symlinked_test_directory") + for name in names: + if (current / name).is_symlink(): + raise LaneError("symlinked_test_directory") + for name in files: + if not (name.startswith("test_") and name.endswith(".py")): + continue + path = current / name + if path.is_symlink() or not path.is_file(): + raise LaneError("symlinked_test_module") + try: + modules.append(path.relative_to(root).as_posix()) + except ValueError as exc: + raise LaneError("invalid_test_module") from exc + if not modules or len(modules) != len(set(modules)): + raise LaneError("invalid_test_inventory") + return tuple(sorted(modules)) + + +def _safe_module_path(value: str) -> bool: + path = PurePosixPath(value) + return ( + not path.is_absolute() + and len(path.parts) >= 2 + and path.parts[0] == "tests" + and path.name.startswith("test_") + and path.suffix == ".py" + and ".." not in path.parts + ) + + +def validate_lane_inventory( + discovered: tuple[str, ...], + *, + lanes: tuple[TestLane, ...] | None = None, +) -> None: + lanes = LANES if lanes is None else lanes + names = [lane.name for lane in lanes] + if len(lanes) != 4 or len(set(names)) != 4 or any(LANE_RE.fullmatch(x) is None for x in names): + raise LaneError("invalid_lane_names") + declared = [module for lane in lanes for module in lane.modules] + if any(not _safe_module_path(module) for module in declared): + raise LaneError("invalid_lane_module") + duplicates = sorted(module for module, count in Counter(declared).items() if count != 1) + if duplicates: + raise LaneError(f"duplicate_lane_modules:{','.join(duplicates)}") + missing = sorted(set(discovered) - set(declared)) + foreign = sorted(set(declared) - set(discovered)) + if missing: + raise LaneError(f"missing_lane_modules:{','.join(missing)}") + if foreign: + raise LaneError(f"foreign_lane_modules:{','.join(foreign)}") + + +def _tree_sha() -> str: + value = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ).stdout.strip() + if SHA_RE.fullmatch(value) is None: + raise LaneError("invalid_tree_sha") + return value + + +def _read_nodes(path: Path, *, allow_empty: bool = False) -> list[str]: + try: + values = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise LaneError("invalid_node_evidence") from exc + if (not allow_empty and not values) or any(not isinstance(value, str) or "::" not in value for value in values): + raise LaneError("invalid_node_evidence") + return values + + +def _module_from_node(node_id: str) -> str: + module = node_id.split("::", 1)[0] + if "::" not in node_id or not _safe_module_path(module): + raise LaneError("invalid_node_id") + return module + + +def _plugin_args() -> list[str]: + return ["-p", "pytest_asyncio.plugin", "-p", "pytest_cov.plugin", "-p", "scripts.run_test_lanes"] + + +def _collection_environment(collected: Path, deselected: Path) -> dict[str, str]: + env = os.environ.copy() + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + env["PYTHONPATH"] = os.pathsep.join( + value for value in (str(ROOT), env.get("PYTHONPATH", "")) if value + ) + env[COLLECTED_ENV] = str(collected.resolve()) + env[DESELECTED_ENV] = str(deselected.resolve()) + return env + + +def collect_nodes(modules: tuple[str, ...], metadata_dir: Path) -> tuple[int, list[str], list[str]]: + collected = metadata_dir / "collection.nodes.jsonl" + deselected = metadata_dir / "collection.deselected.jsonl" + _exclusive_file(collected) + _exclusive_file(deselected) + result = subprocess.run( + [sys.executable, "-m", "pytest", "--collect-only", "-q", *_plugin_args(), *modules], + cwd=ROOT, env=_collection_environment(collected, deselected), check=False, + ) + nodes = _read_nodes(collected) if result.returncode == 0 else [] + deselected_nodes = _read_nodes(deselected, allow_empty=True) + if result.returncode == 0: + expected = set(modules) + if len(nodes) != len(set(nodes)) or any(_module_from_node(node) not in expected for node in nodes): + raise LaneError("invalid_collected_nodes") + if set(_module_from_node(node) for node in nodes) != expected: + raise LaneError("zero_collected_module") + return result.returncode, sorted(nodes), sorted(deselected_nodes) + + +def build_manifest(tree_sha: str, nodes: list[str]) -> dict[str, Any]: + """Build the validator-owned, raw-byte-digested canonical manifest.""" + if not nodes or len(nodes) != len(set(nodes)): + raise LaneError("invalid_collected_nodes") + nodes = sorted(nodes) + lane_by_module = {module: lane.name for lane in LANES for module in lane.modules} + return { + "head_sha": tree_sha, + "nodes": [ + { + "execution_kind": ADMIN_KIND + if _module_from_node(node) == ADMIN_RUNNER_MODULE + else ORDINARY_KIND, + "lane": lane_by_module[_module_from_node(node)], + "module": _module_from_node(node), + "nodeid": node, + } + for node in nodes + ], + "schema_version": SCHEMA_VERSION, + } + + +def lane_environment( + lane: TestLane, + metadata_dir: Path, + coverage_path: Path, + evidence_stem: str | None = None, +) -> dict[str, str]: + env = os.environ.copy() + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + env["PYTHONPATH"] = os.pathsep.join( + value for value in (str(ROOT), env.get("PYTHONPATH", "")) if value + ) + env["COVERAGE_FILE"] = str(coverage_path.resolve()) + stem = evidence_stem or lane.name + env[COLLECTED_ENV] = str((metadata_dir / f"{stem}.collected.jsonl").resolve()) + env[COMPLETED_ENV] = str((metadata_dir / f"{stem}.completed.jsonl").resolve()) + env[SKIPPED_ENV] = str((metadata_dir / f"{stem}.skipped.jsonl").resolve()) + env[DESELECTED_ENV] = str((metadata_dir / f"{stem}.deselected.jsonl").resolve()) + return env + + +def admin_runner_environment( + lane: TestLane, metadata_dir: Path, coverage_path: Path, evidence_stem: str +) -> dict[str, str]: + """Retain only the admin URL needed to test the isolation owner itself.""" + env = lane_environment(lane, metadata_dir, coverage_path, evidence_stem) + env.pop("WORKSTREAM_DATABASE_URL", None) + env.pop("WORKSTREAM_TEST_DATABASE_URL", None) + return env + + +def lane_command( + lane: TestLane, + nodes: list[str], + metadata_dir: Path, + timeout_seconds: float, + tree_sha: str, +) -> list[str]: + pytest_command = [ + sys.executable, "-m", "pytest", "-q", *_plugin_args(), "--cov=app", "--cov-report=", + "--durations=25", *nodes, + ] + return [ + sys.executable, str(ISOLATED_RUNNER), "--metadata-json", + str(metadata_dir / f"{lane.name}.database.json"), "--lane", lane.name, + "--tree-sha", tree_sha, "--timeout-seconds", f"{timeout_seconds:g}", + "--", *pytest_command, + ] + + +def admin_runner_command(nodes: list[str]) -> list[str]: + """Run isolation-runner self-tests directly, never inside an owned database.""" + return [ + sys.executable, "-m", "pytest", "-q", *_plugin_args(), "--cov=app", + "--cov-report=", "--durations=25", *nodes, + ] + + +def _prepare_outputs(metadata_dir: Path, summary_json: Path) -> None: + if metadata_dir.exists() or metadata_dir.is_symlink() or summary_json.exists() or summary_json.is_symlink(): + raise LaneError("invalid_lane_outputs") + if not metadata_dir.parent.is_dir() or not summary_json.parent.is_dir(): + raise LaneError("invalid_lane_outputs") + metadata_dir.mkdir(mode=0o700) + + +def _signal_lane(active: ActiveLane, value: int) -> None: + if active.process.poll() is None: + try: + os.killpg(active.process.pid, value) + except ProcessLookupError: + pass + + +def _handle_interrupt(_signum: int, _frame: object) -> None: + global INTERRUPTED + INTERRUPTED = True + + +def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str, Any]: + active.log.flush() + active.log.close() + collected = _read_nodes(active.evidence_path.with_name(f"{active.key}.collected.jsonl"), allow_empty=True) + completed = _read_nodes(active.evidence_path.with_name(f"{active.key}.completed.jsonl"), allow_empty=True) + skipped = _read_nodes(active.evidence_path.with_name(f"{active.key}.skipped.jsonl"), allow_empty=True) + deselected = _read_nodes(active.evidence_path.with_name(f"{active.key}.deselected.jsonl"), allow_empty=True) + return { + "collection_exit_code": 0 + if sorted(collected) == sorted(active.expected_nodes) + else 1, + "collected_nodes": collected, + "completed_nodes": completed, + "coverage_path": active.coverage_path, + "deselected_nodes": deselected, + "elapsed_seconds": round(elapsed, 3), + "execution_exit_code": exit_code, + "interrupted": active.interrupted_at is not None or active.timed_out, + "skipped_nodes": skipped, + } + + +def _combine_coverage(sources: list[Path], destination: Path) -> None: + regular = [path for path in sources if path.is_file() and not path.is_symlink()] + if len(regular) != len(sources): + return + if len(regular) == 1: + shutil.copyfile(regular[0], destination) + return + result = subprocess.run( + [sys.executable, "-m", "coverage", "combine", "--data-file", str(destination), + *(str(path) for path in regular)], + cwd=ROOT, check=False, + ) + if result.returncode != 0: + raise LaneError("lane_coverage_combine_failed") + + +def _finalize_lane( + lane: TestLane, units: list[dict[str, Any]], metadata_dir: Path +) -> dict[str, Any]: + evidence_path = metadata_dir / f"{lane.name}.json" + isolation_path = metadata_dir / f"{lane.name}.database.json" + coverage_path = metadata_dir / f".coverage.{lane.name}" + _combine_coverage([unit["coverage_path"] for unit in units], coverage_path) + evidence = { + "collected_nodes": sorted(node for unit in units for node in unit["collected_nodes"]), + "completed_nodes": sorted(node for unit in units for node in unit["completed_nodes"]), + "deselected_nodes": sorted(set(node for unit in units for node in unit["deselected_nodes"])), + "isolation_metadata_file": isolation_path.name, + "isolation_metadata_sha256": _sha256(isolation_path.read_bytes()), + "skipped_nodes": sorted(set(node for unit in units for node in unit["skipped_nodes"])), + } + evidence_path.write_bytes(_json_bytes(evidence)) + return { + "collection_exit_code": max(unit["collection_exit_code"] for unit in units), + "coverage_file": coverage_path.name, + "coverage_sha256": _sha256(coverage_path.read_bytes()) + if coverage_path.is_file() and not coverage_path.is_symlink() else None, + "elapsed_seconds": round(max(unit["elapsed_seconds"] for unit in units), 3), + "evidence_file": evidence_path.name, + "evidence_sha256": _sha256(evidence_path.read_bytes()), + "execution_exit_code": max(unit["execution_exit_code"] for unit in units), + "interrupted": any(unit["interrupted"] for unit in units), + "name": lane.name, + } + + +def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, collect_only: bool = False) -> int: + """Collect canonical nodes and optionally execute all lanes concurrently.""" + global INTERRUPTED + INTERRUPTED = False + if timeout_seconds <= 0: + raise LaneError("invalid_lane_timeout") + modules = discover_test_modules() + validate_lane_inventory(modules) + _prepare_outputs(metadata_dir, summary_json) + tree_sha = _tree_sha() + collection_exit, nodes, deselected = collect_nodes(modules, metadata_dir) + manifest = build_manifest(tree_sha, nodes) + manifest_path = metadata_dir / "manifest.json" + manifest_path.write_bytes(_json_bytes(manifest)) + manifest_digest = _sha256(manifest_path.read_bytes()) + if collection_exit != 0 or deselected: + raise LaneError("pytest_collection_failed") + if collect_only: + lane_rows = [] + for lane in LANES: + lane_nodes = [row["nodeid"] for row in manifest["nodes"] if row["lane"] == lane.name] + evidence_path = metadata_dir / f"{lane.name}.json" + evidence_path.write_bytes(_json_bytes({ + "collected_nodes": lane_nodes, "completed_nodes": [], "deselected_nodes": [], + "isolation_metadata_file": None, "isolation_metadata_sha256": None, + "skipped_nodes": [], + })) + lane_rows.append({ + "collection_exit_code": 0, "coverage_file": None, "coverage_sha256": None, + "elapsed_seconds": 0.0, "evidence_file": evidence_path.name, + "evidence_sha256": _sha256(evidence_path.read_bytes()), "execution_exit_code": None, + "interrupted": False, "name": lane.name, + }) + summary = {"canonical_node_count": len(nodes), "elapsed_seconds": 0.0, + "head_sha": tree_sha, "lanes": lane_rows, "manifest_file": manifest_path.name, + "manifest_sha256": manifest_digest, "mode": "collect", "schema_version": SCHEMA_VERSION} + summary_json.write_bytes(_json_bytes(summary)) + return 0 + if any(lane.requires_postgres for lane in LANES) and not os.environ.get(ADMIN_ENV): + raise LaneError("missing_admin_database_url") + + active: dict[str, ActiveLane] = {} + unit_results: dict[str, list[dict[str, Any]]] = {lane.name: [] for lane in LANES} + started = time.monotonic() + stopping = False + old_int = signal.signal(signal.SIGINT, _handle_interrupt) + old_term = signal.signal(signal.SIGTERM, _handle_interrupt) + try: + for lane in LANES: + lane_rows = [row for row in manifest["nodes"] if row["lane"] == lane.name] + kinds = [ORDINARY_KIND] + if any(row["execution_kind"] == ADMIN_KIND for row in lane_rows): + kinds.append(ADMIN_KIND) + for kind in kinds: + unit_nodes = [row["nodeid"] for row in lane_rows if row["execution_kind"] == kind] + if not unit_nodes: + continue + key = lane.name if kind == ORDINARY_KIND else f"{lane.name}.admin" + for suffix in ("collected", "completed", "skipped", "deselected"): + _exclusive_file(metadata_dir / f"{key}.{suffix}.jsonl") + log_path = metadata_dir / f"{key}.log" + log = log_path.open("x", encoding="utf-8") + coverage_path = metadata_dir / f".coverage.unit.{key}" + if kind == ADMIN_KIND: + command = admin_runner_command(unit_nodes) + env = admin_runner_environment(lane, metadata_dir, coverage_path, key) + else: + command = lane_command(lane, unit_nodes, metadata_dir, timeout_seconds, tree_sha) + env = lane_environment(lane, metadata_dir, coverage_path, key) + process = subprocess.Popen( + command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True, + ) + active[key] = ActiveLane( + key, lane, kind, tuple(unit_nodes), process, log, log_path, + metadata_dir / f"{key}.json", + coverage_path, time.monotonic(), + ) + while active: + now = time.monotonic() + if INTERRUPTED and not stopping: + stopping = True + for item in active.values(): + item.interrupted_at = now + _signal_lane(item, signal.SIGINT) + for key, item in list(active.items()): + elapsed = now - item.started_at + if elapsed >= timeout_seconds and item.interrupted_at is None: + item.timed_out = True + item.interrupted_at = now + _signal_lane(item, signal.SIGINT) + elif item.interrupted_at is not None and now - item.interrupted_at >= CLEANUP_GRACE_SECONDS: + _signal_lane(item, signal.SIGKILL) + code = item.process.poll() + if code is None: + continue + unit_results[item.lane.name].append(_finish_unit(item, code, elapsed)) + del active[key] + if code != 0 and not stopping: + stopping = True + for other in active.values(): + other.interrupted_at = now + _signal_lane(other, signal.SIGINT) + if active: + time.sleep(POLL_SECONDS) + finally: + for item in active.values(): + _signal_lane(item, signal.SIGKILL) + item.process.wait() + item.log.close() + signal.signal(signal.SIGINT, old_int) + signal.signal(signal.SIGTERM, old_term) + results = { + lane.name: _finalize_lane(lane, unit_results[lane.name], metadata_dir) + for lane in LANES + if unit_results[lane.name] + } + summary = { + "canonical_node_count": len(nodes), "elapsed_seconds": round(time.monotonic() - started, 3), + "head_sha": tree_sha, "lanes": [results[lane.name] for lane in LANES if lane.name in results], + "manifest_file": manifest_path.name, "manifest_sha256": manifest_digest, + "mode": "run", "schema_version": SCHEMA_VERSION, + } + summary_json.write_bytes(_json_bytes(summary)) + return 0 if len(results) == 4 and all(row["execution_exit_code"] == 0 for row in results.values()) else 1 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--metadata-dir", required=True, type=Path) + parser.add_argument("--summary-json", required=True, type=Path) + parser.add_argument("--timeout-seconds", default=1200.0, type=float) + parser.add_argument("--collect-only", action="store_true") + args = parser.parse_args() + try: + return run_lanes(args.metadata_dir, args.summary_json, args.timeout_seconds, collect_only=args.collect_only) + except (LaneError, OSError, subprocess.SubprocessError) as exc: + code = exc.args[0] if isinstance(exc, LaneError) else "lane_operation_failed" + print(f"test lane runner failed: {code}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py new file mode 100644 index 00000000..174a5672 --- /dev/null +++ b/backend/scripts/validate_test_lane_evidence.py @@ -0,0 +1,377 @@ +"""Fail-closed validation for exact-custody semantic test-lane evidence.""" + +from __future__ import annotations + +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path, PurePosixPath +import re +import stat +import subprocess +import sys +from typing import Any + + +SCHEMA_VERSION = 1 +LANE_COUNT = 4 +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +LANE_RE = re.compile(r"^[a-z][a-z0-9_]*$") +ADMIN_RUNNER_MODULE = "tests/test_isolated_database_runner.py" +ORDINARY_KIND = "ordinary_isolated" +ADMIN_KIND = "admin_runner_self_test" +DATABASE_IDENTIFIER_RE = re.compile(r"^[a-z][a-z0-9_]*$") +BUCKET_RE = re.compile(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$") + + +class EvidenceError(RuntimeError): + """Test-lane evidence is incomplete, unsafe, or inconsistent.""" + + +def _object(value: Any, keys: set[str], error: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != keys: + raise EvidenceError(error) + return value + + +def _json_bytes(data: bytes, error: str) -> dict[str, Any]: + try: + value = json.loads(data.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvidenceError(error) from exc + if not isinstance(value, dict): + raise EvidenceError(error) + return value + + +def _digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _current_head(repository_root: Path) -> str: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repository_root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + head = result.stdout.strip() + if result.returncode != 0 or SHA_RE.fullmatch(head) is None: + raise EvidenceError("invalid_git_head") + return head + + +def _safe_file(metadata_dir: Path, name: Any) -> Path: + if not isinstance(name, str) or not name: + raise EvidenceError("unsafe_evidence_path") + pure = PurePosixPath(name) + if pure.is_absolute() or ".." in pure.parts or str(pure) != name: + raise EvidenceError("unsafe_evidence_path") + root = metadata_dir.resolve(strict=True) + candidate = metadata_dir / Path(*pure.parts) + try: + relative = candidate.relative_to(metadata_dir) + except ValueError as exc: + raise EvidenceError("unsafe_evidence_path") from exc + current = metadata_dir + for part in relative.parts: + current = current / part + try: + mode = current.lstat().st_mode + except OSError as exc: + raise EvidenceError("missing_evidence_file") from exc + if stat.S_ISLNK(mode): + raise EvidenceError("unsafe_evidence_path") + try: + if ( + candidate.resolve(strict=True).parent != root + and root not in candidate.resolve(strict=True).parents + ): + raise EvidenceError("unsafe_evidence_path") + if not stat.S_ISREG(candidate.stat().st_mode): + raise EvidenceError("unsafe_evidence_path") + except OSError as exc: + raise EvidenceError("missing_evidence_file") from exc + return candidate + + +def _bound_bytes(metadata_dir: Path, name: Any, digest: Any) -> bytes: + if not isinstance(digest, str) or DIGEST_RE.fullmatch(digest) is None: + raise EvidenceError("invalid_evidence_digest") + try: + data = _safe_file(metadata_dir, name).read_bytes() + except OSError as exc: + raise EvidenceError("unreadable_evidence_file") from exc + if _digest(data) != digest: + raise EvidenceError("evidence_digest_mismatch") + return data + + +def _nodes(value: Any, field: str) -> list[str]: + if not isinstance(value, list) or any(not isinstance(node, str) or not node for node in value): + raise EvidenceError(f"invalid_{field}") + return value + + +def validate_evidence( + metadata_dir: Path, summary_json: Path, repository_root: Path | None = None +) -> dict[str, Any]: + """Validate one complete collection or execution evidence set.""" + if metadata_dir.is_symlink() or not metadata_dir.is_dir(): + raise EvidenceError("invalid_metadata_dir") + if summary_json.is_symlink() or not summary_json.is_file(): + raise EvidenceError("invalid_summary_file") + summary = _object( + _json_bytes(summary_json.read_bytes(), "invalid_summary"), + { + "canonical_node_count", + "elapsed_seconds", + "head_sha", + "lanes", + "manifest_file", + "manifest_sha256", + "mode", + "schema_version", + }, + "invalid_summary", + ) + mode = summary["mode"] + if summary["schema_version"] != SCHEMA_VERSION or mode not in {"collect", "run"}: + raise EvidenceError("invalid_summary") + root = repository_root or Path(__file__).resolve().parents[2] + head = summary["head_sha"] + if not isinstance(head, str) or head != _current_head(root): + raise EvidenceError("head_sha_mismatch") + + manifest = _object( + _json_bytes( + _bound_bytes(metadata_dir, summary["manifest_file"], summary["manifest_sha256"]), + "invalid_manifest", + ), + {"head_sha", "nodes", "schema_version"}, + "invalid_manifest", + ) + if manifest["schema_version"] != SCHEMA_VERSION or manifest["head_sha"] != head: + raise EvidenceError("manifest_head_mismatch") + raw_manifest_nodes = manifest["nodes"] + if not isinstance(raw_manifest_nodes, list) or not raw_manifest_nodes: + raise EvidenceError("zero_canonical_nodes") + canonical: list[tuple[str, str, str, str]] = [] + for value in raw_manifest_nodes: + row = _object( + value, + {"execution_kind", "lane", "module", "nodeid"}, + "invalid_manifest_node", + ) + if not all(isinstance(row[key], str) and row[key] for key in row): + raise EvidenceError("invalid_manifest_node") + module_path = PurePosixPath(row["module"]) + if ( + module_path.is_absolute() + or ".." in module_path.parts + or not module_path.parts + or module_path.parts[0] != "tests" + or not module_path.name.startswith("test_") + or module_path.suffix != ".py" + or not row["nodeid"].startswith(f"{row['module']}::") + or LANE_RE.fullmatch(row["lane"]) is None + or row["execution_kind"] not in {ORDINARY_KIND, ADMIN_KIND} + or (row["execution_kind"] == ADMIN_KIND and row["module"] != ADMIN_RUNNER_MODULE) + or (row["execution_kind"] == ORDINARY_KIND and row["module"] == ADMIN_RUNNER_MODULE) + ): + raise EvidenceError("invalid_manifest_node") + canonical.append((row["nodeid"], row["module"], row["lane"], row["execution_kind"])) + if ( + canonical != sorted(canonical) + or len(canonical) != len(set(canonical)) + or len({nodeid for nodeid, _module, _lane, _kind in canonical}) != len(canonical) + ): + raise EvidenceError("noncanonical_or_duplicate_manifest_nodes") + if not any(module == ADMIN_RUNNER_MODULE for _node, module, _lane, _kind in canonical): + raise EvidenceError("missing_admin_runner_self_tests") + if isinstance(summary["canonical_node_count"], bool) or summary["canonical_node_count"] != len( + canonical + ): + raise EvidenceError("canonical_node_count_mismatch") + + lanes = summary["lanes"] + if not isinstance(lanes, list) or len(lanes) != LANE_COUNT: + raise EvidenceError("invalid_lane_count") + expected_by_lane: dict[str, list[str]] = {} + for nodeid, _module, lane, _kind in canonical: + expected_by_lane.setdefault(lane, []).append(nodeid) + lane_names = [lane.get("name") if isinstance(lane, dict) else None for lane in lanes] + if len(set(lane_names)) != LANE_COUNT or set(lane_names) != set(expected_by_lane): + raise EvidenceError("lane_inventory_mismatch") + + all_collected: list[str] = [] + all_completed: list[str] = [] + isolation_namespaces: list[tuple[str, str, str, str]] = [] + coverage_files: list[str] = [] + isolation_files: list[str] = [] + for lane_value in lanes: + lane = _object( + lane_value, + { + "collection_exit_code", + "coverage_file", + "coverage_sha256", + "elapsed_seconds", + "evidence_file", + "evidence_sha256", + "execution_exit_code", + "interrupted", + "name", + }, + "invalid_lane_summary", + ) + name = lane["name"] + if lane["collection_exit_code"] != 0: + raise EvidenceError("collection_failed") + if lane["interrupted"] is not False: + raise EvidenceError("lane_interrupted") + expected_execution = None if mode == "collect" else 0 + if lane["execution_exit_code"] != expected_execution: + raise EvidenceError("execution_incomplete") + evidence = _object( + _json_bytes( + _bound_bytes(metadata_dir, lane["evidence_file"], lane["evidence_sha256"]), + "invalid_lane_evidence", + ), + { + "collected_nodes", + "completed_nodes", + "deselected_nodes", + "isolation_metadata_file", + "isolation_metadata_sha256", + "skipped_nodes", + }, + "invalid_lane_evidence", + ) + collected = _nodes(evidence["collected_nodes"], "collected_nodes") + completed = _nodes(evidence["completed_nodes"], "completed_nodes") + skipped = _nodes(evidence["skipped_nodes"], "skipped_nodes") + deselected = _nodes(evidence["deselected_nodes"], "deselected_nodes") + if skipped: + raise EvidenceError("unexpected_skipped_nodes") + if deselected: + raise EvidenceError("unexpected_deselected_nodes") + if Counter(collected) != Counter(expected_by_lane[name]): + raise EvidenceError("collected_node_reconciliation_failed") + if len(collected) != len(set(collected)): + raise EvidenceError("duplicate_collected_nodes") + if mode == "collect": + if ( + completed + or lane["coverage_file"] is not None + or lane["coverage_sha256"] is not None + ): + raise EvidenceError("invalid_collect_mode_artifacts") + if ( + evidence["isolation_metadata_file"] is not None + or evidence["isolation_metadata_sha256"] is not None + ): + raise EvidenceError("invalid_collect_mode_artifacts") + else: + if Counter(completed) != Counter(collected) or len(completed) != len(set(completed)): + raise EvidenceError("partial_or_duplicate_completion") + _bound_bytes(metadata_dir, lane["coverage_file"], lane["coverage_sha256"]) + coverage_files.append(lane["coverage_file"]) + isolation_files.append(evidence["isolation_metadata_file"]) + isolation = _object( + _json_bytes( + _bound_bytes( + metadata_dir, + evidence["isolation_metadata_file"], + evidence["isolation_metadata_sha256"], + ), + "invalid_isolation_metadata", + ), + { + "alembic_head", + "cleanup_complete", + "database_name", + "database_cleanup_complete", + "database_provisioned", + "database_role", + "lane", + "minio_bucket", + "minio_cleanup_complete", + "minio_prefix", + "minio_probe_complete", + "minio_provisioned", + "schema_version", + "tree_sha", + }, + "invalid_isolation_metadata", + ) + namespace_fields = ( + "database_name", + "database_role", + "minio_bucket", + "minio_prefix", + ) + if ( + isolation["schema_version"] != 2 + or isolation["tree_sha"] != head + or isolation["lane"] != name + or isolation["database_provisioned"] is not True + or isolation["minio_provisioned"] is not True + or isolation["database_cleanup_complete"] is not True + or isolation["minio_probe_complete"] is not True + or isolation["minio_cleanup_complete"] is not True + or isolation["cleanup_complete"] is not True + or not isinstance(isolation["alembic_head"], str) + or not isolation["alembic_head"] + or any( + not isinstance(isolation[field], str) or not isolation[field] + for field in namespace_fields + ) + or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_name"]) is None + or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_role"]) is None + or BUCKET_RE.fullmatch(isolation["minio_bucket"]) is None + or PurePosixPath(isolation["minio_prefix"]).is_absolute() + or ".." in PurePosixPath(isolation["minio_prefix"]).parts + ): + raise EvidenceError("invalid_isolation_metadata") + isolation_namespaces.append(tuple(isolation[field] for field in namespace_fields)) + all_collected.extend(collected) + all_completed.extend(completed) + + canonical_ids = [nodeid for nodeid, _module, _lane, _kind in canonical] + if Counter(all_collected) != Counter(canonical_ids): + raise EvidenceError("global_collection_reconciliation_failed") + if mode == "run" and Counter(all_completed) != Counter(canonical_ids): + raise EvidenceError("global_completion_reconciliation_failed") + if mode == "run" and any( + len({namespace[index] for namespace in isolation_namespaces}) != LANE_COUNT + for index in range(4) + ): + raise EvidenceError("shared_isolation_namespace") + if mode == "run" and ( + len(set(coverage_files)) != LANE_COUNT or len(set(isolation_files)) != LANE_COUNT + ): + raise EvidenceError("shared_lane_artifact") + return summary + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--metadata-dir", type=Path, required=True) + parser.add_argument("--summary-json", type=Path, required=True) + args = parser.parse_args(argv) + try: + validate_evidence(args.metadata_dir, args.summary_json) + except (EvidenceError, OSError) as exc: + print(f"test lane evidence invalid: {exc}", file=sys.stderr) + return 1 + print("test lane evidence valid") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py new file mode 100644 index 00000000..b41e029a --- /dev/null +++ b/backend/tests/test_ci_test_lanes.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from dataclasses import replace +import hashlib +import json +from pathlib import Path +import sys +import time + +import pytest # type: ignore[import-not-found] + +import scripts.run_test_lanes as runner +from scripts.run_test_lanes import LANES, LaneError, TestLane as LaneDefinition + + +def test_committed_lanes_cover_recursive_inventory_exactly_once() -> None: + discovered = runner.discover_test_modules() + runner.validate_lane_inventory(discovered) + + assigned = [module for lane in LANES for module in lane.modules] + assert len(LANES) == 4 + assert all(lane.requires_postgres for lane in LANES) + assert len(assigned) == len(set(assigned)) + assert set(assigned) == set(discovered) + assert runner.ADMIN_RUNNER_MODULE in next( + lane.modules for lane in LANES if lane.name == "schema_contracts" + ) + + +def test_discovery_is_recursive_and_lexically_canonical(tmp_path: Path) -> None: + tests = tmp_path / "tests" + nested = tests / "nested" + nested.mkdir(parents=True) + (nested / "test_z.py").write_text("def test_z(): pass\n", encoding="utf-8") + (tests / "test_a.py").write_text("def test_a(): pass\n", encoding="utf-8") + + assert runner.discover_test_modules(tests, tmp_path) == ( + "tests/nested/test_z.py", + "tests/test_a.py", + ) + + +@pytest.mark.parametrize("kind", ("file", "directory")) +def test_discovery_rejects_symlinks(tmp_path: Path, kind: str) -> None: + tests = tmp_path / "tests" + tests.mkdir() + if kind == "file": + target = tmp_path / "target.py" + target.write_text("def test_x(): pass\n", encoding="utf-8") + (tests / "test_link.py").symlink_to(target) + else: + target = tmp_path / "target" + target.mkdir() + (tests / "nested").symlink_to(target, target_is_directory=True) + + with pytest.raises(LaneError, match="symlink"): + runner.discover_test_modules(tests, tmp_path) + + +@pytest.mark.parametrize( + ("mutation", "error"), + (("missing", "missing_lane_modules"), ("duplicate", "duplicate_lane_modules"), + ("foreign", "foreign_lane_modules"), ("unsafe", "invalid_lane_module"), + ("name", "invalid_lane_names")), +) +def test_inventory_fails_closed(mutation: str, error: str) -> None: + discovered = runner.discover_test_modules() + lanes = list(LANES) + first = lanes[0] + if mutation == "missing": + lanes[0] = replace(first, modules=first.modules[1:]) + elif mutation == "duplicate": + lanes[0] = replace(first, modules=(*first.modules, lanes[1].modules[0])) + elif mutation == "foreign": + lanes[0] = replace(first, modules=(*first.modules, "tests/test_foreign.py")) + elif mutation == "unsafe": + lanes[0] = replace(first, modules=(*first.modules, "../test_escape.py")) + else: + lanes[0] = replace(first, name="Invalid Lane") + with pytest.raises(LaneError, match=error): + runner.validate_lane_inventory(discovered, lanes=tuple(lanes)) + + +def test_manifest_contains_sorted_exact_node_ids() -> None: + nodes = [ + f"{LANES[0].modules[0]}::test_b[value::x]", + f"{LANES[0].modules[0]}::test_a", + ] + manifest = runner.build_manifest("a" * 40, sorted(nodes)) + + assert set(manifest) == {"schema_version", "head_sha", "nodes"} + assert manifest["head_sha"] == "a" * 40 + assert [row["nodeid"] for row in manifest["nodes"]] == sorted(nodes) + assert set(manifest["nodes"][0]) == {"execution_kind", "nodeid", "module", "lane"} + assert all(row["execution_kind"] == runner.ORDINARY_KIND for row in manifest["nodes"]) + + +def test_manifest_classifies_only_runner_self_tests_as_admin_kind() -> None: + ordinary = f"{LANES[1].modules[0]}::test_migration" + admin = f"{runner.ADMIN_RUNNER_MODULE}::test_admin_owner" + rows = runner.build_manifest("a" * 40, sorted((ordinary, admin)))["nodes"] + + assert {row["nodeid"]: row["execution_kind"] for row in rows} == { + admin: runner.ADMIN_KIND, + ordinary: runner.ORDINARY_KIND, + } + assert next(row for row in rows if row["nodeid"] == admin)["lane"] == "schema_contracts" + + +def test_manifest_has_no_exclusion_escape_hatch() -> None: + admin = f"{runner.ADMIN_RUNNER_MODULE}::test_admin_owner" + manifest = runner.build_manifest("a" * 40, [admin]) + + assert "excluded_modules" not in manifest + assert manifest["nodes"] == [{ + "execution_kind": runner.ADMIN_KIND, + "lane": "schema_contracts", + "module": runner.ADMIN_RUNNER_MODULE, + "nodeid": admin, + }] + + +def test_lane_command_uses_exact_nodes_and_isolation_contract(tmp_path: Path) -> None: + lane = LANES[0] + nodes = [f"{lane.modules[0]}::test_exact[param]"] + command = runner.lane_command(lane, nodes, tmp_path, 1200, "b" * 40) + + assert command[1].endswith("scripts/run_isolated_tests.py") + assert command[command.index("--lane") + 1] == lane.name + assert command[command.index("--tree-sha") + 1] == "b" * 40 + assert command[-1] == nodes[0] + assert "--cov=app" in command + assert "--cov-report=" in command + + +def test_lane_environment_uses_private_evidence_and_coverage(tmp_path: Path) -> None: + coverage = tmp_path / ".coverage.no_postgres" + env = runner.lane_environment(LANES[0], tmp_path, coverage) + + assert env["COVERAGE_FILE"] == str(coverage.resolve()) + paths = [Path(env[name]) for name in ( + runner.COLLECTED_ENV, runner.COMPLETED_ENV, runner.SKIPPED_ENV, runner.DESELECTED_ENV, + )] + assert len(set(paths)) == 4 + assert all(path.parent == tmp_path for path in paths) + + +def test_admin_runner_environment_retains_only_admin_database_url( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(runner.ADMIN_ENV, "postgresql+asyncpg://admin:secret@localhost/postgres") + monkeypatch.setenv("WORKSTREAM_DATABASE_URL", "postgresql+asyncpg://app:secret@localhost/app") + monkeypatch.setenv("WORKSTREAM_TEST_DATABASE_URL", "postgresql+asyncpg://test:secret@localhost/test") + lane = next(lane for lane in LANES if lane.name == "schema_contracts") + + env = runner.admin_runner_environment(lane, tmp_path, tmp_path / ".coverage", "admin") + + assert runner.ADMIN_ENV in env + assert "WORKSTREAM_DATABASE_URL" not in env + assert "WORKSTREAM_TEST_DATABASE_URL" not in env + command = runner.admin_runner_command([f"{runner.ADMIN_RUNNER_MODULE}::test_one"]) + assert "run_isolated_tests.py" not in " ".join(command) + + +def test_collect_only_writes_raw_digest_bound_validator_schema( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + modules = tuple(module for lane in LANES for module in lane.modules) + nodes = sorted(f"{module}::test_one" for module in modules) + monkeypatch.setattr(runner, "discover_test_modules", lambda: tuple(sorted(modules))) + monkeypatch.setattr(runner, "_tree_sha", lambda: "c" * 40) + monkeypatch.setattr(runner, "collect_nodes", lambda *_args: (0, nodes, [])) + metadata = tmp_path / "metadata" + summary_path = tmp_path / "summary.json" + + assert runner.run_lanes(metadata, summary_path, 10, collect_only=True) == 0 + summary = json.loads(summary_path.read_text(encoding="utf-8")) + manifest_bytes = (metadata / summary["manifest_file"]).read_bytes() + assert set(summary) == { + "canonical_node_count", "elapsed_seconds", "head_sha", "lanes", "manifest_file", + "manifest_sha256", "mode", "schema_version", + } + assert summary["mode"] == "collect" + assert summary["canonical_node_count"] == len(nodes) + assert summary["manifest_sha256"] == hashlib.sha256(manifest_bytes).hexdigest() + assert len(summary["lanes"]) == 4 + for lane in summary["lanes"]: + assert lane["coverage_file"] is None + assert lane["execution_exit_code"] is None + evidence = metadata / lane["evidence_file"] + assert lane["evidence_sha256"] == hashlib.sha256(evidence.read_bytes()).hexdigest() + + +def test_collection_rejects_duplicate_or_foreign_nodes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = "tests/test_alpha.py" + + class Result: + returncode = 0 + + def fake_run(*_args, **kwargs): + collected = Path(kwargs["env"][runner.COLLECTED_ENV]) + collected.write_text('"tests/test_foreign.py::test_x"\n', encoding="utf-8") + return Result() + + tmp_path.joinpath("metadata").mkdir() + monkeypatch.setattr(runner.subprocess, "run", fake_run) + with pytest.raises(LaneError, match="invalid_collected_nodes"): + runner.collect_nodes((module,), tmp_path / "metadata") + + +def test_failure_interrupts_sibling_process_groups_and_records_all_lanes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + lanes = tuple( + LaneDefinition(f"lane_{index}", (f"tests/test_{index}.py",)) for index in range(4) + ) + modules = tuple(lane.modules[0] for lane in lanes) + nodes = [f"{lane.modules[0]}::test_one" for lane in lanes] + monkeypatch.setattr(runner, "LANES", lanes) + monkeypatch.setattr(runner, "discover_test_modules", lambda: modules) + monkeypatch.setattr(runner, "_tree_sha", lambda: "d" * 40) + monkeypatch.setattr(runner, "collect_nodes", lambda *_args: (0, nodes, [])) + monkeypatch.setenv(runner.ADMIN_ENV, "postgresql+asyncpg://admin@localhost/postgres") + monkeypatch.setattr(runner, "CLEANUP_GRACE_SECONDS", 0.2) + + def fake_command(lane, _nodes, metadata, _timeout, _sha): + database = metadata / f"{lane.name}.database.json" + database.write_text("{}\n", encoding="utf-8") + coverage = metadata / f".coverage.{lane.name}" + coverage.write_bytes(b"coverage") + if lane.name == "lane_0": + return [sys.executable, "-c", "raise SystemExit(1)"] + return [sys.executable, "-c", "import time; time.sleep(30)"] + + monkeypatch.setattr(runner, "lane_command", fake_command) + metadata = tmp_path / "metadata" + summary_path = tmp_path / "summary.json" + started = time.monotonic() + result = runner.run_lanes(metadata, summary_path, 5) + summary = json.loads(summary_path.read_text(encoding="utf-8")) + + assert result == 1 + assert time.monotonic() - started < 3 + assert len(summary["lanes"]) == 4 + assert summary["lanes"][0]["execution_exit_code"] == 1 + assert all(row["interrupted"] for row in summary["lanes"][1:]) diff --git a/backend/tests/test_ci_test_shards.py b/backend/tests/test_ci_test_shards.py deleted file mode 100644 index d2506a40..00000000 --- a/backend/tests/test_ci_test_shards.py +++ /dev/null @@ -1,448 +0,0 @@ -"""Tests for deterministic backend CI shard evidence.""" - -from __future__ import annotations - -import importlib.util -import json -from pathlib import Path - -import pytest - - -SCRIPT = Path(__file__).resolve().parents[1] / "scripts/ci_test_shards.py" -SPEC = importlib.util.spec_from_file_location("ci_test_shards", SCRIPT) -assert SPEC is not None and SPEC.loader is not None -shards = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(shards) - -TREE_SHA = "a" * 40 - - -def _nodes() -> dict[str, list[str]]: - return { - "backend/tests/test_alpha.py": [ - "tests/test_alpha.py::test_a", - "tests/test_alpha.py::test_b[value]", - "tests/test_alpha.py::test_c", - "tests/test_alpha.py::test_d", - ], - "backend/tests/test_beta.py": [ - "tests/test_beta.py::test_a", - "tests/test_beta.py::test_b", - "tests/test_beta.py::test_c", - ], - "backend/tests/test_delta.py": ["tests/test_delta.py::test_a"], - "backend/tests/test_epsilon.py": ["tests/test_epsilon.py::test_a"], - "backend/tests/test_gamma.py": [ - "tests/test_gamma.py::test_a", - "tests/test_gamma.py::test_b", - ], - } - - -def _write_bundle_set(root: Path, manifest: dict) -> None: - root.mkdir() - modules = { - row["path"]: [ - f"{item['base']}[runtime-{index}]" if item["count"] > 1 else item["base"] - for item in row["node_signature"] - for index in range(item["count"]) - ] - for row in manifest["modules"] - } - for shard in manifest["shards"]: - bundle = root / f"shard-{shard['id']}" - bundle.mkdir() - coverage = bundle / "coverage.data" - coverage.write_bytes(f"coverage-{shard['id']}".encode()) - observed = sorted(node for module in shard["modules"] for node in modules[module]) - result = { - "coverage_file": "coverage.data", - "coverage_sha256": shards._sha256(coverage.read_bytes()), - "duration_seconds": 1.25, - "manifest_sha256": manifest["manifest_sha256"], - "modules": shard["modules"], - "collected_node_ids": observed, - "completed_node_ids": observed, - "schema_version": shards.SCHEMA_VERSION, - "shard_id": shard["id"], - "tree_sha": manifest["tree_sha"], - } - (bundle / "result.json").write_text( - json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" - ) - - -def test_manifest_is_deterministic_and_balanced() -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - assert manifest == shards.build_manifest(TREE_SHA, dict(reversed(_nodes().items())), 4) - assert [row["weight"] for row in manifest["shards"]] == [4, 3, 2, 2] - assigned = [module for row in manifest["shards"] for module in row["modules"]] - assert sorted(assigned) == sorted(_nodes()) - assert len(assigned) == len(set(assigned)) - assert shards.validate_manifest(manifest) == manifest - - -@pytest.mark.parametrize( - ("tree_sha", "nodes", "count", "message"), - [ - ("bad", _nodes(), 4, "invalid_tree_sha"), - (TREE_SHA, {}, 4, "invalid_module_inventory"), - (TREE_SHA, _nodes(), 3, "invalid_shard_count"), - ( - TREE_SHA, - {"backend/tests/test_zero.py": []}, - 4, - "invalid_node_inventory", - ), - ( - TREE_SHA, - {"backend/tests/test_alpha.py": ["tests/test_other.py::test_a"]}, - 4, - "node_module_mismatch", - ), - ( - TREE_SHA, - {"backend/tests/../test_escape.py": ["tests/test_escape.py::test_a"]}, - 4, - "invalid_test_module", - ), - ], -) -def test_manifest_rejects_invalid_inventory( - tree_sha: str, nodes: dict[str, list[str]], count: int, message: str -) -> None: - with pytest.raises(shards.ShardError, match=message): - shards.build_manifest(tree_sha, nodes, count) - - -def test_manifest_rejects_digest_or_assignment_tampering() -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - manifest["shards"][0]["modules"].append("backend/tests/test_foreign.py") - with pytest.raises(shards.ShardError, match="manifest_digest_mismatch"): - shards.validate_manifest(manifest) - - -def test_discovery_rejects_symlinked_module(tmp_path: Path) -> None: - tests = tmp_path / "backend/tests" - tests.mkdir(parents=True) - target = tmp_path / "target.py" - target.write_text("def test_a(): pass\n", encoding="utf-8") - (tests / "test_link.py").symlink_to(target) - with pytest.raises(shards.ShardError, match="invalid_test_module"): - shards.discover_modules(tmp_path) - - -def test_collect_nodes_rejects_collection_failure( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - class Result: - returncode = 2 - stdout = "" - stderr = "failure" - - monkeypatch.setattr(shards.subprocess, "run", lambda *args, **kwargs: Result()) - with pytest.raises(shards.ShardError, match="pytest_collection_failed"): - shards.collect_nodes(tmp_path, ["backend/tests/test_alpha.py"]) - - -def test_collect_nodes_rejects_zero_or_foreign_nodes( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - class Result: - returncode = 0 - stdout = "tests/test_other.py::test_a\n" - stderr = "" - - monkeypatch.setattr(shards.subprocess, "run", lambda *args, **kwargs: Result()) - with pytest.raises(shards.ShardError, match="foreign_or_duplicate_node"): - shards.collect_nodes(tmp_path, ["backend/tests/test_alpha.py"]) - - -def test_collect_nodes_accepts_parameterized_nodes_and_sets_safe_environment( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - observed: dict = {} - - class Result: - returncode = 0 - stdout = ( - "tests/test_alpha.py::test_a[value]\ntests/test_alpha.py::test_b\n2 tests collected\n" - ) - stderr = "" - - def fake_run(command, **kwargs): - observed["command"] = command - observed["env"] = kwargs["env"] - return Result() - - monkeypatch.setattr(shards.subprocess, "run", fake_run) - result = shards.collect_nodes(tmp_path, ["backend/tests/test_alpha.py"]) - assert result == { - "backend/tests/test_alpha.py": [ - "tests/test_alpha.py::test_a[value]", - "tests/test_alpha.py::test_b", - ] - } - assert observed["command"][-1] == "tests/test_alpha.py" - assert observed["env"]["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" - - -def test_discovery_accepts_regular_modules_and_explicit_exclusion(tmp_path: Path) -> None: - tests = tmp_path / "backend/tests" - tests.mkdir(parents=True) - (tests / "test_alpha.py").write_text("def test_a(): pass\n", encoding="utf-8") - (tests / "test_beta.py").write_text("def test_b(): pass\n", encoding="utf-8") - (tests / "test_isolated_database_runner.py").write_text( - "def test_runner(): pass\n", encoding="utf-8" - ) - assert shards.discover_modules(tmp_path) == [ - "backend/tests/test_alpha.py", - "backend/tests/test_beta.py", - ] - - -def test_manifest_file_round_trip_and_invalid_json(tmp_path: Path) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - path = tmp_path / "manifest.json" - path.write_text(json.dumps(manifest), encoding="utf-8") - assert shards.load_manifest(path) == manifest - path.write_text("not-json", encoding="utf-8") - with pytest.raises(shards.ShardError, match="invalid_manifest_file"): - shards.load_manifest(path) - - -@pytest.mark.parametrize( - ("field", "value"), - [ - ("schema_version", 99), - ("shard_count", True), - ("manifest_sha256", "bad"), - ("excluded_modules", []), - ("shards", "bad"), - ], -) -def test_manifest_rejects_invalid_top_level_schema(field: str, value: object) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - manifest[field] = value - with pytest.raises(shards.ShardError, match="invalid_manifest"): - shards.validate_manifest(manifest) - - -def test_run_shard_uses_python_argv_and_writes_authenticated_bundle( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - manifest_path = tmp_path / "manifest.json" - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - shard = manifest["shards"][0] - module_nodes = {module: nodes for module, nodes in _nodes().items()} - monkeypatch.setattr(shards, "_assert_checked_out_tree", lambda *args: None) - captured: dict = {} - - class Result: - returncode = 0 - - def fake_run(command, **kwargs): - captured["command"] = command - captured["env"] = kwargs["env"] - Path(kwargs["env"]["COVERAGE_FILE"]).write_bytes(b"real-coverage") - runtime = sorted(node for module in shard["modules"] for node in module_nodes[module]) - for variable in (shards.COLLECTED_NODES_ENV, shards.COMPLETED_NODES_ENV): - Path(kwargs["env"][variable]).write_text( - "".join(json.dumps(node) + "\n" for node in runtime), encoding="utf-8" - ) - return Result() - - monkeypatch.setattr(shards.subprocess, "run", fake_run) - bundle = tmp_path / "bundle" - shards.run_shard( - tmp_path, - manifest_path, - shard["id"], - bundle, - tmp_path / "database.json", - ) - result = json.loads((bundle / "result.json").read_text(encoding="utf-8")) - assert captured["command"][:2] == [shards.sys.executable, "scripts/run_isolated_tests.py"] - invoked_modules = [ - argument for argument in captured["command"] if argument.startswith("tests/") - ] - assert invoked_modules == [ - str(Path(module).relative_to("backend")) for module in shard["modules"] - ] - assert "scripts.ci_test_shards" in captured["command"] - assert captured["env"]["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] == "1" - assert result["modules"] == shard["modules"] - assert result["coverage_sha256"] == shards._sha256(b"real-coverage") - - -def test_run_shard_rejects_failed_test_process( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - path = tmp_path / "manifest.json" - path.write_text(json.dumps(manifest), encoding="utf-8") - shard = manifest["shards"][0] - monkeypatch.setattr(shards, "_assert_checked_out_tree", lambda *args: None) - - class Result: - returncode = 1 - - monkeypatch.setattr(shards.subprocess, "run", lambda *args, **kwargs: Result()) - with pytest.raises(shards.ShardError, match="shard_tests_failed"): - shards.run_shard(tmp_path, path, shard["id"], tmp_path / "bundle", tmp_path / "db") - - -def test_run_shard_rejects_collected_but_not_executed_node( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - path = tmp_path / "manifest.json" - path.write_text(json.dumps(manifest), encoding="utf-8") - shard = manifest["shards"][0] - module_nodes = _nodes() - monkeypatch.setattr(shards, "_assert_checked_out_tree", lambda *args: None) - - class Result: - returncode = 0 - - def fake_run(command, **kwargs): - Path(kwargs["env"]["COVERAGE_FILE"]).write_bytes(b"coverage") - expected = sorted(node for module in shard["modules"] for node in module_nodes[module]) - Path(kwargs["env"][shards.COLLECTED_NODES_ENV]).write_text( - "".join(json.dumps(node) + "\n" for node in expected), - encoding="utf-8", - ) - Path(kwargs["env"][shards.COMPLETED_NODES_ENV]).write_text( - "".join(json.dumps(node) + "\n" for node in expected[1:]), - encoding="utf-8", - ) - return Result() - - monkeypatch.setattr(shards.subprocess, "run", fake_run) - with pytest.raises(shards.ShardError, match="shard_execution_mismatch"): - shards.run_shard(tmp_path, path, shard["id"], tmp_path / "bundle", tmp_path / "db") - - -def test_pytest_hooks_record_runtime_collection_and_completion( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - collected = tmp_path / "collected.jsonl" - completed = tmp_path / "completed.jsonl" - collected.write_text("", encoding="utf-8") - completed.write_text("", encoding="utf-8") - monkeypatch.setenv(shards.COLLECTED_NODES_ENV, str(collected)) - monkeypatch.setenv(shards.COMPLETED_NODES_ENV, str(completed)) - session = type( - "Session", (), {"items": [type("Item", (), {"nodeid": "tests/test_alpha.py::test_a"})()]} - )() - shards.pytest_collection_finish(session) - shards.pytest_runtest_logfinish("tests/test_alpha.py::test_a", ("file", 1, "test_a")) - assert collected.read_text(encoding="utf-8") == '"tests/test_alpha.py::test_a"\n' - assert completed.read_text(encoding="utf-8") == '"tests/test_alpha.py::test_a"\n' - - -def test_runtime_signature_tolerates_changed_parameter_display_values() -> None: - before = ["tests/test_alpha.py::test_a[550e8400::nested[value]]"] - after = ["tests/test_alpha.py::test_a[6ba7b810::other[value]]"] - assert shards._node_signature(before) == shards._node_signature(after) - first = shards.build_manifest( - TREE_SHA, - { - "backend/tests/test_alpha.py": before, - **{ - key: value - for key, value in _nodes().items() - if key != "backend/tests/test_alpha.py" - }, - }, - 4, - ) - second = shards.build_manifest( - TREE_SHA, - { - "backend/tests/test_alpha.py": after, - **{ - key: value - for key, value in _nodes().items() - if key != "backend/tests/test_alpha.py" - }, - }, - 4, - ) - assert first == second - - -def test_fan_in_accepts_exact_authenticated_bundles(tmp_path: Path) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - bundles = tmp_path / "bundles" - _write_bundle_set(bundles, manifest) - summary_path = tmp_path / "fan-in-summary.json" - outputs = shards.validate_fan_in(manifest, bundles, tmp_path / "combined", summary_path) - assert [path.name for path in outputs] == [ - ".coverage.shard-1", - ".coverage.shard-2", - ".coverage.shard-3", - ".coverage.shard-4", - ] - summary = json.loads(summary_path.read_text(encoding="utf-8")) - assert summary["tree_sha"] == TREE_SHA - assert summary["manifest_sha256"] == manifest["manifest_sha256"] - assert summary["node_count"] == sum(len(nodes) for nodes in _nodes().values()) - assert [row["node_count"] for row in summary["shards"]] == [4, 3, 2, 2] - assert summary["timing"] == { - "imbalance_seconds": 0.0, - "maximum_seconds": 1.25, - "total_runner_seconds": 5.0, - } - - -@pytest.mark.parametrize("mutation", ["missing", "extra", "coverage", "nodes", "tree"]) -def test_fan_in_rejects_incomplete_or_tampered_evidence(tmp_path: Path, mutation: str) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - bundles = tmp_path / "bundles" - _write_bundle_set(bundles, manifest) - if mutation == "missing": - (bundles / "shard-1/result.json").unlink() - elif mutation == "extra": - (bundles / "surplus").mkdir() - elif mutation == "coverage": - (bundles / "shard-1/coverage.data").write_bytes(b"changed") - else: - path = bundles / "shard-1/result.json" - result = json.loads(path.read_text(encoding="utf-8")) - if mutation == "nodes": - result["completed_node_ids"] = result["completed_node_ids"][1:] - else: - result["tree_sha"] = "b" * 40 - path.write_text(json.dumps(result), encoding="utf-8") - with pytest.raises(shards.ShardError): - shards.validate_fan_in(manifest, bundles, tmp_path / "combined") - - -def test_fan_in_rejects_symlink_and_unexpected_file(tmp_path: Path) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - bundles = tmp_path / "bundles" - _write_bundle_set(bundles, manifest) - (bundles / "shard-1/unexpected").write_text("no", encoding="utf-8") - with pytest.raises(shards.ShardError, match="unexpected_bundle_path"): - shards.validate_fan_in(manifest, bundles, tmp_path / "combined") - (bundles / "shard-1/unexpected").unlink() - coverage = bundles / "shard-1/coverage.data" - coverage.unlink() - coverage.symlink_to(bundles / "shard-2/coverage.data") - with pytest.raises(shards.ShardError, match="unexpected_bundle_path"): - shards.validate_fan_in(manifest, bundles, tmp_path / "combined") - - -def test_fan_in_rejects_runtime_collection_completion_mismatch(tmp_path: Path) -> None: - manifest = shards.build_manifest(TREE_SHA, _nodes(), 4) - bundles = tmp_path / "bundles" - _write_bundle_set(bundles, manifest) - second = bundles / "shard-2/result.json" - two = json.loads(second.read_text(encoding="utf-8")) - two["completed_node_ids"] = two["completed_node_ids"][1:] - second.write_text(json.dumps(two), encoding="utf-8") - with pytest.raises(shards.ShardError, match="shard_node_mismatch"): - shards.validate_fan_in(manifest, bundles, tmp_path / "combined") diff --git a/backend/tests/test_isolated_database_runner.py b/backend/tests/test_isolated_database_runner.py index 2f3198b2..628a3642 100644 --- a/backend/tests/test_isolated_database_runner.py +++ b/backend/tests/test_isolated_database_runner.py @@ -53,6 +53,7 @@ async def noop(*_): return None async def observed(*_): return "0015" calls = iter([(0, b"", b""), (0, b"", b"")]) monkeypatch.setenv(ADMIN_ENV, "postgresql+asyncpg://local@localhost/postgres") + monkeypatch.delenv(runner.MINIO_ENDPOINT_ENV, raising=False) monkeypatch.setattr(runner, "_urls", lambda _: ("workstream_test_012345abcdef", "workstream_role_012345abcdef", "password", "target")) monkeypatch.setattr(runner, "_create", noop) monkeypatch.setattr(runner, "_observed_head", observed) @@ -62,6 +63,127 @@ async def observed(*_): return "0015" monkeypatch.setattr(sys, "argv", [str(RUNNER), "--metadata-json", str(tmp_path / "db.json"), "--", "child"]) +def test_lane_namespaces_bind_real_s3_traffic_and_separate_other_lanes() -> None: + """Only the S3 lane receives the application's hardcoded integration bucket.""" + s3_bucket, s3_prefix = runner._minio_namespace("no_postgres", "012345abcdef") + control_bucket, control_prefix = runner._minio_namespace( + "control_plane", "012345abcdef" + ) + execution_bucket, execution_prefix = runner._minio_namespace( + "execution_plane", "fedcba543210" + ) + assert (s3_bucket, s3_prefix) == ( + "workstream-artifacts", + "ci/no_postgres/012345abcdef", + ) + assert len({s3_bucket, control_bucket, execution_bucket}) == 3 + assert len({s3_prefix, control_prefix, execution_prefix}) == 3 + with pytest.raises(runner.RunnerError, match="invalid_lane"): + runner._minio_namespace("../foreign", "012345abcdef") + + +def test_child_environment_binds_lane_namespace_without_admin_custody() -> None: + """Children receive resource targets but never the destructive admin authority.""" + env = runner._child_env( + "postgresql+asyncpg://role:password@localhost/database", + minio_bucket="workstream-artifacts", + minio_prefix="ci/no_postgres/012345abcdef", + ) + assert env["WORKSTREAM_TEST_MINIO_BUCKET"] == "workstream-artifacts" + assert env["WORKSTREAM_TEST_MINIO_PREFIX"] == "ci/no_postgres/012345abcdef" + assert runner.ADMIN_ENV not in env + assert runner.OVERRIDE_ENV not in env + + +@pytest.mark.parametrize( + "endpoint", + [ + "https://localhost:9000", + "http://user:secret@localhost:9000", + "http://localhost:9000/foreign", + "http://minio.example.com:9000", + "http://localhost:not-a-port", + ], +) +def test_minio_boundary_rejects_nonlocal_or_credentialed_endpoints(endpoint: str) -> None: + """Cleanup authority is restricted to one uncredentialed loopback origin.""" + with pytest.raises(runner.RunnerError, match="unsafe_minio_endpoint"): + runner._minio_client(endpoint) + + +def test_metadata_is_private_deterministic_and_refuses_symlinks(tmp_path: Path) -> None: + """Custody evidence is stable, mode 0600, and never follows an output symlink.""" + path = tmp_path / "runner.json" + runner._write_metadata(path, {"tree_sha": "1" * 40, "lane": "control_plane"}) + assert path.stat().st_mode & 0o777 == 0o600 + assert path.read_text(encoding="utf-8") == ( + '{\n "lane": "control_plane",\n "tree_sha": "' + "1" * 40 + '"\n}\n' + ) + path.unlink() + path.symlink_to(tmp_path / "outside.json") + with pytest.raises(OSError): + runner._write_metadata(path, {"secret": "must-not-land"}) + assert not (tmp_path / "outside.json").exists() + + +def test_run_emits_secret_free_heartbeat_and_uses_bounded_timeout_cleanup( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Liveness contains only elapsed time and timeout escalates TERM to KILL.""" + class Process: + pid = 123 + returncode = 0 + + def __init__(self) -> None: + self.calls = 0 + + def communicate(self, timeout=None): + self.calls += 1 + if self.calls < 3: + raise runner.TimeoutExpired("private-command", timeout) + return b"private stdout", b"private stderr" + + process = Process() + signals: list[int] = [] + clock = iter((0.0, 0.0, 60.0, 120.0)) + monkeypatch.setattr(runner.subprocess, "Popen", lambda *args, **kwargs: process) + monkeypatch.setattr(runner.time, "monotonic", lambda: next(clock)) + monkeypatch.setattr(runner, "_signal_group", lambda _process, value: signals.append(value)) + code, stdout, stderr = runner._run(["private-command"], {}, 120.0) + assert (code, stdout, stderr) == (124, b"private stdout", b"private stderr") + assert signals == [signal.SIGTERM, signal.SIGKILL] + heartbeat = capsys.readouterr().out + assert heartbeat == "isolated-test child active: elapsed_seconds=60\n" + assert "private" not in heartbeat + + +def test_tree_binding_rejects_foreign_expected_head( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A lane cannot provision resources for metadata naming another commit.""" + monkeypatch.setenv(ADMIN_ENV, "postgresql+asyncpg://local@localhost/postgres") + monkeypatch.delenv(runner.MINIO_ENDPOINT_ENV, raising=False) + monkeypatch.setattr(runner, "_tree_sha", lambda: "1" * 40) + monkeypatch.setattr( + sys, + "argv", + [ + str(RUNNER), + "--metadata-json", + str(tmp_path / "db.json"), + "--lane", + "control_plane", + "--tree-sha", + "2" * 40, + "--", + "child", + ], + ) + assert runner.main() == 2 + assert capsys.readouterr().err == "isolated-test runner failed: tree_sha_mismatch\n" + assert not (tmp_path / "db.json").exists() + + def test_runner_rejects_nonlocal_admin_without_exposing_it( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py new file mode 100644 index 00000000..7ccf5e16 --- /dev/null +++ b/backend/tests/test_test_lane_evidence.py @@ -0,0 +1,296 @@ +"""Tests for independent semantic test-lane evidence validation.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +from pathlib import Path + +import pytest + + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts/validate_test_lane_evidence.py" +SPEC = importlib.util.spec_from_file_location("validate_test_lane_evidence", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +validator = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(validator) +HEAD = "a" * 40 +LANES = ("no_postgres", "schema_contracts", "control_plane", "execution_plane") + + +def _write(path: Path, value: object) -> str: + data = json.dumps(value, sort_keys=True).encode() + path.write_bytes(data) + return hashlib.sha256(data).hexdigest() + + +def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: + metadata = tmp_path / "metadata" + metadata.mkdir(parents=True) + nodes = [ + { + "execution_kind": "ordinary_isolated", + "lane": lane, + "module": f"tests/test_{index}.py", + "nodeid": f"tests/test_{index}.py::test_ok", + } + for index, lane in enumerate(LANES) + ] + nodes.append( + { + "execution_kind": "admin_runner_self_test", + "lane": "schema_contracts", + "module": "tests/test_isolated_database_runner.py", + "nodeid": "tests/test_isolated_database_runner.py::test_admin_custody", + } + ) + nodes.sort(key=lambda row: row["nodeid"]) + manifest_name = "manifest.json" + manifest_digest = _write( + metadata / manifest_name, + {"head_sha": HEAD, "nodes": nodes, "schema_version": 1}, + ) + lane_rows = [] + for lane in LANES: + nodeids = [row["nodeid"] for row in nodes if row["lane"] == lane] + isolation_file = None + isolation_digest = None + coverage_file = None + coverage_digest = None + if mode == "run": + isolation_file = f"{lane}.isolation.json" + isolation_digest = _write( + metadata / isolation_file, + { + "alembic_head": "head", + "cleanup_complete": True, + "database_cleanup_complete": True, + "database_name": f"database_{lane}", + "database_provisioned": True, + "database_role": f"role_{lane}", + "lane": lane, + "minio_bucket": f"bucket-{lane.replace('_', '-')}", + "minio_cleanup_complete": True, + "minio_prefix": f"prefix/{lane}", + "minio_probe_complete": True, + "minio_provisioned": True, + "schema_version": 2, + "tree_sha": HEAD, + }, + ) + coverage_file = f"coverage.{lane}" + (metadata / coverage_file).write_bytes(f"coverage:{lane}".encode()) + coverage_digest = hashlib.sha256((metadata / coverage_file).read_bytes()).hexdigest() + evidence_file = f"{lane}.evidence.json" + evidence_digest = _write( + metadata / evidence_file, + { + "collected_nodes": nodeids, + "completed_nodes": nodeids if mode == "run" else [], + "deselected_nodes": [], + "isolation_metadata_file": isolation_file, + "isolation_metadata_sha256": isolation_digest, + "skipped_nodes": [], + }, + ) + lane_rows.append( + { + "collection_exit_code": 0, + "coverage_file": coverage_file, + "coverage_sha256": coverage_digest, + "elapsed_seconds": 1.0, + "evidence_file": evidence_file, + "evidence_sha256": evidence_digest, + "execution_exit_code": 0 if mode == "run" else None, + "interrupted": False, + "name": lane, + } + ) + summary = { + "canonical_node_count": 5, + "elapsed_seconds": 2.0, + "head_sha": HEAD, + "lanes": lane_rows, + "manifest_file": manifest_name, + "manifest_sha256": manifest_digest, + "mode": mode, + "schema_version": 1, + } + summary_path = tmp_path / "summary.json" + _write(summary_path, summary) + return metadata, summary_path, summary + + +@pytest.fixture(autouse=True) +def exact_head(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(validator, "_current_head", lambda _root: HEAD) + + +@pytest.mark.parametrize("mode", ["collect", "run"]) +def test_validates_complete_exact_custody(tmp_path: Path, mode: str) -> None: + metadata, summary_path, summary = _bundle(tmp_path, mode) + assert validator.validate_evidence(metadata, summary_path, tmp_path) == summary + + +def test_rejects_wrong_head_and_noncanonical_manifest(tmp_path: Path) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + summary["head_sha"] = "b" * 40 + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="head_sha_mismatch"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + metadata, summary_path, summary = _bundle(tmp_path / "second") + manifest = json.loads((metadata / "manifest.json").read_text()) + manifest["nodes"].reverse() + summary["manifest_sha256"] = _write(metadata / "manifest.json", manifest) + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="noncanonical"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda evidence: evidence["collected_nodes"].clear(), "collected_node_reconciliation"), + ( + lambda evidence: evidence["collected_nodes"].append("tests/test_0.py::test_ok"), + "collected_node_reconciliation", + ), + ( + lambda evidence: evidence["collected_nodes"].append("tests/test_foreign.py::test_bad"), + "collected_node_reconciliation", + ), + (lambda evidence: evidence["completed_nodes"].clear(), "partial_or_duplicate_completion"), + ( + lambda evidence: evidence["skipped_nodes"].append(evidence["collected_nodes"][0]), + "unexpected_skipped", + ), + ( + lambda evidence: evidence["deselected_nodes"].append(evidence["collected_nodes"][0]), + "unexpected_deselected", + ), + ], +) +def test_rejects_node_custody_failures(tmp_path: Path, mutation, message: str) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + lane = summary["lanes"][0] + evidence_path = metadata / lane["evidence_file"] + evidence = json.loads(evidence_path.read_text()) + mutation(evidence) + lane["evidence_sha256"] = _write(evidence_path, evidence) + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match=message): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("collection_exit_code", 1, "collection_failed"), + ("execution_exit_code", None, "execution_incomplete"), + ("interrupted", True, "lane_interrupted"), + ], +) +def test_rejects_failed_or_partial_lane( + tmp_path: Path, field: str, value: object, message: str +) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + summary["lanes"][0][field] = value + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match=message): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +def test_rejects_digest_tampering_and_unsafe_paths(tmp_path: Path) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + (metadata / summary["lanes"][0]["coverage_file"]).write_bytes(b"tampered") + with pytest.raises(validator.EvidenceError, match="evidence_digest_mismatch"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + metadata, summary_path, summary = _bundle(tmp_path / "link-case") + target = metadata / summary["lanes"][0]["evidence_file"] + real = metadata / "real.json" + target.rename(real) + target.symlink_to(real) + with pytest.raises(validator.EvidenceError, match="unsafe_evidence_path"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +def test_rejects_wrong_lane_count_zero_nodes_and_unknown_keys(tmp_path: Path) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + summary["lanes"].pop() + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="invalid_lane_count"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + metadata, summary_path, summary = _bundle(tmp_path / "zero") + manifest = {"head_sha": HEAD, "nodes": [], "schema_version": 1} + summary["manifest_sha256"] = _write(metadata / "manifest.json", manifest) + summary["canonical_node_count"] = 0 + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="zero_canonical_nodes"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + metadata, summary_path, summary = _bundle(tmp_path / "keys") + summary["unexpected"] = True + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="invalid_summary"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + ( + lambda rows: rows.__setitem__( + slice(None), + [row for row in rows if row["module"] != validator.ADMIN_RUNNER_MODULE], + ), + "missing_admin_runner_self_tests", + ), + ( + lambda rows: rows[-1].__setitem__("execution_kind", validator.ORDINARY_KIND), + "invalid_manifest_node", + ), + ( + lambda rows: rows[0].__setitem__("execution_kind", validator.ADMIN_KIND), + "invalid_manifest_node", + ), + (lambda rows: rows.append(dict(rows[-1])), "noncanonical_or_duplicate"), + ], +) +def test_rejects_missing_duplicate_or_wrong_kind_admin_custody( + tmp_path: Path, mutate, message: str +) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + manifest_path = metadata / summary["manifest_file"] + manifest = json.loads(manifest_path.read_text()) + mutate(manifest["nodes"]) + manifest["nodes"].sort(key=lambda row: row["nodeid"]) + summary["canonical_node_count"] = len(manifest["nodes"]) + summary["manifest_sha256"] = _write(manifest_path, manifest) + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match=message): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +def test_rejects_recorded_database_environment_or_shared_coverage(tmp_path: Path) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + lane = summary["lanes"][0] + evidence = json.loads((metadata / lane["evidence_file"]).read_text()) + isolation_path = metadata / evidence["isolation_metadata_file"] + isolation = json.loads(isolation_path.read_text()) + isolation["admin_database_url"] = "postgresql://admin:secret@example.invalid/db" + evidence["isolation_metadata_sha256"] = _write(isolation_path, isolation) + lane["evidence_sha256"] = _write(metadata / lane["evidence_file"], evidence) + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="invalid_isolation_metadata"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + metadata, summary_path, summary = _bundle(tmp_path / "shared") + summary["lanes"][1]["coverage_file"] = summary["lanes"][0]["coverage_file"] + summary["lanes"][1]["coverage_sha256"] = summary["lanes"][0]["coverage_sha256"] + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="shared_lane_artifact"): + validator.validate_evidence(metadata, summary_path, tmp_path) diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index 3fb88f93..3a802d88 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -17,7 +17,9 @@ export WORKSTREAM_TEST_ADMIN_DATABASE_URL='postgresql+asyncpg://USER:PASSWORD@lo unset WORKSTREAM_TEST_ADMIN_DATABASE_URL ``` -Run both phases. The second can exceed three hours locally; CI gives the child 210 minutes and the job 240 minutes so cleanup retains a bounded window. +Run both phases for the legacy sequential local diagnostic. Hosted CI instead +uses four concurrent semantic lanes with a 20-minute lane limit inside a +45-minute job, leaving a bounded validation and cleanup window. The runner removes the admin URL before child launch, overwrites both child database URLs, removes the nonlocal override, redacts complete URLs, and writes only credential-free metadata. @@ -57,59 +59,62 @@ Do not use `WORKSTREAM_ALLOW_NONLOCAL_E2E_DATABASE` for ordinary proof. If provisioning fails, confirm the local PostgreSQL provisioning credential can create/drop databases and roles, terminate owned sessions, and reach the named admin database. Diagnostics omit credentials. -## Hosted parallel full-suite proof - -The required GitHub check remains `Backend / test`. It is the final fan-in for: - -1. `preflight`: evidence gate, lint, docstrings, isolated-runner test, exact test - collection, and deterministic four-shard plan; -2. `shards`: four independent jobs, each with its own digest-pinned PostgreSQL - service, runner-owned migrated database, real digest-pinned MinIO, and - coverage file; -3. `api_e2e`: the real API contract proof in a separate isolated database; and -4. `test`: exact artifact validation, coverage combination, the 78 percent - repository floor, and all protected 90 percent subsystem floors. - -Matrix job state is the live progress view. The isolated runner continues to -buffer and redact pytest output, so a running shard does not stream individual -test names. The final check reports shard duration and balance metadata after -all evidence is authenticated. - -### Evidence bundles - -The preflight plan and four fixed shard bundles are retained for seven days. -Their names include the actual checked-out tree SHA. Each shard bundle contains -only `coverage.data` and allowlisted `result.json`; the result binds the tree, -manifest, shard, modules, collected and completed pytest node IDs, duration, and -SHA-256 of the exact coverage bytes. Repository-owned pytest hooks record the -final selected inventory and lifecycle completions in the same process; fan-in -requires exact equality and matches stable test-base cardinalities to preflight. -Bundles never contain database URLs or passwords, MinIO -credentials, environment dumps, or runner database metadata. - -The fan-in accepts exactly four expected regular-file bundles. It rejects stale -tree or manifest bindings, missing/extra/duplicate nodes or modules, altered -coverage, symlinks, path traversal, unexpected files, failed/cancelled/skipped -upstream jobs, and missing artifacts before `coverage combine` runs. +## Hosted semantic-lane full-suite proof + +The required GitHub check remains `Backend / test`. One job owns one +digest-pinned PostgreSQL service, one digest-pinned MinIO service, and four +concurrent dependency lanes. This avoids arbitrary shard fan-out and artifact +fan-in while retaining exact node and coverage custody. + +The job binds the checkout to `GITHUB_SHA`, installs and asserts exact Ruff +`0.15.22`, runs lint, docstrings, and the isolated-resource runner tests, then +collects every canonical pytest node. The independent evidence validator must +accept the collection before execution begins. Each lane receives a distinct +runner-created database and role plus a distinct MinIO bucket/prefix custody +record. The S3 lane owns the actual `workstream-artifacts` test bucket and a +unique run prefix; other lanes create, probe, and remove distinct buckets. +The isolated-runner self-tests remain in the canonical manifest as the explicit +`admin_runner_self_test` execution kind. The lane orchestrator runs only those +nodes directly with the admin URL while stripping application database URLs; +every ordinary node remains behind isolated-runner custody and never receives +the admin credential. + +After execution, independent validation rejects missing, duplicated, foreign, +deselected, unexpectedly skipped, interrupted, or partially completed nodes. +It also binds the exact head, manifest, per-lane isolation metadata, evidence, +and coverage-file SHA-256 digests. Only then are exactly four regular, +non-symlink coverage files copied byte-for-byte for one literal +`coverage combine`. The 78 percent global floor and every protected 90 percent +subsystem floor remain blocking. The real API contract drill remains a separate +isolated invocation inside the same required job. + +### Evidence bundle + +The workflow uploads the `.ci/test-lanes` tree even on failure. Its summary +records the exact head, canonical node count, four lane results, elapsed time, +and raw-file digests. Per-lane evidence records collected, completed, skipped, +and deselected exact node IDs plus the bound resource-isolation metadata and +coverage digest. Resource metadata is mode `0600`, omits credentials, and proves +database, role, bucket, prefix, probe, and cleanup custody. + +The validator accepts only safe repository-local regular files and exact schema +keys. It rejects symlinks, traversal, stale heads, digest drift, unexpected +lanes, zero collection, incomplete execution, resource cleanup failure, and +coverage tampering before coverage combination. ### Failure diagnosis and reruns -- `preflight` failure: inspect evidence/lint/runner/collection output. No shard - evidence is valid until preflight succeeds. -- one `shards` matrix failure: inspect that shard's database, MinIO, collection, - or test failure. The final required check must fail even if other shards pass. -- `api_e2e` failure: inspect the independent API contract job; coverage cannot - compensate for it. -- `test` failure: inspect dependency-result validation, exact bundle fan-in, then - the named global or subsystem coverage report. - -A complete workflow rerun creates evidence for the same checked-out tree and is -the clearest recovery. GitHub may rerun failed jobs, but the fan-in still rejects -missing or stale artifacts; never upload or edit bundles manually. A new commit -always requires a complete new run because its tree SHA differs. - -Four shards reduce wall-clock latency by using more concurrent runner minutes. -Review shard durations and total Actions consumption after deployment before -changing the shard count. If parallel execution is unstable or does not justify -its cost, revert the single implementation PR to restore the prior sequential -workflow; do not lower coverage, skip a shard, or add a silent fallback. +- Collection or collection-validation failure: inspect the canonical manifest, + lane assignment, and exact-head binding. No execution evidence is valid. +- Lane failure: inspect the named private log and evidence result; confirm its + database/role and MinIO namespace cleanup without exposing credentials. +- Execution-validation failure: inspect node reconciliation, isolation metadata, + and raw coverage digests before considering the test output. +- API contract or coverage failure: the required job remains failed; lane + completion cannot compensate for either boundary. + +Rerun the complete job on the same exact head. Never edit or upload evidence +manually. Every new commit requires a complete new run because its head and +digests differ. More than eight minutes remains a blocking measured outcome +unless the user explicitly accepts that exact-head timing risk; never lower +coverage, skip nodes, or add a silent fallback to meet the target. diff --git a/scripts/test_agent_gates.py b/scripts/test_agent_gates.py index 7f95e73e..f0d697d9 100644 --- a/scripts/test_agent_gates.py +++ b/scripts/test_agent_gates.py @@ -6424,7 +6424,7 @@ def test_local_minio_compose_is_regression_protected() -> None: def test_backend_coverage_thresholds_are_regression_protected() -> None: - """Keep parallel full-suite fan-in and every coverage floor fail closed.""" + """Keep exact-custody semantic lanes and every coverage floor fail closed.""" workflow_path = ROOT / ".github/workflows/backend.yml" workflow = workflow_path.read_text(encoding="utf-8") parsed_workflow = yaml.safe_load(workflow) @@ -6433,98 +6433,114 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert "pull_request_target" not in workflow assert "paths-ignore" not in workflow and "continue-on-error" not in workflow jobs = parsed_workflow["jobs"] - assert set(jobs) == {"preflight", "shards", "api_e2e", "test"} + assert set(jobs) == {"test"} postgres_image = ( "public.ecr.aws/docker/library/postgres:16@sha256:" "33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20" ) - for job_name in ("preflight", "shards", "api_e2e"): - assert jobs[job_name]["services"]["postgres"]["image"] == postgres_image - - preflight = jobs["preflight"] - assert set(preflight["outputs"]) == {"tree_sha"} - assert any( + test_job = jobs["test"] + assert set(test_job) == { + "runs-on", "timeout-minutes", "services", "steps", + } + assert test_job["services"]["postgres"]["image"] == postgres_image + steps = test_job["steps"] + assert not any( step.get("name") == "Isolated database runner test" - and step.get("run") == "python -m pytest -q tests/test_isolated_database_runner.py" - for step in preflight["steps"] + or "tests/test_isolated_database_runner.py" in str(step.get("run", "")) + for step in steps ) - plan_steps = [ - step for step in preflight["steps"] if step.get("name") == "Collect and plan exact test inventory" + checkout = [step for step in steps if "actions/checkout@" in step.get("uses", "")] + assert checkout == [{ + "uses": "actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5", + "with": {"persist-credentials": False, "fetch-depth": 0}, + }] + identity = [step for step in steps if step.get("name") == "Bind exact checked-out tree"] + assert len(identity) == 1 + assert '${tree_sha}' in str(identity[0]["run"]) + assert '${GITHUB_SHA}' in str(identity[0]["run"]) + + install = [step for step in steps if step.get("name") == "Install backend and exact Ruff"] + assert len(install) == 1 + assert "python -m pip install ruff==0.15.22" in str(install[0]["run"]) + assert 'test "$(ruff --version)" = "ruff 0.15.22"' in str(install[0]["run"]) + + collect = [ + step for step in steps + if step.get("name") == "Collect canonical semantic-lane inventory" ] - assert len(plan_steps) == 1 - assert "ci_test_shards.py plan" in str(plan_steps[0]["run"]) - assert "--shards 4" in str(plan_steps[0]["run"]) - - shard_job = jobs["shards"] - assert shard_job["needs"] == "preflight" - assert shard_job["strategy"] == { - "fail-fast": False, - "matrix": {"shard": [1, 2, 3, 4]}, - } - shard_steps = shard_job["steps"] + assert len(collect) == 1 + assert "run_test_lanes.py" in str(collect[0]["run"]) + assert "--collect-only" in str(collect[0]["run"]) + assert ".ci/test-lanes/collect-summary.json" in str(collect[0]["run"]) + validators = [ + step for step in steps + if "validate_test_lane_evidence.py" in str(step.get("run", "")) + ] + assert len(validators) == 2 + assert ".ci/test-lanes/collect-summary.json" in str(validators[0]["run"]) + assert ".ci/test-lanes/run-summary.json" in str(validators[1]["run"]) + minio_steps = [ - step for step in shard_steps if step.get("name") == "Start real MinIO artifact provider" + step for step in steps if step.get("name") == "Start real MinIO artifact provider" ] assert len(minio_steps) == 1 assert "${MINIO_IMAGE}" in str(minio_steps[0]["run"]) - run_shard_steps = [ - step for step in shard_steps if str(step.get("name", "")).startswith("Run isolated shard") + run_lane_steps = [ + step for step in steps if step.get("name") == "Execute four semantic lanes" ] - assert len(run_shard_steps) == 1 - assert "ci_test_shards.py run-shard" in str(run_shard_steps[0]["run"]) - assert "--cov-fail-under" not in str(run_shard_steps[0]["run"]) + assert len(run_lane_steps) == 1 + assert "run_test_lanes.py" in str(run_lane_steps[0]["run"]) + assert "--timeout-seconds 1200" in str(run_lane_steps[0]["run"]) + assert ".ci/test-lanes/run.exit" in str(run_lane_steps[0]["run"]) + assert "--cov-fail-under" not in str(run_lane_steps[0]["run"]) + assert run_lane_steps[0]["env"]["WORKSTREAM_TEST_ADMIN_DATABASE_URL"] == ( + "postgresql+asyncpg://workstream:workstream@localhost:5433/postgres" + ) + lane_runner = (ROOT / "backend/scripts/run_test_lanes.py").read_text( + encoding="utf-8" + ) + lane_validator = ( + ROOT / "backend/scripts/validate_test_lane_evidence.py" + ).read_text(encoding="utf-8") + assert "tests/test_isolated_database_runner.py" in lane_runner + assert "admin_runner_self_test" in lane_runner + assert "execution_kind" in lane_runner + assert "run_isolated_tests.py" in lane_runner + assert "admin_runner_self_test" in lane_validator + assert "execution_kind" in lane_validator - api_steps = jobs["api_e2e"]["steps"] api_e2e_steps = [ - step for step in api_steps if step.get("name") == "API contract real API e2e" + step for step in steps if step.get("name") == "API contract real API e2e" ] assert len(api_e2e_steps) == 1 assert "scripts/run_isolated_tests.py" in str(api_e2e_steps[0]["run"]) assert "scripts/api_contract_e2e.py" in str(api_e2e_steps[0]["run"]) - shard_tool = (ROOT / "backend/scripts/ci_test_shards.py").read_text(encoding="utf-8") - assert 4800 <= jobs["shards"]["timeout-minutes"] * 60 - 600 - assert '"4800"' in shard_tool - assert 1500 <= jobs["api_e2e"]["timeout-minutes"] * 60 - 300 + assert 1500 <= test_job["timeout-minutes"] * 60 - 300 assert "--timeout-seconds 1500" in str(api_e2e_steps[0]["run"]) - - test_job = parsed_workflow["jobs"]["test"] - assert set(test_job) == {"if", "needs", "runs-on", "timeout-minutes", "steps"} - assert test_job["if"] == "${{ always() }}" - assert test_job["needs"] == ["preflight", "shards", "api_e2e"] - steps = test_job["steps"] - upstream = [step for step in steps if step.get("name") == "Require every upstream proof"] - assert len(upstream) == 1 - assert upstream[0]["env"] == { - "PREFLIGHT_RESULT": "${{ needs.preflight.result }}", - "SHARDS_RESULT": "${{ needs.shards.result }}", - "API_E2E_RESULT": "${{ needs.api_e2e.result }}", - } - assert str(upstream[0]["run"]).count('= success') == 3 downloads = [step for step in steps if "actions/download-artifact@" in step.get("uses", "")] - assert len(downloads) == 5 - assert all( - step["uses"] - == "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093" - and "pattern" not in step.get("with", {}) - for step in downloads - ) + assert downloads == [] uploads = [ - step - for job in jobs.values() - for step in job["steps"] + step for step in steps if "actions/upload-artifact@" in step.get("uses", "") ] - assert len(uploads) == 3 - assert all( - step["uses"] - == "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - for step in uploads - ) - fan_in = [step for step in steps if step.get("name") == "Validate exact fan-in and combine coverage"] - assert len(fan_in) == 1 - assert "ci_test_shards.py fan-in" in str(fan_in[0]["run"]) - assert "coverage combine ../.ci/combined-coverage" in str(fan_in[0]["run"]) + assert len(uploads) == 1 + assert uploads[0]["uses"] == ( + "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + ) + assert uploads[0]["if"] == "${{ always() }}" + assert uploads[0]["with"]["path"] == "backend/.ci/test-lanes/**" + combines = [ + step for step in steps + if step.get("name") == "Combine semantic-lane coverage exactly once" + ] + assert len(combines) == 1 + combine_command = str(combines[0]["run"]) + assert combine_command.count("coverage combine") == 1 + assert 'test "${#coverage_files[@]}" -eq 4' in combine_command + assert "test ! -L" in combine_command + assert "sha256sum" in combine_command + assert steps.index(validators[1]) < steps.index(combines[0]) full_suite_steps = [ step for step in steps if step.get("name") == "Backend full-suite coverage" From b771c64a13a5ff18389fe572b99ae88a7548a2d3 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 16:14:46 +0100 Subject: [PATCH 02/31] chore(agent-loop): declare semantic lanes merge outcome --- .agent-loop/merge-intents/WS-CI-001-02B.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .agent-loop/merge-intents/WS-CI-001-02B.json diff --git a/.agent-loop/merge-intents/WS-CI-001-02B.json b/.agent-loop/merge-intents/WS-CI-001-02B.json new file mode 100644 index 00000000..ff0d1859 --- /dev/null +++ b/.agent-loop/merge-intents/WS-CI-001-02B.json @@ -0,0 +1,9 @@ +{ + "chunk_id": "WS-CI-001-02B", + "chunk_title": "Exact-Custody Semantic Test Lanes", + "initiative_id": "WS-CI-001", + "next_chunk_id": null, + "next_chunk_title": null, + "next_requires_explicit_start": true, + "schema_version": 2 +} From a3ad393b1dab7a63c312f7e315bc1b6486a9fca9 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 16:26:03 +0100 Subject: [PATCH 03/31] fix(ci): close semantic lane custody gaps --- .github/workflows/backend.yml | 150 +++++++++++++++++ backend/scripts/run_isolated_tests.py | 12 +- backend/scripts/run_test_lanes.py | 58 ++++++- .../scripts/validate_test_lane_evidence.py | 155 +++++++++++++++++- backend/tests/test_ci_test_lanes.py | 89 +++++++++- .../tests/test_isolated_database_runner.py | 35 ++++ backend/tests/test_test_lane_evidence.py | 74 +++++++++ scripts/test_agent_gates.py | 37 +++++ 8 files changed, 598 insertions(+), 12 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 6a1c46fb..228b57e3 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -47,10 +47,12 @@ jobs: shell: bash run: | set -euo pipefail + job_start_epoch="$(date +%s)" tree_sha="$(git rev-parse HEAD)" test "${tree_sha}" = "${GITHUB_SHA}" test -z "$(git status --porcelain)" test "$(git rev-parse "${tree_sha}^{tree}")" = "$(git rev-parse HEAD^{tree})" + echo "job_start_epoch=${job_start_epoch}" >> "${GITHUB_OUTPUT}" echo "tree_sha=${tree_sha}" >> "${GITHUB_OUTPUT}" - name: Internal review evidence gate @@ -239,6 +241,154 @@ jobs: --precision=2 --fail-under=90 + - name: Record fail-closed hosted timing and coverage evidence + working-directory: backend + env: + EXPECTED_HEAD_SHA: ${{ steps.identity.outputs.tree_sha }} + JOB_START_EPOCH: ${{ steps.identity.outputs.job_start_epoch }} + shell: bash + run: | + set -euo pipefail + coverage json -o .ci/test-lanes/coverage.json + python - <<'PY' + from __future__ import annotations + + from collections import Counter + import hashlib + import json + import math + import os + from pathlib import Path + import subprocess + import time + + evidence_root = Path(".ci/test-lanes") + run_root = evidence_root / "run" + summary_path = evidence_root / "run-summary.json" + coverage_path = evidence_root / "coverage.json" + output_path = Path(".ci/test-lanes/hosted-evidence.json") + + def read_json(path: Path) -> dict[str, object]: + if not path.is_file() or path.is_symlink(): + raise SystemExit(f"invalid hosted evidence input: {path}") + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise SystemExit(f"invalid hosted evidence object: {path}") + return value + + def digest(path: Path) -> str: + if not path.is_file() or path.is_symlink(): + raise SystemExit(f"invalid hosted digest input: {path}") + return hashlib.sha256(path.read_bytes()).hexdigest() + + expected_head = os.environ["EXPECTED_HEAD_SHA"] + actual_head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip() + summary = read_json(summary_path) + coverage = read_json(coverage_path) + if actual_head != expected_head or summary.get("head_sha") != expected_head: + raise SystemExit("hosted evidence head drift") + + lanes = summary.get("lanes") + if not isinstance(lanes, list) or len(lanes) != 4: + raise SystemExit("invalid hosted lane inventory") + collected: list[str] = [] + completed: list[str] = [] + lane_elapsed: list[float] = [] + for lane in lanes: + if not isinstance(lane, dict): + raise SystemExit("invalid hosted lane row") + elapsed = lane.get("elapsed_seconds") + if ( + isinstance(elapsed, bool) + or not isinstance(elapsed, (int, float)) + or not math.isfinite(elapsed) + or elapsed < 0 + ): + raise SystemExit("invalid hosted lane timing") + lane_elapsed.append(float(elapsed)) + for file_key, digest_key in ( + ("evidence_file", "evidence_sha256"), + ("coverage_file", "coverage_sha256"), + ): + name = lane.get(file_key) + expected_digest = lane.get(digest_key) + if not isinstance(name, str) or not isinstance(expected_digest, str): + raise SystemExit("invalid hosted lane digest binding") + path = run_root / name + if path.parent != run_root or digest(path) != expected_digest: + raise SystemExit("hosted lane digest drift") + lane_evidence = read_json(run_root / str(lane["evidence_file"])) + lane_collected = lane_evidence.get("collected_nodes") + lane_completed = lane_evidence.get("completed_nodes") + if not isinstance(lane_collected, list) or not isinstance(lane_completed, list): + raise SystemExit("invalid hosted node custody") + collected.extend(lane_collected) + completed.extend(lane_completed) + + canonical_count = summary.get("canonical_node_count") + if ( + isinstance(canonical_count, bool) + or not isinstance(canonical_count, int) + or canonical_count <= 0 + or len(collected) != canonical_count + or len(completed) != canonical_count + or Counter(collected) != Counter(completed) + or len(set(collected)) != canonical_count + ): + raise SystemExit("hosted node count mismatch") + + aggregate = summary.get("aggregate_runner_seconds") + slowest = summary.get("slowest_lane_seconds") + for value in (aggregate, slowest): + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + raise SystemExit("invalid hosted summary timing") + if not math.isclose( + float(aggregate), math.fsum(lane_elapsed), rel_tol=0.0, abs_tol=1e-9 + ): + raise SystemExit("aggregate runner timing drift") + if not math.isclose( + float(slowest), max(lane_elapsed), rel_tol=0.0, abs_tol=1e-9 + ): + raise SystemExit("slowest lane timing drift") + + start_epoch = int(os.environ["JOB_START_EPOCH"]) + total_wall = time.time() - start_epoch + if not math.isfinite(total_wall) or total_wall < 0 or total_wall > 480: + raise SystemExit("Backend hosted wall time exceeds 480 seconds") + totals = coverage.get("totals") + percent = totals.get("percent_covered") if isinstance(totals, dict) else None + if ( + isinstance(percent, bool) + or not isinstance(percent, (int, float)) + or not math.isfinite(percent) + or percent < 78 + or percent > 100 + ): + raise SystemExit("invalid hosted global coverage") + + hosted = { + "aggregate_runner_seconds": float(aggregate), + "canonical_collected_count": len(collected), + "completed_count": len(completed), + "global_coverage_percent": float(percent), + "global_coverage_sha256": digest(coverage_path), + "head_sha": expected_head, + "run_summary_sha256": digest(summary_path), + "slowest_lane_seconds": float(slowest), + "total_backend_wall_seconds": round(total_wall, 3), + } + output_path.write_text( + json.dumps(hosted, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + PY + - name: Reassert exact tree custody if: ${{ always() }} shell: bash diff --git a/backend/scripts/run_isolated_tests.py b/backend/scripts/run_isolated_tests.py index c77eacbc..a084845b 100644 --- a/backend/scripts/run_isolated_tests.py +++ b/backend/scripts/run_isolated_tests.py @@ -37,6 +37,7 @@ S3_TRAFFIC_LANE = "no_postgres" S3_TRAFFIC_BUCKET = "workstream-artifacts" LANE_RE = re.compile(r"[a-z][a-z0-9_]{0,62}") +BUCKET_RE = re.compile(r"[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?") class RunnerError(RuntimeError): @@ -189,9 +190,16 @@ def _tree_sha() -> str: def _minio_namespace(lane: str, suffix: str) -> tuple[str, str]: """Return a lane-bound bucket and prefix without accepting arbitrary names.""" - if LANE_RE.fullmatch(lane) is None: + if LANE_RE.fullmatch(lane) is None or re.fullmatch(r"[a-f0-9]{12}", suffix) is None: raise RunnerError("invalid_lane") - bucket = S3_TRAFFIC_BUCKET if lane == S3_TRAFFIC_LANE else f"workstream-ci-{lane}-{suffix}" + lane_component = lane.replace("_", "-") + bucket = ( + S3_TRAFFIC_BUCKET + if lane == S3_TRAFFIC_LANE + else f"workstream-ci-{lane_component}-{suffix}" + ) + if len(bucket) > 63 or BUCKET_RE.fullmatch(bucket) is None: + raise RunnerError("invalid_minio_namespace") return bucket, f"ci/{lane}/{suffix}" diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 7b44c08a..9afc0d35 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -30,6 +30,19 @@ ADMIN_RUNNER_MODULE = "tests/test_isolated_database_runner.py" ORDINARY_KIND = "ordinary_isolated" ADMIN_KIND = "admin_runner_self_test" +ADMIN_REDACTING_WRAPPER = """ +import os +import subprocess +import sys + +result = subprocess.run(sys.argv[1:], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +data = result.stdout +secret = os.environ.get('WORKSTREAM_TEST_ADMIN_DATABASE_URL', '').encode() +if secret: + data = data.replace(secret, b'[REDACTED_ADMIN_DATABASE_URL]') +sys.stdout.buffer.write(data) +raise SystemExit(result.returncode) +""".strip() HEARTBEAT_SECONDS = 60.0 CLEANUP_GRACE_SECONDS = 10.0 POLL_SECONDS = 0.05 @@ -383,10 +396,11 @@ def lane_command( def admin_runner_command(nodes: list[str]) -> list[str]: """Run isolation-runner self-tests directly, never inside an owned database.""" - return [ + pytest_command = [ sys.executable, "-m", "pytest", "-q", *_plugin_args(), "--cov=app", "--cov-report=", "--durations=25", *nodes, ] + return [sys.executable, "-c", ADMIN_REDACTING_WRAPPER, *pytest_command] def _prepare_outputs(metadata_dir: Path, summary_json: Path) -> None: @@ -464,20 +478,54 @@ def _finalize_lane( "skipped_nodes": sorted(set(node for unit in units for node in unit["skipped_nodes"])), } evidence_path.write_bytes(_json_bytes(evidence)) + if coverage_path.is_file() and not coverage_path.is_symlink(): + for unit in units: + source = unit["coverage_path"] + if source != coverage_path and source.is_file() and not source.is_symlink(): + source.unlink() return { - "collection_exit_code": max(unit["collection_exit_code"] for unit in units), + "collection_exit_code": _aggregate_exit_codes( + [unit["collection_exit_code"] for unit in units] + ), "coverage_file": coverage_path.name, "coverage_sha256": _sha256(coverage_path.read_bytes()) if coverage_path.is_file() and not coverage_path.is_symlink() else None, "elapsed_seconds": round(max(unit["elapsed_seconds"] for unit in units), 3), "evidence_file": evidence_path.name, "evidence_sha256": _sha256(evidence_path.read_bytes()), - "execution_exit_code": max(unit["execution_exit_code"] for unit in units), + "execution_exit_code": _aggregate_exit_codes( + [unit["execution_exit_code"] for unit in units] + ), "interrupted": any(unit["interrupted"] for unit in units), "name": lane.name, } +def _aggregate_exit_codes(codes: list[int]) -> int: + """Return success only when every subprocess exited successfully.""" + if not codes or any(not isinstance(code, int) or isinstance(code, bool) for code in codes): + raise LaneError("invalid_lane_exit_codes") + return 0 if all(code == 0 for code in codes) else 1 + + +def _timing_summary(lanes: list[dict[str, Any]]) -> dict[str, float]: + """Derive aggregate timing only from the exact four emitted lane rows.""" + if len(lanes) != 4: + raise LaneError("invalid_lane_timing_inventory") + elapsed = [row["elapsed_seconds"] for row in lanes] + if any( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or value < 0 + for value in elapsed + ): + raise LaneError("invalid_lane_timing") + return { + "aggregate_runner_seconds": round(sum(elapsed), 3), + "slowest_lane_seconds": round(max(elapsed), 3), + } + + def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, collect_only: bool = False) -> int: """Collect canonical nodes and optionally execute all lanes concurrently.""" global INTERRUPTED @@ -513,7 +561,8 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, }) summary = {"canonical_node_count": len(nodes), "elapsed_seconds": 0.0, "head_sha": tree_sha, "lanes": lane_rows, "manifest_file": manifest_path.name, - "manifest_sha256": manifest_digest, "mode": "collect", "schema_version": SCHEMA_VERSION} + "manifest_sha256": manifest_digest, "mode": "collect", "schema_version": SCHEMA_VERSION, + **_timing_summary(lane_rows)} summary_json.write_bytes(_json_bytes(summary)) return 0 if any(lane.requires_postgres for lane in LANES) and not os.environ.get(ADMIN_ENV): @@ -601,6 +650,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, "manifest_file": manifest_path.name, "manifest_sha256": manifest_digest, "mode": "run", "schema_version": SCHEMA_VERSION, } + summary.update(_timing_summary(summary["lanes"])) summary_json.write_bytes(_json_bytes(summary)) return 0 if len(results) == 4 and all(row["execution_exit_code"] == 0 for row in results.values()) else 1 diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 174a5672..7d65d60f 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -6,11 +6,14 @@ from collections import Counter import hashlib import json +import math +import os from pathlib import Path, PurePosixPath import re import stat import subprocess import sys +import tempfile from typing import Any @@ -24,12 +27,32 @@ ADMIN_KIND = "admin_runner_self_test" DATABASE_IDENTIFIER_RE = re.compile(r"^[a-z][a-z0-9_]*$") BUCKET_RE = re.compile(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$") +VALIDATOR_COLLECTION_ENV = "WORKSTREAM_VALIDATOR_COLLECTION_FILE" class EvidenceError(RuntimeError): """Test-lane evidence is incomplete, unsafe, or inconsistent.""" +def pytest_collection_finish(session: Any) -> None: + """Record exact node IDs for the validator-owned collection subprocess.""" + destination = os.environ.get(VALIDATOR_COLLECTION_ENV) + if destination is None: + return + path = Path(destination) + data = b"".join( + json.dumps(item.nodeid).encode("utf-8") + b"\n" + for item in sorted(session.items, key=lambda item: item.nodeid) + ) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + view = memoryview(data) + while view: + view = view[os.write(descriptor, view) :] + finally: + os.close(descriptor) + + def _object(value: Any, keys: set[str], error: str) -> dict[str, Any]: if not isinstance(value, dict) or set(value) != keys: raise EvidenceError(error) @@ -65,6 +88,85 @@ def _current_head(repository_root: Path) -> str: return head +def _discover_current_modules(backend_root: Path) -> list[str]: + tests_root = backend_root / "tests" + if tests_root.is_symlink() or not tests_root.is_dir(): + raise EvidenceError("invalid_current_test_inventory") + if any(path.is_symlink() for path in tests_root.rglob("*")): + raise EvidenceError("invalid_current_test_inventory") + modules: list[str] = [] + for path in sorted(tests_root.rglob("test_*.py")): + try: + relative = path.relative_to(backend_root) + except ValueError as exc: + raise EvidenceError("invalid_current_test_inventory") from exc + current = backend_root + for part in relative.parts: + current = current / part + try: + mode = current.lstat().st_mode + except OSError as exc: + raise EvidenceError("invalid_current_test_inventory") from exc + if stat.S_ISLNK(mode): + raise EvidenceError("invalid_current_test_inventory") + if not stat.S_ISREG(path.stat().st_mode): + raise EvidenceError("invalid_current_test_inventory") + modules.append(relative.as_posix()) + if not modules: + raise EvidenceError("zero_current_test_modules") + return modules + + +def _collect_current_nodes(repository_root: Path) -> list[str]: + backend_root = repository_root / "backend" + modules = _discover_current_modules(backend_root) + with tempfile.TemporaryDirectory(prefix="workstream-validator-") as temporary: + output = Path(temporary) / "nodes.jsonl" + environment = os.environ.copy() + environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + environment[VALIDATOR_COLLECTION_ENV] = str(output) + validator_backend = str(Path(__file__).resolve().parents[1]) + existing_pythonpath = environment.get("PYTHONPATH") + environment["PYTHONPATH"] = ( + f"{validator_backend}{os.pathsep}{existing_pythonpath}" + if existing_pythonpath + else validator_backend + ) + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "-q", + "-p", + "pytest_asyncio.plugin", + "-p", + "scripts.validate_test_lane_evidence", + *modules, + ], + cwd=backend_root, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode != 0: + raise EvidenceError("independent_pytest_collection_failed") + try: + nodes = [ + json.loads(line) + for line in output.read_text(encoding="utf-8").splitlines() + ] + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvidenceError("invalid_independent_collection") from exc + if not nodes or any(not isinstance(node, str) or "::" not in node for node in nodes): + raise EvidenceError("invalid_independent_collection") + if nodes != sorted(nodes) or len(nodes) != len(set(nodes)): + raise EvidenceError("invalid_independent_collection") + return nodes + + def _safe_file(metadata_dir: Path, name: Any) -> Path: if not isinstance(name, str) or not name: raise EvidenceError("unsafe_evidence_path") @@ -129,6 +231,7 @@ def validate_evidence( _json_bytes(summary_json.read_bytes(), "invalid_summary"), { "canonical_node_count", + "aggregate_runner_seconds", "elapsed_seconds", "head_sha", "lanes", @@ -136,6 +239,7 @@ def validate_evidence( "manifest_sha256", "mode", "schema_version", + "slowest_lane_seconds", }, "invalid_summary", ) @@ -193,6 +297,14 @@ def validate_evidence( raise EvidenceError("noncanonical_or_duplicate_manifest_nodes") if not any(module == ADMIN_RUNNER_MODULE for _node, module, _lane, _kind in canonical): raise EvidenceError("missing_admin_runner_self_tests") + independently_collected = _collect_current_nodes(root) + canonical_ids = [nodeid for nodeid, _module, _lane, _kind in canonical] + if Counter(independently_collected) != Counter(canonical_ids): + raise EvidenceError("current_node_inventory_mismatch") + if {node.split("::", 1)[0] for node in independently_collected} != { + module for _node, module, _lane, _kind in canonical + }: + raise EvidenceError("current_module_inventory_mismatch") if isinstance(summary["canonical_node_count"], bool) or summary["canonical_node_count"] != len( canonical ): @@ -213,6 +325,7 @@ def validate_evidence( isolation_namespaces: list[tuple[str, str, str, str]] = [] coverage_files: list[str] = [] isolation_files: list[str] = [] + lane_elapsed_seconds: list[float] = [] for lane_value in lanes: lane = _object( lane_value, @@ -230,12 +343,31 @@ def validate_evidence( "invalid_lane_summary", ) name = lane["name"] - if lane["collection_exit_code"] != 0: + elapsed = lane["elapsed_seconds"] + if ( + isinstance(elapsed, bool) + or not isinstance(elapsed, (int, float)) + or not math.isfinite(elapsed) + or elapsed < 0 + ): + raise EvidenceError("invalid_lane_elapsed_seconds") + lane_elapsed_seconds.append(float(elapsed)) + if ( + isinstance(lane["collection_exit_code"], bool) + or not isinstance(lane["collection_exit_code"], int) + or lane["collection_exit_code"] != 0 + ): raise EvidenceError("collection_failed") if lane["interrupted"] is not False: raise EvidenceError("lane_interrupted") - expected_execution = None if mode == "collect" else 0 - if lane["execution_exit_code"] != expected_execution: + if (mode == "collect" and lane["execution_exit_code"] is not None) or ( + mode == "run" + and ( + isinstance(lane["execution_exit_code"], bool) + or not isinstance(lane["execution_exit_code"], int) + or lane["execution_exit_code"] != 0 + ) + ): raise EvidenceError("execution_incomplete") evidence = _object( _json_bytes( @@ -342,7 +474,6 @@ def validate_evidence( all_collected.extend(collected) all_completed.extend(completed) - canonical_ids = [nodeid for nodeid, _module, _lane, _kind in canonical] if Counter(all_collected) != Counter(canonical_ids): raise EvidenceError("global_collection_reconciliation_failed") if mode == "run" and Counter(all_completed) != Counter(canonical_ids): @@ -356,6 +487,22 @@ def validate_evidence( len(set(coverage_files)) != LANE_COUNT or len(set(isolation_files)) != LANE_COUNT ): raise EvidenceError("shared_lane_artifact") + aggregate = summary["aggregate_runner_seconds"] + slowest = summary["slowest_lane_seconds"] + if any( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + for value in (aggregate, slowest, summary["elapsed_seconds"]) + ): + raise EvidenceError("invalid_summary_timing") + expected_aggregate = math.fsum(lane_elapsed_seconds) + expected_slowest = max(lane_elapsed_seconds) + if not math.isclose( + float(aggregate), expected_aggregate, rel_tol=0.0, abs_tol=1e-9 + ) or not math.isclose(float(slowest), expected_slowest, rel_tol=0.0, abs_tol=1e-9): + raise EvidenceError("summary_timing_mismatch") return summary diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index b41e029a..289ea634 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -3,7 +3,9 @@ from dataclasses import replace import hashlib import json +import os from pathlib import Path +import subprocess import sys import time @@ -162,6 +164,29 @@ def test_admin_runner_environment_retains_only_admin_database_url( assert "run_isolated_tests.py" not in " ".join(command) +def test_admin_wrapper_redacts_admin_url_before_persisted_output() -> None: + secret = "postgresql+asyncpg://admin:secret@localhost/postgres" + env = os.environ.copy() + env[runner.ADMIN_ENV] = secret + result = subprocess.run( + [ + sys.executable, + "-c", + runner.ADMIN_REDACTING_WRAPPER, + sys.executable, + "-c", + f"import os; print(os.environ[{runner.ADMIN_ENV!r}])", + ], + env=env, + check=True, + stdout=subprocess.PIPE, + text=True, + ) + + assert secret not in result.stdout + assert result.stdout.strip() == "[REDACTED_ADMIN_DATABASE_URL]" + + def test_collect_only_writes_raw_digest_bound_validator_schema( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -177,13 +202,20 @@ def test_collect_only_writes_raw_digest_bound_validator_schema( summary = json.loads(summary_path.read_text(encoding="utf-8")) manifest_bytes = (metadata / summary["manifest_file"]).read_bytes() assert set(summary) == { - "canonical_node_count", "elapsed_seconds", "head_sha", "lanes", "manifest_file", - "manifest_sha256", "mode", "schema_version", + "aggregate_runner_seconds", "canonical_node_count", "elapsed_seconds", "head_sha", + "lanes", "manifest_file", "manifest_sha256", "mode", "schema_version", + "slowest_lane_seconds", } assert summary["mode"] == "collect" assert summary["canonical_node_count"] == len(nodes) assert summary["manifest_sha256"] == hashlib.sha256(manifest_bytes).hexdigest() assert len(summary["lanes"]) == 4 + assert summary["slowest_lane_seconds"] == max( + lane["elapsed_seconds"] for lane in summary["lanes"] + ) + assert summary["aggregate_runner_seconds"] == sum( + lane["elapsed_seconds"] for lane in summary["lanes"] + ) for lane in summary["lanes"]: assert lane["coverage_file"] is None assert lane["execution_exit_code"] is None @@ -210,6 +242,53 @@ def fake_run(*_args, **kwargs): runner.collect_nodes((module,), tmp_path / "metadata") +def test_timing_summary_is_derived_from_exact_four_lanes() -> None: + lanes = [{"elapsed_seconds": value} for value in (1.125, 2.25, 0.5, 3.75)] + + assert runner._timing_summary(lanes) == { + "aggregate_runner_seconds": 7.625, + "slowest_lane_seconds": 3.75, + } + with pytest.raises(LaneError, match="invalid_lane_timing_inventory"): + runner._timing_summary(lanes[:3]) + with pytest.raises(LaneError, match="invalid_lane_timing"): + runner._timing_summary([*lanes[:3], {"elapsed_seconds": -1.0}]) + + +def test_exit_aggregation_cannot_hide_signal_failure() -> None: + assert runner._aggregate_exit_codes([0, 0]) == 0 + assert runner._aggregate_exit_codes([0, 3]) == 1 + assert runner._aggregate_exit_codes([0, -15]) == 1 + with pytest.raises(LaneError, match="invalid_lane_exit_codes"): + runner._aggregate_exit_codes([]) + + +def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path) -> None: + rows = [] + for index, lane in enumerate(LANES): + source = tmp_path / f".coverage.unit.{lane.name}" + source.write_bytes(f"coverage-{lane.name}".encode()) + (tmp_path / f"{lane.name}.database.json").write_text("{}\n", encoding="utf-8") + unit = { + "collected_nodes": [f"{lane.modules[0]}::test_one"], + "collection_exit_code": 0, + "completed_nodes": [f"{lane.modules[0]}::test_one"], + "coverage_path": source, + "deselected_nodes": [], + "elapsed_seconds": float(index + 1), + "execution_exit_code": 0, + "interrupted": False, + "skipped_nodes": [], + } + rows.append(runner._finalize_lane(lane, [unit], tmp_path)) + + assert [row["name"] for row in rows] == [lane.name for lane in LANES] + assert sorted(path.name for path in tmp_path.glob(".coverage.*")) == sorted( + f".coverage.{lane.name}" for lane in LANES + ) + assert not list(tmp_path.glob(".coverage.unit.*")) + + def test_failure_interrupts_sibling_process_groups_and_records_all_lanes( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -246,3 +325,9 @@ def fake_command(lane, _nodes, metadata, _timeout, _sha): assert len(summary["lanes"]) == 4 assert summary["lanes"][0]["execution_exit_code"] == 1 assert all(row["interrupted"] for row in summary["lanes"][1:]) + assert summary["slowest_lane_seconds"] == max( + row["elapsed_seconds"] for row in summary["lanes"] + ) + assert summary["aggregate_runner_seconds"] == round( + sum(row["elapsed_seconds"] for row in summary["lanes"]), 3 + ) diff --git a/backend/tests/test_isolated_database_runner.py b/backend/tests/test_isolated_database_runner.py index 628a3642..b3c3a2f3 100644 --- a/backend/tests/test_isolated_database_runner.py +++ b/backend/tests/test_isolated_database_runner.py @@ -82,6 +82,41 @@ def test_lane_namespaces_bind_real_s3_traffic_and_separate_other_lanes() -> None runner._minio_namespace("../foreign", "012345abcdef") +@pytest.mark.parametrize( + ("lane", "expected_bucket"), + [ + ("no_postgres", "workstream-artifacts"), + ("schema_contracts", "workstream-ci-schema-contracts-012345abcdef"), + ("control_plane", "workstream-ci-control-plane-012345abcdef"), + ("execution_plane", "workstream-ci-execution-plane-012345abcdef"), + ], +) +def test_committed_lane_buckets_use_validator_compatible_s3_grammar( + lane: str, expected_bucket: str +) -> None: + """Every committed lane maps deterministically to a valid S3 bucket name.""" + bucket, prefix = runner._minio_namespace(lane, "012345abcdef") + assert bucket == expected_bucket + assert 3 <= len(bucket) <= 63 + assert runner.BUCKET_RE.fullmatch(bucket) + assert "_" not in bucket + assert prefix == f"ci/{lane}/012345abcdef" + + +def test_lane_namespaces_do_not_collide_across_lanes_or_runner_suffixes() -> None: + """Separate lane identities and runner invocations cannot share custody.""" + namespaces = { + runner._minio_namespace(lane, suffix) + for lane in ("no_postgres", "schema_contracts", "control_plane", "execution_plane") + for suffix in ("012345abcdef", "fedcba543210") + } + assert len(namespaces) == 8 + with pytest.raises(runner.RunnerError, match="invalid_lane"): + runner._minio_namespace("control_plane", "not-hex") + with pytest.raises(runner.RunnerError, match="invalid_minio_namespace"): + runner._minio_namespace("a" * 63, "012345abcdef") + + def test_child_environment_binds_lane_namespace_without_admin_custody() -> None: """Children receive resource targets but never the destructive admin authority.""" env = runner._child_env( diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index 7ccf5e16..aa7a9411 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -15,6 +15,7 @@ assert SPEC is not None and SPEC.loader is not None validator = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(validator) +REAL_COLLECT_CURRENT_NODES = validator._collect_current_nodes HEAD = "a" * 40 LANES = ("no_postgres", "schema_contracts", "control_plane", "execution_plane") @@ -108,6 +109,7 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: } ) summary = { + "aggregate_runner_seconds": 4.0, "canonical_node_count": 5, "elapsed_seconds": 2.0, "head_sha": HEAD, @@ -116,6 +118,7 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: "manifest_sha256": manifest_digest, "mode": mode, "schema_version": 1, + "slowest_lane_seconds": 1.0, } summary_path = tmp_path / "summary.json" _write(summary_path, summary) @@ -125,6 +128,16 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: @pytest.fixture(autouse=True) def exact_head(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(validator, "_current_head", lambda _root: HEAD) + monkeypatch.setattr( + validator, + "_collect_current_nodes", + lambda _root: sorted( + [ + *(f"tests/test_{index}.py::test_ok" for index in range(4)), + "tests/test_isolated_database_runner.py::test_admin_custody", + ] + ), + ) @pytest.mark.parametrize("mode", ["collect", "run"]) @@ -188,7 +201,11 @@ def test_rejects_node_custody_failures(tmp_path: Path, mutation, message: str) - ("field", "value", "message"), [ ("collection_exit_code", 1, "collection_failed"), + ("collection_exit_code", -9, "collection_failed"), + ("collection_exit_code", False, "collection_failed"), ("execution_exit_code", None, "execution_incomplete"), + ("execution_exit_code", -15, "execution_incomplete"), + ("execution_exit_code", False, "execution_incomplete"), ("interrupted", True, "lane_interrupted"), ], ) @@ -294,3 +311,60 @@ def test_rejects_recorded_database_environment_or_shared_coverage(tmp_path: Path _write(summary_path, summary) with pytest.raises(validator.EvidenceError, match="shared_lane_artifact"): validator.validate_evidence(metadata, summary_path, tmp_path) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("aggregate_runner_seconds", 4.01, "summary_timing_mismatch"), + ("slowest_lane_seconds", 0.99, "summary_timing_mismatch"), + ("aggregate_runner_seconds", float("nan"), "invalid_summary_timing"), + ("slowest_lane_seconds", float("inf"), "invalid_summary_timing"), + ("elapsed_seconds", -1.0, "invalid_summary_timing"), + ], +) +def test_rejects_invalid_or_drifted_summary_timing( + tmp_path: Path, field: str, value: object, message: str +) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + summary[field] = value + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match=message): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +@pytest.mark.parametrize("value", [-0.001, float("nan"), float("inf"), True]) +def test_rejects_invalid_lane_timing(tmp_path: Path, value: object) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + summary["lanes"][0]["elapsed_seconds"] = value + _write(summary_path, summary) + with pytest.raises(validator.EvidenceError, match="invalid_lane_elapsed_seconds"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + +@pytest.mark.parametrize("mutation", ["missing", "foreign"]) +def test_real_collection_rejects_missing_or_foreign_manifest_node( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mutation: str +) -> None: + tests = tmp_path / "backend/tests" + tests.mkdir(parents=True) + for index in range(4): + (tests / f"test_{index}.py").write_text("def test_ok():\n pass\n", encoding="utf-8") + (tests / "test_isolated_database_runner.py").write_text( + "def test_admin_custody():\n pass\n", encoding="utf-8" + ) + metadata, summary_path, summary = _bundle(tmp_path) + manifest_path = metadata / summary["manifest_file"] + manifest = json.loads(manifest_path.read_text()) + if mutation == "missing": + manifest["nodes"].pop(0) + else: + manifest["nodes"][0]["module"] = "tests/test_foreign.py" + manifest["nodes"][0]["nodeid"] = "tests/test_foreign.py::test_ok" + manifest["nodes"].sort(key=lambda row: row["nodeid"]) + summary["canonical_node_count"] = len(manifest["nodes"]) + summary["manifest_sha256"] = _write(manifest_path, manifest) + _write(summary_path, summary) + monkeypatch.setattr(validator, "_collect_current_nodes", REAL_COLLECT_CURRENT_NODES) + with pytest.raises(validator.EvidenceError, match="current_node_inventory_mismatch"): + validator.validate_evidence(metadata, summary_path, tmp_path) diff --git a/scripts/test_agent_gates.py b/scripts/test_agent_gates.py index f0d697d9..ab284c49 100644 --- a/scripts/test_agent_gates.py +++ b/scripts/test_agent_gates.py @@ -6459,6 +6459,11 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert len(identity) == 1 assert '${tree_sha}' in str(identity[0]["run"]) assert '${GITHUB_SHA}' in str(identity[0]["run"]) + identity_command = str(identity[0]["run"]) + assert identity_command.index('job_start_epoch="$(date +%s)"') < ( + identity_command.index('tree_sha="$(git rev-parse HEAD)"') + ) + assert 'job_start_epoch=${job_start_epoch}' in identity_command install = [step for step in steps if step.get("name") == "Install backend and exact Ruff"] assert len(install) == 1 @@ -6564,6 +6569,38 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert coverage_step.get("working-directory") == "backend" for forbidden_key in ("if", "continue-on-error", "shell", "env"): assert forbidden_key not in coverage_step + hosted_steps = [ + step for step in steps + if step.get("name") == "Record fail-closed hosted timing and coverage evidence" + ] + assert len(hosted_steps) == 1 + hosted_step = hosted_steps[0] + hosted_command = str(hosted_step["run"]) + assert steps.index(hosted_step) > max(steps.index(step) for step in auth_coverage_steps) + assert "coverage json -o .ci/test-lanes/coverage.json" in hosted_command + assert ".ci/test-lanes/hosted-evidence.json" in hosted_command + assert 'summary.get("head_sha") != expected_head' in hosted_command + assert 'summary.get("canonical_node_count")' in hosted_command + assert 'summary.get("aggregate_runner_seconds")' in hosted_command + assert 'summary.get("slowest_lane_seconds")' in hosted_command + assert 'total_wall > 480' in hosted_command + assert 'percent < 78' in hosted_command + assert "math.isfinite" in hosted_command + assert "Counter(collected) != Counter(completed)" in hosted_command + assert "hosted lane digest drift" in hosted_command + for required_field in ( + "head_sha", + "total_backend_wall_seconds", + "slowest_lane_seconds", + "aggregate_runner_seconds", + "canonical_collected_count", + "completed_count", + "global_coverage_percent", + "global_coverage_sha256", + "run_summary_sha256", + ): + assert f'"{required_field}"' in hosted_command + assert "waiver" not in hosted_command.lower() active_phase = active_artifact_coverage_phase() expected_coverage = artifact_expected_coverage_commands_for(active_phase) actual_coverage = tuple( From 848fa03d5492e97f9adf21340a3f2a18fd47ca4f Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 16:38:08 +0100 Subject: [PATCH 04/31] fix(ci): stabilize independent node inventory --- backend/scripts/run_test_lanes.py | 87 +++++++++++++++++-- .../scripts/validate_test_lane_evidence.py | 74 ++++++++++++++-- backend/tests/test_ci_test_lanes.py | 39 ++++++++- backend/tests/test_test_lane_evidence.py | 28 +++++- 4 files changed, 212 insertions(+), 16 deletions(-) diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 9afc0d35..006d061a 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -7,6 +7,7 @@ from collections import Counter from dataclasses import dataclass import hashlib +import inspect import json import os from pathlib import Path, PurePosixPath @@ -17,6 +18,7 @@ import sys import time from typing import Any, TextIO +import uuid ROOT = Path(__file__).resolve().parents[1] TESTS_DIR = ROOT / "tests" @@ -26,6 +28,7 @@ COMPLETED_ENV = "WORKSTREAM_LANE_COMPLETED_NODES" SKIPPED_ENV = "WORKSTREAM_LANE_SKIPPED_NODES" DESELECTED_ENV = "WORKSTREAM_LANE_DESELECTED_NODES" +HEAD_ENV = "WORKSTREAM_LANE_HEAD_SHA" SCHEMA_VERSION = 1 ADMIN_RUNNER_MODULE = "tests/test_isolated_database_runner.py" ORDINARY_KIND = "ordinary_isolated" @@ -49,6 +52,9 @@ SHA_RE = re.compile(r"[0-9a-f]{40}") LANE_RE = re.compile(r"[a-z][a-z0-9_]*") INTERRUPTED = False +_ORIGINAL_UUID4: Any = None +_UUID4_ORDINALS: dict[tuple[str, int], int] = {} +_UUID4_HEAD = "" class LaneError(RuntimeError): @@ -198,6 +204,59 @@ def pytest_deselected(items: list[Any]) -> None: _append_node(destination, item.nodeid) +def _deterministic_uuid4() -> uuid.UUID: + """Return a stable UUIDv4 for one repository callsite and ordinal.""" + frame = inspect.currentframe() + caller = frame.f_back if frame is not None else None + try: + if caller is None or _ORIGINAL_UUID4 is None: + return uuid.UUID(bytes=os.urandom(16), version=4) + try: + relative = Path(caller.f_code.co_filename).resolve().relative_to(ROOT).as_posix() + except (OSError, ValueError): + return _ORIGINAL_UUID4() + callsite = (relative, caller.f_lineno) + ordinal = _UUID4_ORDINALS.get(callsite, 0) + _UUID4_ORDINALS[callsite] = ordinal + 1 + key = f"{_UUID4_HEAD}\0{relative}\0{caller.f_lineno}\0{ordinal}".encode() + return uuid.UUID(bytes=hashlib.sha256(key).digest()[:16], version=4) + finally: + del frame + del caller + + +def pytest_make_parametrize_id(config: Any, val: Any, argname: str) -> str | None: + """Keep deterministic UUID parameter bytes visible in the exact node ID.""" + del config, argname + return str(val) if isinstance(val, uuid.UUID) else None + + +def pytest_sessionstart(session: Any) -> None: + """Stabilize UUID-backed parameter values before test-module imports.""" + del session + global _ORIGINAL_UUID4, _UUID4_HEAD + if _ORIGINAL_UUID4 is not None: + uuid.uuid4 = _ORIGINAL_UUID4 + head = os.environ.get(HEAD_ENV, "") + if SHA_RE.fullmatch(head) is None: + head = _tree_sha() + _ORIGINAL_UUID4 = uuid.uuid4 + _UUID4_HEAD = head + _UUID4_ORDINALS.clear() + uuid.uuid4 = _deterministic_uuid4 + + +def pytest_sessionfinish(session: Any, exitstatus: int) -> None: + """Restore process-global UUID behavior after pytest completes.""" + del session, exitstatus + global _ORIGINAL_UUID4, _UUID4_HEAD + if _ORIGINAL_UUID4 is not None: + uuid.uuid4 = _ORIGINAL_UUID4 + _ORIGINAL_UUID4 = None + _UUID4_HEAD = "" + _UUID4_ORDINALS.clear() + + def discover_test_modules(tests_dir: Path = TESTS_DIR, root: Path = ROOT) -> tuple[str, ...]: """Recursively discover regular test modules without following symlinks.""" if tests_dir.is_symlink() or not tests_dir.is_dir(): @@ -291,7 +350,7 @@ def _plugin_args() -> list[str]: return ["-p", "pytest_asyncio.plugin", "-p", "pytest_cov.plugin", "-p", "scripts.run_test_lanes"] -def _collection_environment(collected: Path, deselected: Path) -> dict[str, str]: +def _collection_environment(collected: Path, deselected: Path, tree_sha: str) -> dict[str, str]: env = os.environ.copy() env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" env["PYTHONPATH"] = os.pathsep.join( @@ -299,17 +358,20 @@ def _collection_environment(collected: Path, deselected: Path) -> dict[str, str] ) env[COLLECTED_ENV] = str(collected.resolve()) env[DESELECTED_ENV] = str(deselected.resolve()) + env[HEAD_ENV] = tree_sha return env -def collect_nodes(modules: tuple[str, ...], metadata_dir: Path) -> tuple[int, list[str], list[str]]: +def collect_nodes( + modules: tuple[str, ...], metadata_dir: Path, tree_sha: str +) -> tuple[int, list[str], list[str]]: collected = metadata_dir / "collection.nodes.jsonl" deselected = metadata_dir / "collection.deselected.jsonl" _exclusive_file(collected) _exclusive_file(deselected) result = subprocess.run( [sys.executable, "-m", "pytest", "--collect-only", "-q", *_plugin_args(), *modules], - cwd=ROOT, env=_collection_environment(collected, deselected), check=False, + cwd=ROOT, env=_collection_environment(collected, deselected, tree_sha), check=False, ) nodes = _read_nodes(collected) if result.returncode == 0 else [] deselected_nodes = _read_nodes(deselected, allow_empty=True) @@ -350,6 +412,7 @@ def lane_environment( metadata_dir: Path, coverage_path: Path, evidence_stem: str | None = None, + tree_sha: str | None = None, ) -> dict[str, str]: env = os.environ.copy() env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" @@ -362,14 +425,20 @@ def lane_environment( env[COMPLETED_ENV] = str((metadata_dir / f"{stem}.completed.jsonl").resolve()) env[SKIPPED_ENV] = str((metadata_dir / f"{stem}.skipped.jsonl").resolve()) env[DESELECTED_ENV] = str((metadata_dir / f"{stem}.deselected.jsonl").resolve()) + if tree_sha is not None: + env[HEAD_ENV] = tree_sha return env def admin_runner_environment( - lane: TestLane, metadata_dir: Path, coverage_path: Path, evidence_stem: str + lane: TestLane, + metadata_dir: Path, + coverage_path: Path, + evidence_stem: str, + tree_sha: str | None = None, ) -> dict[str, str]: """Retain only the admin URL needed to test the isolation owner itself.""" - env = lane_environment(lane, metadata_dir, coverage_path, evidence_stem) + env = lane_environment(lane, metadata_dir, coverage_path, evidence_stem, tree_sha) env.pop("WORKSTREAM_DATABASE_URL", None) env.pop("WORKSTREAM_TEST_DATABASE_URL", None) return env @@ -536,7 +605,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, validate_lane_inventory(modules) _prepare_outputs(metadata_dir, summary_json) tree_sha = _tree_sha() - collection_exit, nodes, deselected = collect_nodes(modules, metadata_dir) + collection_exit, nodes, deselected = collect_nodes(modules, metadata_dir, tree_sha) manifest = build_manifest(tree_sha, nodes) manifest_path = metadata_dir / "manifest.json" manifest_path.write_bytes(_json_bytes(manifest)) @@ -592,10 +661,12 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, coverage_path = metadata_dir / f".coverage.unit.{key}" if kind == ADMIN_KIND: command = admin_runner_command(unit_nodes) - env = admin_runner_environment(lane, metadata_dir, coverage_path, key) + env = admin_runner_environment( + lane, metadata_dir, coverage_path, key, tree_sha + ) else: command = lane_command(lane, unit_nodes, metadata_dir, timeout_seconds, tree_sha) - env = lane_environment(lane, metadata_dir, coverage_path, key) + env = lane_environment(lane, metadata_dir, coverage_path, key, tree_sha) process = subprocess.Popen( command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 7d65d60f..39e105ab 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -5,6 +5,7 @@ import argparse from collections import Counter import hashlib +import inspect import json import math import os @@ -15,6 +16,7 @@ import sys import tempfile from typing import Any +import uuid SCHEMA_VERSION = 1 @@ -28,12 +30,64 @@ DATABASE_IDENTIFIER_RE = re.compile(r"^[a-z][a-z0-9_]*$") BUCKET_RE = re.compile(r"^[a-z0-9][a-z0-9.-]*[a-z0-9]$") VALIDATOR_COLLECTION_ENV = "WORKSTREAM_VALIDATOR_COLLECTION_FILE" +VALIDATOR_HEAD_ENV = "WORKSTREAM_VALIDATOR_HEAD_SHA" +VALIDATOR_ROOT_ENV = "WORKSTREAM_VALIDATOR_BACKEND_ROOT" +_ORIGINAL_UUID4 = uuid.uuid4 +_UUID_CALLSITE_COUNTS: dict[tuple[str, int], int] = {} +_UUID_ROOT: Path | None = None +_UUID_HEAD: str | None = None class EvidenceError(RuntimeError): """Test-lane evidence is incomplete, unsafe, or inconsistent.""" +def _deterministic_uuid4() -> uuid.UUID: + """Return a caller-stable UUIDv4 for repository import-time parameters.""" + frame = inspect.currentframe() + caller = frame.f_back if frame is not None else None + try: + if caller is None or _UUID_ROOT is None or _UUID_HEAD is None: + return _ORIGINAL_UUID4() + try: + relative = Path(caller.f_code.co_filename).resolve().relative_to(_UUID_ROOT) + except (OSError, ValueError): + return _ORIGINAL_UUID4() + relative_file = relative.as_posix() + callsite = (relative_file, caller.f_lineno) + ordinal = _UUID_CALLSITE_COUNTS.get(callsite, 0) + _UUID_CALLSITE_COUNTS[callsite] = ordinal + 1 + key = (f"{_UUID_HEAD}\0{relative_file}\0{caller.f_lineno}\0{ordinal}").encode("utf-8") + return uuid.UUID(bytes=hashlib.sha256(key).digest()[:16], version=4) + finally: + del frame + del caller + + +def pytest_sessionstart(session: Any) -> None: + """Install deterministic UUIDv4 generation before test-module collection.""" + del session + global _UUID_ROOT, _UUID_HEAD + root = os.environ.get(VALIDATOR_ROOT_ENV) + head = os.environ.get(VALIDATOR_HEAD_ENV) + if root is None or head is None or SHA_RE.fullmatch(head) is None: + return + _UUID_ROOT = Path(root).resolve(strict=True) + _UUID_HEAD = head + _UUID_CALLSITE_COUNTS.clear() + uuid.uuid4 = _deterministic_uuid4 + + +def pytest_sessionfinish(session: Any, exitstatus: int) -> None: + """Restore process UUID behavior after validator-owned collection.""" + del session, exitstatus + global _UUID_ROOT, _UUID_HEAD + uuid.uuid4 = _ORIGINAL_UUID4 + _UUID_CALLSITE_COUNTS.clear() + _UUID_ROOT = None + _UUID_HEAD = None + + def pytest_collection_finish(session: Any) -> None: """Record exact node IDs for the validator-owned collection subprocess.""" destination = os.environ.get(VALIDATOR_COLLECTION_ENV) @@ -53,6 +107,12 @@ def pytest_collection_finish(session: Any) -> None: os.close(descriptor) +def pytest_make_parametrize_id(config: Any, val: Any, argname: str) -> str | None: + """Keep deterministic UUID parameter values in full raw pytest node IDs.""" + del config, argname + return str(val) if isinstance(val, uuid.UUID) else None + + def _object(value: Any, keys: set[str], error: str) -> dict[str, Any]: if not isinstance(value, dict) or set(value) != keys: raise EvidenceError(error) @@ -117,7 +177,7 @@ def _discover_current_modules(backend_root: Path) -> list[str]: return modules -def _collect_current_nodes(repository_root: Path) -> list[str]: +def _collect_current_nodes(repository_root: Path, head_sha: str | None = None) -> list[str]: backend_root = repository_root / "backend" modules = _discover_current_modules(backend_root) with tempfile.TemporaryDirectory(prefix="workstream-validator-") as temporary: @@ -125,6 +185,11 @@ def _collect_current_nodes(repository_root: Path) -> list[str]: environment = os.environ.copy() environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" environment[VALIDATOR_COLLECTION_ENV] = str(output) + validated_head = head_sha or _current_head(repository_root) + if SHA_RE.fullmatch(validated_head) is None: + raise EvidenceError("invalid_git_head") + environment[VALIDATOR_HEAD_ENV] = validated_head + environment[VALIDATOR_ROOT_ENV] = str(backend_root.resolve(strict=True)) validator_backend = str(Path(__file__).resolve().parents[1]) existing_pythonpath = environment.get("PYTHONPATH") environment["PYTHONPATH"] = ( @@ -154,10 +219,7 @@ def _collect_current_nodes(repository_root: Path) -> list[str]: if result.returncode != 0: raise EvidenceError("independent_pytest_collection_failed") try: - nodes = [ - json.loads(line) - for line in output.read_text(encoding="utf-8").splitlines() - ] + nodes = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: raise EvidenceError("invalid_independent_collection") from exc if not nodes or any(not isinstance(node, str) or "::" not in node for node in nodes): @@ -297,7 +359,7 @@ def validate_evidence( raise EvidenceError("noncanonical_or_duplicate_manifest_nodes") if not any(module == ADMIN_RUNNER_MODULE for _node, module, _lane, _kind in canonical): raise EvidenceError("missing_admin_runner_self_tests") - independently_collected = _collect_current_nodes(root) + independently_collected = _collect_current_nodes(root, head) canonical_ids = [nodeid for nodeid, _module, _lane, _kind in canonical] if Counter(independently_collected) != Counter(canonical_ids): raise EvidenceError("current_node_inventory_mismatch") diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 289ea634..077f7007 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -122,6 +122,43 @@ def test_manifest_has_no_exclusion_escape_hatch() -> None: }] +def test_deterministic_uuid_nodeids_match_across_full_subset_and_repeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(runner.HEAD_ENV, "e" * 40) + + def collect(module_names: tuple[str, ...]) -> dict[str, list[str]]: + runner.pytest_sessionstart(object()) + try: + result: dict[str, list[str]] = {} + for module_name in module_names: + namespace: dict[str, object] = {} + source = "import uuid\nvalues = [uuid.uuid4(), uuid.uuid4()]\n" + filename = runner.ROOT / "tests" / f"test_{module_name}.py" + exec(compile(source, str(filename), "exec"), namespace) + values = namespace["values"] + assert isinstance(values, list) + result[module_name] = [ + f"tests/test_{module_name}.py::test_value[{value}]" for value in values + ] + return result + finally: + runner.pytest_sessionfinish(object(), 0) + + first_full = collect(("alpha", "beta")) + second_full = collect(("alpha", "beta")) + subset = collect(("beta",)) + + assert first_full == second_full + assert subset["beta"] == first_full["beta"] + assert len(set(first_full["alpha"] + first_full["beta"])) == 4 + for nodeid in first_full["alpha"] + first_full["beta"]: + value = nodeid.rsplit("[", 1)[1][:-1] + generated = __import__("uuid").UUID(value) + assert generated.version == 4 + assert generated.variant == __import__("uuid").RFC_4122 + + def test_lane_command_uses_exact_nodes_and_isolation_contract(tmp_path: Path) -> None: lane = LANES[0] nodes = [f"{lane.modules[0]}::test_exact[param]"] @@ -239,7 +276,7 @@ def fake_run(*_args, **kwargs): tmp_path.joinpath("metadata").mkdir() monkeypatch.setattr(runner.subprocess, "run", fake_run) with pytest.raises(LaneError, match="invalid_collected_nodes"): - runner.collect_nodes((module,), tmp_path / "metadata") + runner.collect_nodes((module,), tmp_path / "metadata", "a" * 40) def test_timing_summary_is_derived_from_exact_four_lanes() -> None: diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index aa7a9411..80f7546d 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -6,6 +6,7 @@ import importlib.util import json from pathlib import Path +import uuid import pytest @@ -131,7 +132,7 @@ def exact_head(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( validator, "_collect_current_nodes", - lambda _root: sorted( + lambda _root, _head: sorted( [ *(f"tests/test_{index}.py::test_ok" for index in range(4)), "tests/test_isolated_database_runner.py::test_admin_custody", @@ -368,3 +369,28 @@ def test_real_collection_rejects_missing_or_foreign_manifest_node( monkeypatch.setattr(validator, "_collect_current_nodes", REAL_COLLECT_CURRENT_NODES) with pytest.raises(validator.EvidenceError, match="current_node_inventory_mismatch"): validator.validate_evidence(metadata, summary_path, tmp_path) + + +def test_independent_collections_preserve_full_deterministic_uuid_nodeid( + tmp_path: Path, +) -> None: + tests = tmp_path / "backend/tests" + tests.mkdir(parents=True) + (tests / "test_uuid_nodes.py").write_text( + "import pytest\n" + "import uuid\n" + "\n" + "VALUE = uuid.uuid4()\n" + "\n" + "@pytest.mark.parametrize('value', [VALUE])\n" + "def test_value(value):\n" + " assert value == VALUE\n", + encoding="utf-8", + ) + first = REAL_COLLECT_CURRENT_NODES(tmp_path, HEAD) + second = REAL_COLLECT_CURRENT_NODES(tmp_path, HEAD) + expected_key = "\0".join((HEAD, "tests/test_uuid_nodes.py", "4", "0")).encode() + expected_uuid = uuid.UUID(bytes=hashlib.sha256(expected_key).digest()[:16], version=4) + + assert first == second + assert first == [f"tests/test_uuid_nodes.py::test_value[{expected_uuid}]"] From cda3f1f88034038f3f7fcb3997a239590e59fc69 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 16:42:52 +0100 Subject: [PATCH 05/31] docs(ci): clarify local diagnostic boundary --- docs/operations_backend_testing.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index 3a802d88..276eb457 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -3,7 +3,13 @@ Workstream's application tests run against a new local Postgres database per invocation. Provisioning and cleanup use the admin database; the application phase receives only a strict `workstream_test_<12 lowercase hex>` database and an ephemeral login without elevated authority. -## Local full suite +## Local PostgreSQL diagnostic + +This legacy sequential command checks PostgreSQL provisioning and cleanup. It +is not complete full-suite proof because it does not start or bind a MinIO +provider. Use the hosted semantic-lane workflow below for authoritative +PostgreSQL, MinIO, exact-node, timing, API, and coverage custody. + Keep the admin URL in the environment with `postgresql+asyncpg` and a loopback host. Never put real or shared credentials in arguments, logs, evidence, or configuration. From 18ead0b15ea8526d146601efa047b253bd9ab498 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 16:48:33 +0100 Subject: [PATCH 06/31] fix(ci): confine deterministic UUIDs to collection --- backend/scripts/run_test_lanes.py | 40 ++++++++-- .../scripts/validate_test_lane_evidence.py | 76 +++++++++++++------ backend/tests/test_ci_test_lanes.py | 36 +++++++++ backend/tests/test_test_lane_evidence.py | 35 +++++++++ 4 files changed, 157 insertions(+), 30 deletions(-) diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 006d061a..91b44a0e 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -174,11 +174,14 @@ def _append_node(destination: str, node_id: str) -> None: def pytest_collection_finish(session: Any) -> None: - """Record exact selected node IDs in the collecting pytest process.""" - destination = os.environ.get(COLLECTED_ENV) - if destination: - for item in session.items: - _append_node(destination, item.nodeid) + """Record exact node IDs, then end deterministic UUID collection scope.""" + try: + destination = os.environ.get(COLLECTED_ENV) + if destination: + for item in session.items: + _append_node(destination, item.nodeid) + finally: + _restore_uuid4() def pytest_runtest_logfinish(nodeid: str, location: tuple[str, int | None, str]) -> None: @@ -249,9 +252,32 @@ def pytest_sessionstart(session: Any) -> None: def pytest_sessionfinish(session: Any, exitstatus: int) -> None: """Restore process-global UUID behavior after pytest completes.""" del session, exitstatus + _restore_uuid4() + + +def _restore_uuid4() -> None: + """Restore uuid4 and repository aliases captured during collection.""" global _ORIGINAL_UUID4, _UUID4_HEAD - if _ORIGINAL_UUID4 is not None: - uuid.uuid4 = _ORIGINAL_UUID4 + original = _ORIGINAL_UUID4 + if original is None: + _UUID4_HEAD = "" + _UUID4_ORDINALS.clear() + return + uuid.uuid4 = original + for module in tuple(sys.modules.values()): + module_file = getattr(module, "__file__", None) + if not isinstance(module_file, str): + continue + try: + Path(module_file).resolve().relative_to(ROOT) + except (OSError, ValueError): + continue + for name, value in tuple(vars(module).items()): + if ( + value is _deterministic_uuid4 + and not (module is sys.modules.get(__name__) and name == "_deterministic_uuid4") + ): + setattr(module, name, original) _ORIGINAL_UUID4 = None _UUID4_HEAD = "" _UUID4_ORDINALS.clear() diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 39e105ab..70b2aa60 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -32,7 +32,7 @@ VALIDATOR_COLLECTION_ENV = "WORKSTREAM_VALIDATOR_COLLECTION_FILE" VALIDATOR_HEAD_ENV = "WORKSTREAM_VALIDATOR_HEAD_SHA" VALIDATOR_ROOT_ENV = "WORKSTREAM_VALIDATOR_BACKEND_ROOT" -_ORIGINAL_UUID4 = uuid.uuid4 +_UUID_ORIGINAL: Any = None _UUID_CALLSITE_COUNTS: dict[tuple[str, int], int] = {} _UUID_ROOT: Path | None = None _UUID_HEAD: str | None = None @@ -47,12 +47,12 @@ def _deterministic_uuid4() -> uuid.UUID: frame = inspect.currentframe() caller = frame.f_back if frame is not None else None try: - if caller is None or _UUID_ROOT is None or _UUID_HEAD is None: - return _ORIGINAL_UUID4() + if caller is None or _UUID_ROOT is None or _UUID_HEAD is None or _UUID_ORIGINAL is None: + return uuid.UUID(bytes=os.urandom(16), version=4) try: relative = Path(caller.f_code.co_filename).resolve().relative_to(_UUID_ROOT) except (OSError, ValueError): - return _ORIGINAL_UUID4() + return _UUID_ORIGINAL() relative_file = relative.as_posix() callsite = (relative_file, caller.f_lineno) ordinal = _UUID_CALLSITE_COUNTS.get(callsite, 0) @@ -67,13 +67,15 @@ def _deterministic_uuid4() -> uuid.UUID: def pytest_sessionstart(session: Any) -> None: """Install deterministic UUIDv4 generation before test-module collection.""" del session - global _UUID_ROOT, _UUID_HEAD + global _UUID_ROOT, _UUID_HEAD, _UUID_ORIGINAL + _restore_uuid4() root = os.environ.get(VALIDATOR_ROOT_ENV) head = os.environ.get(VALIDATOR_HEAD_ENV) if root is None or head is None or SHA_RE.fullmatch(head) is None: return _UUID_ROOT = Path(root).resolve(strict=True) _UUID_HEAD = head + _UUID_ORIGINAL = uuid.uuid4 _UUID_CALLSITE_COUNTS.clear() uuid.uuid4 = _deterministic_uuid4 @@ -81,30 +83,58 @@ def pytest_sessionstart(session: Any) -> None: def pytest_sessionfinish(session: Any, exitstatus: int) -> None: """Restore process UUID behavior after validator-owned collection.""" del session, exitstatus - global _UUID_ROOT, _UUID_HEAD - uuid.uuid4 = _ORIGINAL_UUID4 - _UUID_CALLSITE_COUNTS.clear() - _UUID_ROOT = None - _UUID_HEAD = None + _restore_uuid4() def pytest_collection_finish(session: Any) -> None: """Record exact node IDs for the validator-owned collection subprocess.""" - destination = os.environ.get(VALIDATOR_COLLECTION_ENV) - if destination is None: - return - path = Path(destination) - data = b"".join( - json.dumps(item.nodeid).encode("utf-8") + b"\n" - for item in sorted(session.items, key=lambda item: item.nodeid) - ) - descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: - view = memoryview(data) - while view: - view = view[os.write(descriptor, view) :] + destination = os.environ.get(VALIDATOR_COLLECTION_ENV) + if destination is None: + return + path = Path(destination) + data = b"".join( + json.dumps(item.nodeid).encode("utf-8") + b"\n" + for item in sorted(session.items, key=lambda item: item.nodeid) + ) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + view = memoryview(data) + while view: + view = view[os.write(descriptor, view) :] + finally: + os.close(descriptor) finally: - os.close(descriptor) + _restore_uuid4() + + +def _restore_uuid4() -> None: + """Restore uuid4 and repository aliases captured during collection.""" + global _UUID_ROOT, _UUID_HEAD, _UUID_ORIGINAL + original = _UUID_ORIGINAL + root = _UUID_ROOT + if original is None: + _UUID_CALLSITE_COUNTS.clear() + _UUID_ROOT = None + _UUID_HEAD = None + return + uuid.uuid4 = original + if root is not None: + for module in tuple(sys.modules.values()): + module_file = getattr(module, "__file__", None) + if not isinstance(module_file, str): + continue + try: + Path(module_file).resolve().relative_to(root) + except (OSError, ValueError): + continue + for name, value in tuple(vars(module).items()): + if value is _deterministic_uuid4: + setattr(module, name, original) + _UUID_ORIGINAL = None + _UUID_CALLSITE_COUNTS.clear() + _UUID_ROOT = None + _UUID_HEAD = None def pytest_make_parametrize_id(config: Any, val: Any, argname: str) -> str | None: diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 077f7007..187616f6 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -8,6 +8,8 @@ import subprocess import sys import time +import types +import uuid import pytest # type: ignore[import-not-found] @@ -159,6 +161,40 @@ def collect(module_names: tuple[str, ...]) -> dict[str, list[str]]: assert generated.variant == __import__("uuid").RFC_4122 +def test_uuid4_and_repository_import_alias_are_restored_before_test_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(runner.HEAD_ENV, "f" * 40) + monkeypatch.delenv(runner.COLLECTED_ENV, raising=False) + original = uuid.uuid4 + module_name = "tests.test_collection_uuid_alias" + module = types.ModuleType(module_name) + module.__file__ = str(runner.ROOT / "tests" / "test_collection_uuid_alias.py") + sys.modules[module_name] = module + runner.pytest_sessionstart(object()) + try: + exec( + compile( + "from uuid import uuid4\nCOLLECTED_VALUE = uuid4()\n", + module.__file__, + "exec", + ), + module.__dict__, + ) + collected_value = module.COLLECTED_VALUE + assert module.uuid4 is runner._deterministic_uuid4 + runner.pytest_collection_finish(type("Session", (), {"items": []})()) + + assert uuid.uuid4 is original + assert module.uuid4 is original + assert module.COLLECTED_VALUE is collected_value + assert module.uuid4() != module.uuid4() + finally: + runner.pytest_sessionfinish(object(), 0) + sys.modules.pop(module_name, None) + assert uuid.uuid4 is original + + def test_lane_command_uses_exact_nodes_and_isolation_contract(tmp_path: Path) -> None: lane = LANES[0] nodes = [f"{lane.modules[0]}::test_exact[param]"] diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index 80f7546d..4b9a0136 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -6,6 +6,8 @@ import importlib.util import json from pathlib import Path +import sys +import types import uuid import pytest @@ -394,3 +396,36 @@ def test_independent_collections_preserve_full_deterministic_uuid_nodeid( assert first == second assert first == [f"tests/test_uuid_nodes.py::test_value[{expected_uuid}]"] + + +def test_collection_finish_restores_uuid4_and_repository_aliases( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + backend = tmp_path / "backend" + backend.mkdir() + destination = tmp_path / "nodes.jsonl" + monkeypatch.setenv(validator.VALIDATOR_ROOT_ENV, str(backend)) + monkeypatch.setenv(validator.VALIDATOR_HEAD_ENV, HEAD) + monkeypatch.setenv(validator.VALIDATOR_COLLECTION_ENV, str(destination)) + original = uuid.uuid4 + validator.pytest_sessionstart(object()) + wrapper = uuid.uuid4 + assert wrapper is validator._deterministic_uuid4 + + repository_module = types.ModuleType("validator_alias_fixture") + repository_module.__file__ = str(backend / "tests/test_alias.py") + repository_module.imported_uuid4 = wrapper + monkeypatch.setitem(sys.modules, repository_module.__name__, repository_module) + session = types.SimpleNamespace( + items=[types.SimpleNamespace(nodeid="tests/test_alias.py::test_value[full-uuid]")] + ) + validator.pytest_collection_finish(session) + + assert destination.read_text(encoding="utf-8").splitlines() == [ + '"tests/test_alias.py::test_value[full-uuid]"' + ] + assert uuid.uuid4 is original + assert repository_module.imported_uuid4 is original + assert validator._UUID_ORIGINAL is None + validator.pytest_sessionfinish(object(), 0) + assert uuid.uuid4 is original From 2dfcf06f21e4dac843c768f5c7fce85d2a8e2a89 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 16:57:10 +0100 Subject: [PATCH 07/31] docs(ci): align hosted proof sequence --- docs/operations_backend_testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index 276eb457..f82af57e 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -73,8 +73,8 @@ concurrent dependency lanes. This avoids arbitrary shard fan-out and artifact fan-in while retaining exact node and coverage custody. The job binds the checkout to `GITHUB_SHA`, installs and asserts exact Ruff -`0.15.22`, runs lint, docstrings, and the isolated-resource runner tests, then -collects every canonical pytest node. The independent evidence validator must +`0.15.22`, runs lint and docstrings, starts MinIO, then collects every canonical +pytest node. The independent evidence validator must accept the collection before execution begins. Each lane receives a distinct runner-created database and role plus a distinct MinIO bucket/prefix custody record. The S3 lane owns the actual `workstream-artifacts` test bucket and a From d3b2a655df1361a7b68d88e4afc5bf2b13ef1c0e Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 17:07:05 +0100 Subject: [PATCH 08/31] docs(agent-loop): record semantic lane trust evidence --- .../WS-CI-001-02B-internal-review-evidence.md | 99 ++++++++ .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 221 ++++++++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 .agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md create mode 100644 .agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md new file mode 100644 index 00000000..ce0548af --- /dev/null +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -0,0 +1,99 @@ +# Internal Review Evidence + +## Chunk + +`WS-CI-001-02B` — Exact-Custody Semantic Test Lanes + +## Required Statements + +open sub-agent sessions: none + +valid findings addressed: yes + +## Reviewed Revision + +Reviewed code SHA: 2dfcf06f21e4dac843c768f5c7fce85d2a8e2a89 + +Reviewed at: 2026-07-24T16:03:45Z + +Reviewer run IDs: ci02b_senior_review, ci02b_qa_review, +ci02b_security_review, ci02b_product_ops_review, +ci02b_restart_arch_review, ci02b_restart_ci_review, ci02b_contract_gap, +ci02b_source_audit, ci02b_test_delta_review + +After the reviewed SHA, only evidence and status files changed. + +## Reviewer Results + +| Reviewer | Result | Blocking findings | Notes | +|---|---:|---|---| +| senior engineering | PASS WITH LOW RISKS | None | Independent UUID implementations must remain behavior-compatible. | +| QA/test | PASS | None | Exact inventory, failure custody, resource isolation, and timing evidence pass. | +| security/auth | PASS WITH LOW RISKS | None | A force-kill after bounded cleanup can leave local resources; hosted CI fails closed. | +| product/ops | PASS | None | Hosted operator evidence and Konan attribution are explicit; product behavior is unchanged. | +| architecture | PASS | None | Collection-only UUID stabilization restores runtime aliases before test bodies. | +| CI integrity | PASS | None | Ruff, API E2E, exact-node validation, coverage floors, and timing remain blocking. | +| docs | PASS | None | Operations and status documentation match the final workflow sequence. | +| reuse/dedup | PASS | None | Canonical isolation runner is reused; independent validator duplication is intentional. | +| test delta | PASS | None | Deleted shard tests are replaced without product-test weakening or deselection. | + +## Valid Findings Addressed + +- Replaced invalid underscore-bearing MinIO bucket names with S3-valid, + collision-tested lane namespaces and bound the real S3 test bucket to its + owning lane. +- Removed the isolated-runner test exclusion. All runner self-test nodes remain + canonical and execute under the explicit `admin_runner_self_test` kind while + ordinary children never receive the admin database URL. +- Added an independent recursive pytest collection that rejects a + self-consistent but missing or foreign runner manifest. +- Preserved full parametrized node IDs while stabilizing import-time UUIDv4 + parameters by exact head, callsite, line, and ordinal. Both plugins restore + `uuid.uuid4` and repository import aliases before test bodies execute. +- Made negative process exits, skip, deselection, interruption, partial + completion, digest drift, and shared database/storage/coverage custody fail. +- Removed intermediate coverage files after authenticated per-lane combination + so the final workflow accepts exactly four public lane artifacts before one + literal `coverage combine`. +- Added fail-closed hosted evidence for total Backend wall time, slowest lane, + aggregate runner seconds, exact node counts, coverage percentage, and raw + digests; total wall time above 480 seconds fails with no silent waiver. +- Redacted direct admin-runner logs and aligned the operations runbook with the + actual hosted sequence and local diagnostic boundary. + +## Commands Run + +```bash +cd backend +ruff check app tests scripts +python -m pytest -q tests/test_ci_test_lanes.py tests/test_test_lane_evidence.py +python scripts/run_test_lanes.py --collect-only --metadata-dir "$tmp/collect" --summary-json "$tmp/collect-summary.json" +python scripts/validate_test_lane_evidence.py --metadata-dir "$tmp/collect" --summary-json "$tmp/collect-summary.json" +cd .. +python3 scripts/test_agent_gates.py +python3 scripts/update_post_merge_memory.py validate-merge-intent --repository-root . --base-ref origin/main +python3 scripts/check_markdown_links.py docs/operations_backend_testing.md .agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +git diff --check origin/main...HEAD +``` + +## Results + +- Ruff passed with exact local `ruff 0.15.22`. +- 60 focused lane-runner and independent-validator tests passed. +- 100 Agent Gate tests passed. +- Two independent full collections agreed on 2,046 exact pytest nodes at the + reviewed head; independent evidence validation passed. +- Merge intent, Markdown links, stale wording, and diff integrity passed. +- Local full service execution was not used as hosted performance evidence. + +## Remaining Risks + +- The exact GitHub Backend job must still prove real PostgreSQL and MinIO + concurrency, API E2E, 78/90 coverage gates, and total wall time at or below + 480 seconds on the final PR head. +- A force-kill after the bounded cleanup grace can leave runner-owned resources + on persistent local services. Evidence fails closed; operators must inspect + and remove only exact recorded resources. +- The runner and independent validator intentionally duplicate the stable UUID + collection specification. Their separate tests must prevent common-mode drift. + diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md new file mode 100644 index 00000000..8d5663e0 --- /dev/null +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -0,0 +1,221 @@ +# PR Trust Bundle + +## Chunk + +`WS-CI-001-02B` — Exact-Custody Semantic Test Lanes + +Merge intent: `.agent-loop/merge-intents/WS-CI-001-02B.json` + +## Goal + +Replace arbitrary Backend shards with four concurrent dependency lanes while +proving every canonical pytest node, isolated resource, coverage byte, and +hosted timing outcome on the exact PR head. + +## Human-approved intent + +- Initiative plan: `../PLAN.md` +- Chunk contract: `../chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` + +## Signed Start Provenance + +- Signed start run: `30101104403` +- Authorized main SHA: `bcf1292e1a591e3e84bf8ee212ee7191d80741fa` +- Phase: `implementation` +- Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` +- Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` +- Reviewed implementation SHA: `2dfcf06f21e4dac843c768f5c7fce85d2a8e2a89` + +Only independently verified signed automation state is canonical authority. +PR prose and checked boxes are navigation evidence, not authorization. + +## What changed + +- Replaced the four-job shard matrix and artifact fan-in with one required job + orchestrating four concurrent semantic test lanes. +- Added exact recursive node collection, independent recollection/validation, + authenticated per-lane evidence, and exactly one coverage combination. +- Extended the canonical isolated runner with per-lane PostgreSQL and MinIO + namespace custody, redaction, heartbeat, and bounded cleanup evidence. +- Pinned and asserted Ruff `0.15.22` in the workflow to remove resolver drift. +- Added hosted exact-head timing, node-count, coverage, and digest evidence with + a blocking 480-second wall-time ceiling. +- Deleted the obsolete shard runner and shard tests and updated operations docs. + +## Why it changed + +The prior shard topology duplicated services and artifact fan-in while Backend +CI remained slow. A newly released unbounded Ruff version also broke every PR. +The replacement must improve critical-path time without hiding tests, sharing +mutable resources, weakening coverage, or trusting self-authored evidence. + +## Design chosen + +One job starts pinned PostgreSQL and MinIO services, then a repository-owned +orchestrator runs four dependency lanes concurrently. Ordinary lanes delegate +resource custody to `run_isolated_tests.py`; its own self-tests execute as a +separately classified admin phase inside canonical node evidence. An independent +validator recollects current pytest nodes and verifies exact head, nodes, +resources, coverage bytes, failures, and timings before combination. + +## Alternatives rejected + +- Wholesale cherry-pick of `e22e9fba`/`c73b2893`: rejected because it changed + forbidden product tests, initiative artifacts, and lacked exact node custody. +- Keep arbitrary shards: rejected because it preserves duplicated services and + artifact fan-in rather than dependency ownership. +- Trust the runner manifest: rejected because one writer could omit nodes while + producing self-consistent evidence. +- Strip parametrized IDs: rejected because it loses exact node identity. +- Ignore new Ruff findings or loosen lint: rejected as CI weakening. + +## Scope control + +### Allowed files changed + +- `.github/workflows/backend.yml` +- `backend/scripts/ci_test_shards.py` (deleted) +- `backend/scripts/run_isolated_tests.py` +- `backend/scripts/run_test_lanes.py` +- `backend/scripts/validate_test_lane_evidence.py` +- `backend/tests/test_ci_test_shards.py` (deleted) +- `backend/tests/test_ci_test_lanes.py` +- `backend/tests/test_isolated_database_runner.py` +- `backend/tests/test_test_lane_evidence.py` +- `docs/operations_backend_testing.md` +- `scripts/test_agent_gates.py` +- Initiative status, two review files, and one merge intent + +### Files outside scope + +- None. + +## Product Behavior + +- [x] No Workstream product behavior changed. + +## Acceptance criteria proof + +- [x] Recursive exact-node inventory with no exclusion escape — 2,046 nodes + independently recollected and validated at the reviewed head. +- [x] Four concurrent dependency lanes with distinct database/role and MinIO + bucket/prefix custody — implementation and adversarial tests pass; hosted + service proof remains required. +- [x] Missing, duplicate, foreign, skipped, deselected, interrupted, partial, + digest-drifted, and shared-custody evidence fails in focused tests. +- [x] Exactly four authenticated coverage artifacts precede one literal + `coverage combine`; global 78 and every protected 90 floor remain blocking. +- [x] Hosted evidence records wall, slowest lane, aggregate runner seconds, + counts, coverage, and digests and blocks above 480 seconds. +- [x] Konan remains a Git contributor through the immutable `Co-authored-by` + trailer on implementation commit `161417ff`; eligible source commits are + `e22e9fba` and `c73b2893`. Later custody repairs are Abiorh-authored. + +## Tests/checks run + +```bash +cd backend +ruff check app tests scripts +python -m pytest -q tests/test_ci_test_lanes.py tests/test_test_lane_evidence.py +python scripts/run_test_lanes.py --collect-only --metadata-dir "$tmp/collect" --summary-json "$tmp/collect-summary.json" +python scripts/validate_test_lane_evidence.py --metadata-dir "$tmp/collect" --summary-json "$tmp/collect-summary.json" +cd .. +python3 scripts/test_agent_gates.py +python3 scripts/update_post_merge_memory.py validate-merge-intent --repository-root . --base-ref origin/main +python3 scripts/check_markdown_links.py docs/operations_backend_testing.md .agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +git diff --check origin/main...HEAD +``` + +Result summary: Ruff passed; 60 focused tests passed; 100 Agent Gates passed; +two independent full collections agreed on 2,046 exact nodes; merge intent, +links, stale wording, and diff integrity passed. + +## Test delta + +### Tests added + +- Semantic-lane orchestration and exact inventory tests. +- Independent evidence-validator corruption and real recollection tests. +- MinIO namespace, cleanup, redaction, signal, and timing custody tests. + +### Tests modified + +- Isolated-runner lifecycle tests and Agent Gate workflow regression tests. + +### Tests removed/skipped + +- Deleted only `test_ci_test_shards.py` with its obsolete shard implementation. +- No product test was skipped, deselected, removed, or weakened. + +## CI integrity + +- [x] Coverage thresholds unchanged and blocking +- [x] Exact Ruff pin removes resolver drift without suppressions +- [x] No typecheck or package-script weakening +- [x] No workflow weakening, `continue-on-error`, or path suppression +- [x] No unpinned new GitHub Action or service image +- [x] Checkout credential persistence disabled +- [ ] Exact final GitHub Backend job passes in at most 480 seconds + +## External review + +External review response file, if findings are posted: + +- `WS-CI-001-02B-external-review-response.md` + +| Source | Status | Notes | +|---|---:|---| +| CodeRabbit | Pending | Supplementary external review after publication. | +| GitHub checks | Pending | Exact final head must pass Agent Gates and Backend. | + +## Reviewer results + +Reviewed code SHA: `2dfcf06f21e4dac843c768f5c7fce85d2a8e2a89` + +Reviewed at: `2026-07-24T16:03:45Z` + +Reviewer run IDs: `ci02b_senior_review`, `ci02b_qa_review`, +`ci02b_security_review`, `ci02b_product_ops_review`, +`ci02b_restart_arch_review`, `ci02b_restart_ci_review`, +`ci02b_contract_gap`, `ci02b_source_audit`, `ci02b_test_delta_review` + +| Reviewer | Result | Blocking findings | Notes | +|---|---:|---|---| +| senior engineering | PASS WITH LOW RISKS | None | Independent collector specifications must remain aligned. | +| QA/test | PASS | None | Exact node, resource, failure, and timing proof passes. | +| security/auth | PASS WITH LOW RISKS | None | Bounded local force-kill cleanup risk remains documented. | +| product/ops | PASS | None | Operator evidence and attribution pass. | +| architecture | PASS | None | No product/runtime boundary drift remains. | +| CI integrity | PASS | None | Existing blocking gates are preserved. | +| docs | PASS | None | Runbook and status match the workflow. | +| reuse/dedup | PASS | None | Independent duplication is intentional. | +| test delta | PASS | None | No weakened product tests or gates. | + +## Remaining risks + +- Hosted PostgreSQL/MinIO execution, API E2E, complete coverage, and the + 480-second target remain unproven until GitHub runs the exact PR head. +- Persistent local services can retain exact runner-owned resources if forced + cleanup exceeds its bounded grace; evidence fails and operators must inspect + only the recorded resource identities. + +## Follow-up work + +After merge and automated memory reconciliation, stop. `WS-CI-001-03` remains +future planning only and is not declared as a successor. + +## Human review focus + +- Exact node custody and independent recollection. +- Admin credential isolation and real S3 bucket ownership. +- Exactly four coverage files, one combine, and unchanged 78/90 floors. +- Hosted wall time and aggregate runner evidence. +- Konan attribution and bounded repair authorship. + +## Human ownership + +- [ ] I can explain what changed. +- [ ] I can explain why it changed. +- [ ] I know what could break. +- [ ] I accept the remaining risks. +- [ ] The user explicitly approved this specific PR for merge. From cc8ee4f1a03305e8c6e5830cff012effa210959b Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 17:19:41 +0100 Subject: [PATCH 09/31] fix(ci): initialize semantic lane evidence root --- .github/workflows/backend.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 228b57e3..6b751118 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -97,11 +97,14 @@ jobs: working-directory: backend env: PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" - run: >- - python scripts/run_test_lanes.py - --collect-only - --metadata-dir .ci/test-lanes/collect - --summary-json .ci/test-lanes/collect-summary.json + shell: bash + run: | + set -euo pipefail + install -d -m 700 .ci/test-lanes + python scripts/run_test_lanes.py \ + --collect-only \ + --metadata-dir .ci/test-lanes/collect \ + --summary-json .ci/test-lanes/collect-summary.json - name: Validate canonical semantic-lane inventory working-directory: backend From cb6371914bf5f8af4f56f0ff58f60a17080f394d Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 17:33:44 +0100 Subject: [PATCH 10/31] docs(agent-loop): record semantic lane bootstrap repair --- .../WS-CI-001-02B-internal-review-evidence.md | 17 +++++++++++++---- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 12 +++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index ce0548af..4263aecb 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,14 +12,16 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: 2dfcf06f21e4dac843c768f5c7fce85d2a8e2a89 +Reviewed code SHA: cc8ee4f1a03305e8c6e5830cff012effa210959b -Reviewed at: 2026-07-24T16:03:45Z +Reviewed at: 2026-07-24T16:30:32Z Reviewer run IDs: ci02b_senior_review, ci02b_qa_review, ci02b_security_review, ci02b_product_ops_review, ci02b_restart_arch_review, ci02b_restart_ci_review, ci02b_contract_gap, -ci02b_source_audit, ci02b_test_delta_review +ci02b_source_audit, ci02b_test_delta_review, ci02b_bootstrap_senior, +ci02b_bootstrap_qa, ci02b_bootstrap_security, ci02b_bootstrap_ops, +ci02b_bootstrap_arch, ci02b_bootstrap_ci After the reviewed SHA, only evidence and status files changed. @@ -37,6 +39,11 @@ After the reviewed SHA, only evidence and status files changed. | reuse/dedup | PASS | None | Canonical isolation runner is reused; independent validator duplication is intentional. | | test delta | PASS | None | Deleted shard tests are replaced without product-test weakening or deselection. | +The bootstrap repair review initially blocked publication because the reviewed +SHA was stale and this file had an extra blank line at EOF. Both evidence +defects are corrected here. All six repair reviewers accepted the fixed-path, +mode-700 evidence-root initialization; CI integrity found no weakened gate. + ## Valid Findings Addressed - Replaced invalid underscore-bearing MinIO bucket names with S3-valid, @@ -60,6 +67,9 @@ After the reviewed SHA, only evidence and status files changed. digests; total wall time above 480 seconds fails with no silent waiver. - Redacted direct admin-runner logs and aligned the operations runbook with the actual hosted sequence and local diagnostic boundary. +- Added explicit fixed-path initialization of `.ci/test-lanes` before the first + hosted collection. This closes the observed `invalid_lane_outputs` bootstrap + failure without allowing the runner to create an unowned parent directory. ## Commands Run @@ -96,4 +106,3 @@ git diff --check origin/main...HEAD and remove only exact recorded resources. - The runner and independent validator intentionally duplicate the stable UUID collection specification. Their separate tests must prevent common-mode drift. - diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 8d5663e0..60963673 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -24,7 +24,7 @@ hosted timing outcome on the exact PR head. - Phase: `implementation` - Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` - Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` -- Reviewed implementation SHA: `2dfcf06f21e4dac843c768f5c7fce85d2a8e2a89` +- Reviewed implementation SHA: `cc8ee4f1a03305e8c6e5830cff012effa210959b` Only independently verified signed automation state is canonical authority. PR prose and checked boxes are navigation evidence, not authorization. @@ -40,6 +40,8 @@ PR prose and checked boxes are navigation evidence, not authorization. - Pinned and asserted Ruff `0.15.22` in the workflow to remove resolver drift. - Added hosted exact-head timing, node-count, coverage, and digest evidence with a blocking 480-second wall-time ceiling. +- Initialized the fixed `.ci/test-lanes` evidence root with mode 700 before the + first collection, closing the hosted `invalid_lane_outputs` bootstrap failure. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed @@ -170,15 +172,19 @@ External review response file, if findings are posted: ## Reviewer results -Reviewed code SHA: `2dfcf06f21e4dac843c768f5c7fce85d2a8e2a89` +Reviewed code SHA: `cc8ee4f1a03305e8c6e5830cff012effa210959b` -Reviewed at: `2026-07-24T16:03:45Z` +Reviewed at: `2026-07-24T16:30:32Z` Reviewer run IDs: `ci02b_senior_review`, `ci02b_qa_review`, `ci02b_security_review`, `ci02b_product_ops_review`, `ci02b_restart_arch_review`, `ci02b_restart_ci_review`, `ci02b_contract_gap`, `ci02b_source_audit`, `ci02b_test_delta_review` +Bootstrap repair reviewer run IDs: `ci02b_bootstrap_senior`, +`ci02b_bootstrap_qa`, `ci02b_bootstrap_security`, `ci02b_bootstrap_ops`, +`ci02b_bootstrap_arch`, `ci02b_bootstrap_ci`. + | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| | senior engineering | PASS WITH LOW RISKS | None | Independent collector specifications must remain aligned. | From 239adb178229c72951862780e5ce237f73113400 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 17:51:20 +0100 Subject: [PATCH 11/31] fix(ci): preserve schema lane coverage custody --- backend/scripts/run_test_lanes.py | 32 +++++++++++-- backend/tests/test_ci_test_lanes.py | 71 +++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 91b44a0e..903c0788 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -535,6 +535,7 @@ def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str "coverage_path": active.coverage_path, "deselected_nodes": deselected, "elapsed_seconds": round(elapsed, 3), + "execution_kind": active.execution_kind, "execution_exit_code": exit_code, "interrupted": active.interrupted_at is not None or active.timed_out, "skipped_nodes": skipped, @@ -544,7 +545,7 @@ def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str def _combine_coverage(sources: list[Path], destination: Path) -> None: regular = [path for path in sources if path.is_file() and not path.is_symlink()] if len(regular) != len(sources): - return + raise LaneError("missing_lane_coverage") if len(regular) == 1: shutil.copyfile(regular[0], destination) return @@ -563,7 +564,30 @@ def _finalize_lane( evidence_path = metadata_dir / f"{lane.name}.json" isolation_path = metadata_dir / f"{lane.name}.database.json" coverage_path = metadata_dir / f".coverage.{lane.name}" - _combine_coverage([unit["coverage_path"] for unit in units], coverage_path) + execution_exit_code = _aggregate_exit_codes( + [unit["execution_exit_code"] for unit in units] + ) + ordinary_coverage = [ + unit["coverage_path"] + for unit in units + if unit["execution_kind"] == ORDINARY_KIND + ] + if execution_exit_code == 0 and not ordinary_coverage: + raise LaneError("missing_ordinary_lane_coverage") + optional_admin_coverage = [ + unit["coverage_path"] + for unit in units + if unit["execution_kind"] == ADMIN_KIND + and unit["coverage_path"].is_file() + and not unit["coverage_path"].is_symlink() + ] + selected_coverage = [*ordinary_coverage, *optional_admin_coverage] + if execution_exit_code == 0: + _combine_coverage(selected_coverage, coverage_path) + elif selected_coverage and all( + path.is_file() and not path.is_symlink() for path in selected_coverage + ): + _combine_coverage(selected_coverage, coverage_path) evidence = { "collected_nodes": sorted(node for unit in units for node in unit["collected_nodes"]), "completed_nodes": sorted(node for unit in units for node in unit["completed_nodes"]), @@ -588,9 +612,7 @@ def _finalize_lane( "elapsed_seconds": round(max(unit["elapsed_seconds"] for unit in units), 3), "evidence_file": evidence_path.name, "evidence_sha256": _sha256(evidence_path.read_bytes()), - "execution_exit_code": _aggregate_exit_codes( - [unit["execution_exit_code"] for unit in units] - ), + "execution_exit_code": execution_exit_code, "interrupted": any(unit["interrupted"] for unit in units), "name": lane.name, } diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 187616f6..92b8f485 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -260,6 +260,70 @@ def test_admin_wrapper_redacts_admin_url_before_persisted_output() -> None: assert result.stdout.strip() == "[REDACTED_ADMIN_DATABASE_URL]" +def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverage( + tmp_path: Path, +) -> None: + lane = next(lane for lane in LANES if lane.name == "schema_contracts") + isolation = tmp_path / f"{lane.name}.database.json" + isolation.write_text("{}\n", encoding="utf-8") + ordinary_coverage = tmp_path / ".coverage.unit.schema_contracts" + ordinary_coverage.write_bytes(b"ordinary coverage") + units = [ + { + "collection_exit_code": 0, + "collected_nodes": ["tests/test_alembic.py::test_one"], + "completed_nodes": ["tests/test_alembic.py::test_one"], + "coverage_path": ordinary_coverage, + "deselected_nodes": [], + "elapsed_seconds": 1.0, + "execution_kind": runner.ORDINARY_KIND, + "execution_exit_code": 0, + "interrupted": False, + "skipped_nodes": [], + }, + { + "collection_exit_code": 0, + "collected_nodes": [f"{runner.ADMIN_RUNNER_MODULE}::test_one"], + "completed_nodes": [f"{runner.ADMIN_RUNNER_MODULE}::test_one"], + "coverage_path": tmp_path / ".coverage.unit.schema_contracts.admin", + "deselected_nodes": [], + "elapsed_seconds": 2.0, + "execution_kind": runner.ADMIN_KIND, + "execution_exit_code": 0, + "interrupted": False, + "skipped_nodes": [], + }, + ] + + row = runner._finalize_lane(lane, units, tmp_path) + + combined = tmp_path / row["coverage_file"] + assert combined.read_bytes() == b"ordinary coverage" + assert row["coverage_sha256"] == hashlib.sha256(b"ordinary coverage").hexdigest() + + +def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None: + lane = LANES[0] + tmp_path.joinpath(f"{lane.name}.database.json").write_text("{}\n", encoding="utf-8") + units = [ + { + "collection_exit_code": 0, + "collected_nodes": [f"{lane.modules[0]}::test_one"], + "completed_nodes": [f"{lane.modules[0]}::test_one"], + "coverage_path": tmp_path / ".coverage.unit.missing", + "deselected_nodes": [], + "elapsed_seconds": 1.0, + "execution_kind": runner.ORDINARY_KIND, + "execution_exit_code": 0, + "interrupted": False, + "skipped_nodes": [], + } + ] + + with pytest.raises(LaneError, match="missing_lane_coverage"): + runner._finalize_lane(lane, units, tmp_path) + + def test_collect_only_writes_raw_digest_bound_validator_schema( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -347,9 +411,10 @@ def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path "collection_exit_code": 0, "completed_nodes": [f"{lane.modules[0]}::test_one"], "coverage_path": source, - "deselected_nodes": [], - "elapsed_seconds": float(index + 1), - "execution_exit_code": 0, + "deselected_nodes": [], + "elapsed_seconds": float(index + 1), + "execution_kind": runner.ORDINARY_KIND, + "execution_exit_code": 0, "interrupted": False, "skipped_nodes": [], } From 80dd8c87f1e7133430ca8abda4a8035a6809fd91 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 17:56:05 +0100 Subject: [PATCH 12/31] docs(agent-loop): record coverage custody repair --- .../WS-CI-001-02B-internal-review-evidence.md | 18 +++++++++++++++--- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 11 +++++++---- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index 4263aecb..59b3b5b5 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,9 +12,9 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: cc8ee4f1a03305e8c6e5830cff012effa210959b +Reviewed code SHA: 239adb178229c72951862780e5ce237f73113400 -Reviewed at: 2026-07-24T16:30:32Z +Reviewed at: 2026-07-24T16:55:21Z Reviewer run IDs: ci02b_senior_review, ci02b_qa_review, ci02b_security_review, ci02b_product_ops_review, @@ -44,6 +44,14 @@ SHA was stale and this file had an extra blank line at EOF. Both evidence defects are corrected here. All six repair reviewers accepted the fixed-path, mode-700 evidence-root initialization; CI integrity found no weakened gate. +The exact-head coverage repair review found no code blocker. Successful lanes +still require non-symlink ordinary coverage; an admin runner self-test coverage +file is combined only when it exists because that direct self-test process can +legitimately collect no `app` data. Failed lanes retain nonzero exit evidence +instead of allowing missing coverage to mask the original failure. Optional +admin coverage absence remains a documented low diagnostic risk, not a product +coverage or exact-node custody bypass. + ## Valid Findings Addressed - Replaced invalid underscore-bearing MinIO bucket names with S3-valid, @@ -70,6 +78,10 @@ mode-700 evidence-root initialization; CI integrity found no weakened gate. - Added explicit fixed-path initialization of `.ci/test-lanes` before the first hosted collection. This closes the observed `invalid_lane_outputs` bootstrap failure without allowing the runner to create an unowned parent directory. +- Repaired schema-lane coverage finalization after hosted run `30109561363` + proved that its ordinary unit emitted coverage while the direct admin + self-test unit legitimately emitted none. Missing ordinary coverage still + fails closed, and regression tests cover both paths. ## Commands Run @@ -89,7 +101,7 @@ git diff --check origin/main...HEAD ## Results - Ruff passed with exact local `ruff 0.15.22`. -- 60 focused lane-runner and independent-validator tests passed. +- 62 focused lane-runner and independent-validator tests passed. - 100 Agent Gate tests passed. - Two independent full collections agreed on 2,046 exact pytest nodes at the reviewed head; independent evidence validation passed. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 60963673..0a21f34b 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -24,7 +24,7 @@ hosted timing outcome on the exact PR head. - Phase: `implementation` - Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` - Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` -- Reviewed implementation SHA: `cc8ee4f1a03305e8c6e5830cff012effa210959b` +- Reviewed implementation SHA: `239adb178229c72951862780e5ce237f73113400` Only independently verified signed automation state is canonical authority. PR prose and checked boxes are navigation evidence, not authorization. @@ -42,6 +42,9 @@ PR prose and checked boxes are navigation evidence, not authorization. a blocking 480-second wall-time ceiling. - Initialized the fixed `.ci/test-lanes` evidence root with mode 700 before the first collection, closing the hosted `invalid_lane_outputs` bootstrap failure. +- Preserved schema-lane ordinary coverage when its direct admin self-test unit + legitimately emits no `app` coverage; missing ordinary coverage remains a + hard failure. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed @@ -128,7 +131,7 @@ python3 scripts/check_markdown_links.py docs/operations_backend_testing.md .agen git diff --check origin/main...HEAD ``` -Result summary: Ruff passed; 60 focused tests passed; 100 Agent Gates passed; +Result summary: Ruff passed; 62 focused tests passed; 100 Agent Gates passed; two independent full collections agreed on 2,046 exact nodes; merge intent, links, stale wording, and diff integrity passed. @@ -172,9 +175,9 @@ External review response file, if findings are posted: ## Reviewer results -Reviewed code SHA: `cc8ee4f1a03305e8c6e5830cff012effa210959b` +Reviewed code SHA: `239adb178229c72951862780e5ce237f73113400` -Reviewed at: `2026-07-24T16:30:32Z` +Reviewed at: `2026-07-24T16:55:21Z` Reviewer run IDs: `ci02b_senior_review`, `ci02b_qa_review`, `ci02b_security_review`, `ci02b_product_ops_review`, From 4afb6fbae89613030b24af3004d770cddab9bafe Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 19:25:25 +0100 Subject: [PATCH 13/31] perf(ci): add exact-custody semantic execution units --- .../STATUS.md | 6 +- backend/scripts/run_test_lanes.py | 144 +++++++++++++++--- .../scripts/validate_test_lane_evidence.py | 140 ++++++++++------- backend/tests/test_ci_test_lanes.py | 73 ++++++++- backend/tests/test_test_lane_evidence.py | 90 +++++++---- docs/operations_backend_testing.md | 28 ++-- scripts/test_agent_gates.py | 9 ++ 7 files changed, 364 insertions(+), 126 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md index bff75000..7580de22 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md @@ -7,8 +7,10 @@ - Later proposed chunk: `WS-CI-001-03` planning only; not a declared successor - Human direction: preserve Konan's authorship and measured work from PR #180, but adopt it only under prospective zero-trust scope and evidence -- Current gate: exact-custody semantic-lane implementation is in deterministic - proof; hosted full Backend execution, exact node/coverage/resource evidence, +- Current gate: exact head `80dd8c87` proved all 2,048 nodes, coverage floors, + API E2E, and four-lane custody but recorded 533.218 seconds, exceeding the + blocking 480-second ceiling. The active repair introduces fixed semantic + execution units with six isolated resource records; hosted timing, refreshed internal review, external review, and user-owned merge decision remain - Deferred option: routing/cache/timing reassessment is future `WS-CI-001-03`, with no start or successor declaration diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 903c0788..92de6843 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -130,6 +130,36 @@ class TestLane: ), ), ) +SEMANTIC_UNITS: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = { + "control_plane": ( + ( + "authority", + ( + "tests/test_actors.py", + "tests/test_api_rate_controls.py", + "tests/test_audit.py", + "tests/test_auth.py", + "tests/test_authorization.py", + ), + ), + ("projects", ("tests/test_projects.py",)), + ), + "execution_plane": ( + ( + "artifacts", + ( + "tests/test_artifact_admission.py", + "tests/test_artifact_operator_api.py", + "tests/test_artifact_recovery.py", + "tests/test_db_session.py", + "tests/test_outbox.py", + ), + ), + ("tasks_checkers", ("tests/test_checkers.py", "tests/test_tasks.py")), + ), +} + + @dataclass class ActiveLane: key: str @@ -140,6 +170,7 @@ class ActiveLane: log: TextIO log_path: Path evidence_path: Path + isolation_path: Path coverage_path: Path started_at: float interrupted_at: float | None = None @@ -472,6 +503,7 @@ def admin_runner_environment( def lane_command( lane: TestLane, + resource_lane: str, nodes: list[str], metadata_dir: Path, timeout_seconds: float, @@ -483,7 +515,7 @@ def lane_command( ] return [ sys.executable, str(ISOLATED_RUNNER), "--metadata-json", - str(metadata_dir / f"{lane.name}.database.json"), "--lane", lane.name, + str(metadata_dir / f"{resource_lane}.database.json"), "--lane", resource_lane, "--tree-sha", tree_sha, "--timeout-seconds", f"{timeout_seconds:g}", "--", *pytest_command, ] @@ -538,6 +570,7 @@ def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str "execution_kind": active.execution_kind, "execution_exit_code": exit_code, "interrupted": active.interrupted_at is not None or active.timed_out, + "isolation_path": active.isolation_path, "skipped_nodes": skipped, } @@ -561,22 +594,28 @@ def _combine_coverage(sources: list[Path], destination: Path) -> None: def _finalize_lane( lane: TestLane, units: list[dict[str, Any]], metadata_dir: Path ) -> dict[str, Any]: + ordered_units = sorted( + units, + key=lambda unit: ( + str(unit["execution_kind"]), + unit["isolation_path"].name, + ), + ) evidence_path = metadata_dir / f"{lane.name}.json" - isolation_path = metadata_dir / f"{lane.name}.database.json" coverage_path = metadata_dir / f".coverage.{lane.name}" execution_exit_code = _aggregate_exit_codes( - [unit["execution_exit_code"] for unit in units] + [unit["execution_exit_code"] for unit in ordered_units] ) ordinary_coverage = [ unit["coverage_path"] - for unit in units + for unit in ordered_units if unit["execution_kind"] == ORDINARY_KIND ] if execution_exit_code == 0 and not ordinary_coverage: raise LaneError("missing_ordinary_lane_coverage") optional_admin_coverage = [ unit["coverage_path"] - for unit in units + for unit in ordered_units if unit["execution_kind"] == ADMIN_KIND and unit["coverage_path"].is_file() and not unit["coverage_path"].is_symlink() @@ -589,31 +628,49 @@ def _finalize_lane( ): _combine_coverage(selected_coverage, coverage_path) evidence = { - "collected_nodes": sorted(node for unit in units for node in unit["collected_nodes"]), - "completed_nodes": sorted(node for unit in units for node in unit["completed_nodes"]), - "deselected_nodes": sorted(set(node for unit in units for node in unit["deselected_nodes"])), - "isolation_metadata_file": isolation_path.name, - "isolation_metadata_sha256": _sha256(isolation_path.read_bytes()), - "skipped_nodes": sorted(set(node for unit in units for node in unit["skipped_nodes"])), + "collected_nodes": sorted( + node for unit in ordered_units for node in unit["collected_nodes"] + ), + "completed_nodes": sorted( + node for unit in ordered_units for node in unit["completed_nodes"] + ), + "deselected_nodes": sorted( + set(node for unit in ordered_units for node in unit["deselected_nodes"]) + ), + "isolation_metadata_files": [ + unit["isolation_path"].name + for unit in ordered_units + if unit["execution_kind"] == ORDINARY_KIND + ], + "isolation_metadata_sha256s": [ + _sha256(unit["isolation_path"].read_bytes()) + for unit in ordered_units + if unit["execution_kind"] == ORDINARY_KIND + ], + "skipped_nodes": sorted( + set(node for unit in ordered_units for node in unit["skipped_nodes"]) + ), } evidence_path.write_bytes(_json_bytes(evidence)) if coverage_path.is_file() and not coverage_path.is_symlink(): - for unit in units: + for unit in ordered_units: source = unit["coverage_path"] if source != coverage_path and source.is_file() and not source.is_symlink(): source.unlink() return { "collection_exit_code": _aggregate_exit_codes( - [unit["collection_exit_code"] for unit in units] + [unit["collection_exit_code"] for unit in ordered_units] ), "coverage_file": coverage_path.name, "coverage_sha256": _sha256(coverage_path.read_bytes()) if coverage_path.is_file() and not coverage_path.is_symlink() else None, - "elapsed_seconds": round(max(unit["elapsed_seconds"] for unit in units), 3), + "elapsed_seconds": round( + max(unit["elapsed_seconds"] for unit in ordered_units), 3 + ), "evidence_file": evidence_path.name, "evidence_sha256": _sha256(evidence_path.read_bytes()), "execution_exit_code": execution_exit_code, - "interrupted": any(unit["interrupted"] for unit in units), + "interrupted": any(unit["interrupted"] for unit in ordered_units), "name": lane.name, } @@ -643,6 +700,34 @@ def _timing_summary(lanes: list[dict[str, Any]]) -> dict[str, float]: } +def _ordinary_units( + lane: TestLane, ordinary_rows: list[dict[str, str]] +) -> list[tuple[str, list[dict[str, str]]]]: + """Partition one public lane into fixed dependency-owned execution units.""" + configured_units = SEMANTIC_UNITS.get(lane.name) + if configured_units is None: + return [(lane.name, ordinary_rows)] + configured_modules = [ + module for _suffix, modules in configured_units for module in modules + ] + observed_modules = {row["module"] for row in ordinary_rows} + if ( + len(configured_modules) != len(set(configured_modules)) + or set(configured_modules) != observed_modules + ): + raise LaneError("invalid_semantic_unit_inventory") + units = [ + ( + f"{lane.name}_{suffix}", + [row for row in ordinary_rows if row["module"] in modules], + ) + for suffix, modules in configured_units + ] + if any(not rows for _key, rows in units): + raise LaneError("empty_semantic_unit") + return units + + def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, collect_only: bool = False) -> int: """Collect canonical nodes and optionally execute all lanes concurrently.""" global INTERRUPTED @@ -667,7 +752,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, evidence_path = metadata_dir / f"{lane.name}.json" evidence_path.write_bytes(_json_bytes({ "collected_nodes": lane_nodes, "completed_nodes": [], "deselected_nodes": [], - "isolation_metadata_file": None, "isolation_metadata_sha256": None, + "isolation_metadata_files": [], "isolation_metadata_sha256s": [], "skipped_nodes": [], })) lane_rows.append({ @@ -694,14 +779,22 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, try: for lane in LANES: lane_rows = [row for row in manifest["nodes"] if row["lane"] == lane.name] - kinds = [ORDINARY_KIND] - if any(row["execution_kind"] == ADMIN_KIND for row in lane_rows): - kinds.append(ADMIN_KIND) - for kind in kinds: - unit_nodes = [row["nodeid"] for row in lane_rows if row["execution_kind"] == kind] + ordinary_rows = [ + row for row in lane_rows if row["execution_kind"] == ORDINARY_KIND + ] + units = [ + (key, ORDINARY_KIND, rows) + for key, rows in _ordinary_units(lane, ordinary_rows) + ] + admin_rows = [ + row for row in lane_rows if row["execution_kind"] == ADMIN_KIND + ] + if admin_rows: + units.append((f"{lane.name}.admin", ADMIN_KIND, admin_rows)) + for key, kind, unit_rows in units: + unit_nodes = [row["nodeid"] for row in unit_rows] if not unit_nodes: - continue - key = lane.name if kind == ORDINARY_KIND else f"{lane.name}.admin" + raise LaneError("empty_semantic_unit") for suffix in ("collected", "completed", "skipped", "deselected"): _exclusive_file(metadata_dir / f"{key}.{suffix}.jsonl") log_path = metadata_dir / f"{key}.log" @@ -713,7 +806,9 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, lane, metadata_dir, coverage_path, key, tree_sha ) else: - command = lane_command(lane, unit_nodes, metadata_dir, timeout_seconds, tree_sha) + command = lane_command( + lane, key, unit_nodes, metadata_dir, timeout_seconds, tree_sha + ) env = lane_environment(lane, metadata_dir, coverage_path, key, tree_sha) process = subprocess.Popen( command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, @@ -722,6 +817,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, active[key] = ActiveLane( key, lane, kind, tuple(unit_nodes), process, log, log_path, metadata_dir / f"{key}.json", + metadata_dir / f"{key}.database.json", coverage_path, time.monotonic(), ) while active: diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 70b2aa60..03d947ef 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -21,6 +21,12 @@ SCHEMA_VERSION = 1 LANE_COUNT = 4 +ISOLATION_LANES = { + "no_postgres": {"no_postgres"}, + "schema_contracts": {"schema_contracts"}, + "control_plane": {"control_plane_authority", "control_plane_projects"}, + "execution_plane": {"execution_plane_artifacts", "execution_plane_tasks_checkers"}, +} SHA_RE = re.compile(r"^[0-9a-f]{40}$") DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") LANE_RE = re.compile(r"^[a-z][a-z0-9_]*$") @@ -470,8 +476,8 @@ def validate_evidence( "collected_nodes", "completed_nodes", "deselected_nodes", - "isolation_metadata_file", - "isolation_metadata_sha256", + "isolation_metadata_files", + "isolation_metadata_sha256s", "skipped_nodes", }, "invalid_lane_evidence", @@ -496,8 +502,8 @@ def validate_evidence( ): raise EvidenceError("invalid_collect_mode_artifacts") if ( - evidence["isolation_metadata_file"] is not None - or evidence["isolation_metadata_sha256"] is not None + evidence["isolation_metadata_files"] != [] + or evidence["isolation_metadata_sha256s"] != [] ): raise EvidenceError("invalid_collect_mode_artifacts") else: @@ -505,64 +511,82 @@ def validate_evidence( raise EvidenceError("partial_or_duplicate_completion") _bound_bytes(metadata_dir, lane["coverage_file"], lane["coverage_sha256"]) coverage_files.append(lane["coverage_file"]) - isolation_files.append(evidence["isolation_metadata_file"]) - isolation = _object( - _json_bytes( - _bound_bytes( - metadata_dir, - evidence["isolation_metadata_file"], - evidence["isolation_metadata_sha256"], + metadata_files = evidence["isolation_metadata_files"] + metadata_digests = evidence["isolation_metadata_sha256s"] + if ( + not isinstance(metadata_files, list) + or not isinstance(metadata_digests, list) + or len(metadata_files) != len(metadata_digests) + or len(set(metadata_files)) != len(metadata_files) + or any(not isinstance(value, str) for value in metadata_files) + or any(not isinstance(value, str) for value in metadata_digests) + ): + raise EvidenceError("invalid_isolation_inventory") + observed_resource_lanes: set[str] = set() + for metadata_file, metadata_digest in zip( + metadata_files, metadata_digests, strict=True + ): + isolation_files.append(metadata_file) + isolation = _object( + _json_bytes( + _bound_bytes(metadata_dir, metadata_file, metadata_digest), + "invalid_isolation_metadata", ), + { + "alembic_head", + "cleanup_complete", + "database_name", + "database_cleanup_complete", + "database_provisioned", + "database_role", + "lane", + "minio_bucket", + "minio_cleanup_complete", + "minio_prefix", + "minio_probe_complete", + "minio_provisioned", + "schema_version", + "tree_sha", + }, "invalid_isolation_metadata", - ), - { - "alembic_head", - "cleanup_complete", + ) + namespace_fields = ( "database_name", - "database_cleanup_complete", - "database_provisioned", "database_role", - "lane", "minio_bucket", - "minio_cleanup_complete", "minio_prefix", - "minio_probe_complete", - "minio_provisioned", - "schema_version", - "tree_sha", - }, - "invalid_isolation_metadata", - ) - namespace_fields = ( - "database_name", - "database_role", - "minio_bucket", - "minio_prefix", - ) - if ( - isolation["schema_version"] != 2 - or isolation["tree_sha"] != head - or isolation["lane"] != name - or isolation["database_provisioned"] is not True - or isolation["minio_provisioned"] is not True - or isolation["database_cleanup_complete"] is not True - or isolation["minio_probe_complete"] is not True - or isolation["minio_cleanup_complete"] is not True - or isolation["cleanup_complete"] is not True - or not isinstance(isolation["alembic_head"], str) - or not isolation["alembic_head"] - or any( - not isinstance(isolation[field], str) or not isolation[field] - for field in namespace_fields ) - or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_name"]) is None - or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_role"]) is None - or BUCKET_RE.fullmatch(isolation["minio_bucket"]) is None - or PurePosixPath(isolation["minio_prefix"]).is_absolute() - or ".." in PurePosixPath(isolation["minio_prefix"]).parts - ): - raise EvidenceError("invalid_isolation_metadata") - isolation_namespaces.append(tuple(isolation[field] for field in namespace_fields)) + resource_lane = isolation["lane"] + if ( + isolation["schema_version"] != 2 + or isolation["tree_sha"] != head + or not isinstance(resource_lane, str) + or resource_lane not in ISOLATION_LANES.get(name, set()) + or isolation["database_provisioned"] is not True + or isolation["minio_provisioned"] is not True + or isolation["database_cleanup_complete"] is not True + or isolation["minio_probe_complete"] is not True + or isolation["minio_cleanup_complete"] is not True + or isolation["cleanup_complete"] is not True + or not isinstance(isolation["alembic_head"], str) + or not isolation["alembic_head"] + or any( + not isinstance(isolation[field], str) or not isolation[field] + for field in namespace_fields + ) + or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_name"]) is None + or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_role"]) is None + or BUCKET_RE.fullmatch(isolation["minio_bucket"]) is None + or PurePosixPath(isolation["minio_prefix"]).is_absolute() + or ".." in PurePosixPath(isolation["minio_prefix"]).parts + ): + raise EvidenceError("invalid_isolation_metadata") + observed_resource_lanes.add(resource_lane) + isolation_namespaces.append( + tuple(isolation[field] for field in namespace_fields) + ) + if observed_resource_lanes != ISOLATION_LANES.get(name): + raise EvidenceError("invalid_isolation_inventory") all_collected.extend(collected) all_completed.extend(completed) @@ -571,12 +595,14 @@ def validate_evidence( if mode == "run" and Counter(all_completed) != Counter(canonical_ids): raise EvidenceError("global_completion_reconciliation_failed") if mode == "run" and any( - len({namespace[index] for namespace in isolation_namespaces}) != LANE_COUNT + len({namespace[index] for namespace in isolation_namespaces}) + != len(isolation_namespaces) for index in range(4) ): raise EvidenceError("shared_isolation_namespace") if mode == "run" and ( - len(set(coverage_files)) != LANE_COUNT or len(set(isolation_files)) != LANE_COUNT + len(set(coverage_files)) != LANE_COUNT + or len(set(isolation_files)) != sum(len(value) for value in ISOLATION_LANES.values()) ): raise EvidenceError("shared_lane_artifact") aggregate = summary["aggregate_runner_seconds"] diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 92b8f485..bf45bcce 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -12,6 +12,7 @@ import uuid import pytest # type: ignore[import-not-found] +from coverage import CoverageData import scripts.run_test_lanes as runner from scripts.run_test_lanes import LANES, LaneError, TestLane as LaneDefinition @@ -198,7 +199,9 @@ def test_uuid4_and_repository_import_alias_are_restored_before_test_body( def test_lane_command_uses_exact_nodes_and_isolation_contract(tmp_path: Path) -> None: lane = LANES[0] nodes = [f"{lane.modules[0]}::test_exact[param]"] - command = runner.lane_command(lane, nodes, tmp_path, 1200, "b" * 40) + command = runner.lane_command( + lane, lane.name, nodes, tmp_path, 1200, "b" * 40 + ) assert command[1].endswith("scripts/run_isolated_tests.py") assert command[command.index("--lane") + 1] == lane.name @@ -279,6 +282,7 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_kind": runner.ORDINARY_KIND, "execution_exit_code": 0, "interrupted": False, + "isolation_path": isolation, "skipped_nodes": [], }, { @@ -291,6 +295,7 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_kind": runner.ADMIN_KIND, "execution_exit_code": 0, "interrupted": False, + "isolation_path": tmp_path / "unused-admin.database.json", "skipped_nodes": [], }, ] @@ -304,7 +309,8 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None: lane = LANES[0] - tmp_path.joinpath(f"{lane.name}.database.json").write_text("{}\n", encoding="utf-8") + isolation = tmp_path / f"{lane.name}.database.json" + isolation.write_text("{}\n", encoding="utf-8") units = [ { "collection_exit_code": 0, @@ -316,6 +322,7 @@ def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None "execution_kind": runner.ORDINARY_KIND, "execution_exit_code": 0, "interrupted": False, + "isolation_path": isolation, "skipped_nodes": [], } ] @@ -324,6 +331,26 @@ def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None runner._finalize_lane(lane, units, tmp_path) +def test_combine_coverage_merges_multiple_semantic_unit_files(tmp_path: Path) -> None: + sources = [] + for index in range(2): + source = tmp_path / f".coverage.unit.{index}" + data = CoverageData(basename=str(source)) + data.add_lines({str(tmp_path / f"module_{index}.py"): {index + 1}}) + data.write() + sources.append(source) + destination = tmp_path / ".coverage.public_lane" + + runner._combine_coverage(sources, destination) + + combined = CoverageData(basename=str(destination)) + combined.read() + assert set(combined.measured_files()) == { + str(tmp_path / "module_0.py"), + str(tmp_path / "module_1.py"), + } + + def test_collect_only_writes_raw_digest_bound_validator_schema( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -392,6 +419,38 @@ def test_timing_summary_is_derived_from_exact_four_lanes() -> None: runner._timing_summary([*lanes[:3], {"elapsed_seconds": -1.0}]) +def test_slow_lanes_use_exact_dependency_owned_execution_units() -> None: + for lane_name, expected_keys in { + "control_plane": {"control_plane_authority", "control_plane_projects"}, + "execution_plane": { + "execution_plane_artifacts", + "execution_plane_tasks_checkers", + }, + }.items(): + lane = next(item for item in LANES if item.name == lane_name) + rows = [ + {"module": module, "nodeid": f"{module}::test_one"} + for module in lane.modules + ] + units = runner._ordinary_units(lane, rows) + + assert {key for key, _rows in units} == expected_keys + assert sorted(row["nodeid"] for _key, unit_rows in units for row in unit_rows) == sorted( + row["nodeid"] for row in rows + ) + + +def test_semantic_execution_units_reject_module_drift() -> None: + lane = next(item for item in LANES if item.name == "control_plane") + rows = [ + {"module": module, "nodeid": f"{module}::test_one"} + for module in lane.modules[:-1] + ] + + with pytest.raises(LaneError, match="invalid_semantic_unit_inventory"): + runner._ordinary_units(lane, rows) + + def test_exit_aggregation_cannot_hide_signal_failure() -> None: assert runner._aggregate_exit_codes([0, 0]) == 0 assert runner._aggregate_exit_codes([0, 3]) == 1 @@ -405,7 +464,8 @@ def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path for index, lane in enumerate(LANES): source = tmp_path / f".coverage.unit.{lane.name}" source.write_bytes(f"coverage-{lane.name}".encode()) - (tmp_path / f"{lane.name}.database.json").write_text("{}\n", encoding="utf-8") + isolation = tmp_path / f"{lane.name}.database.json" + isolation.write_text("{}\n", encoding="utf-8") unit = { "collected_nodes": [f"{lane.modules[0]}::test_one"], "collection_exit_code": 0, @@ -415,6 +475,7 @@ def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path "elapsed_seconds": float(index + 1), "execution_kind": runner.ORDINARY_KIND, "execution_exit_code": 0, + "isolation_path": isolation, "interrupted": False, "skipped_nodes": [], } @@ -442,10 +503,10 @@ def test_failure_interrupts_sibling_process_groups_and_records_all_lanes( monkeypatch.setenv(runner.ADMIN_ENV, "postgresql+asyncpg://admin@localhost/postgres") monkeypatch.setattr(runner, "CLEANUP_GRACE_SECONDS", 0.2) - def fake_command(lane, _nodes, metadata, _timeout, _sha): - database = metadata / f"{lane.name}.database.json" + def fake_command(lane, resource_lane, _nodes, metadata, _timeout, _sha): + database = metadata / f"{resource_lane}.database.json" database.write_text("{}\n", encoding="utf-8") - coverage = metadata / f".coverage.{lane.name}" + coverage = metadata / f".coverage.unit.{resource_lane}" coverage.write_bytes(b"coverage") if lane.name == "lane_0": return [sys.executable, "-c", "raise SystemExit(1)"] diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index 4b9a0136..ad5bc922 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -21,6 +21,12 @@ REAL_COLLECT_CURRENT_NODES = validator._collect_current_nodes HEAD = "a" * 40 LANES = ("no_postgres", "schema_contracts", "control_plane", "execution_plane") +ISOLATION_LANES = { + "no_postgres": ("no_postgres",), + "schema_contracts": ("schema_contracts",), + "control_plane": ("control_plane_authority", "control_plane_projects"), + "execution_plane": ("execution_plane_artifacts", "execution_plane_tasks_checkers"), +} def _write(path: Path, value: object) -> str: @@ -58,31 +64,35 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: lane_rows = [] for lane in LANES: nodeids = [row["nodeid"] for row in nodes if row["lane"] == lane] - isolation_file = None - isolation_digest = None + isolation_files = [] + isolation_digests = [] coverage_file = None coverage_digest = None if mode == "run": - isolation_file = f"{lane}.isolation.json" - isolation_digest = _write( - metadata / isolation_file, - { - "alembic_head": "head", - "cleanup_complete": True, - "database_cleanup_complete": True, - "database_name": f"database_{lane}", - "database_provisioned": True, - "database_role": f"role_{lane}", - "lane": lane, - "minio_bucket": f"bucket-{lane.replace('_', '-')}", - "minio_cleanup_complete": True, - "minio_prefix": f"prefix/{lane}", - "minio_probe_complete": True, - "minio_provisioned": True, - "schema_version": 2, - "tree_sha": HEAD, - }, - ) + for resource_lane in ISOLATION_LANES[lane]: + isolation_file = f"{resource_lane}.isolation.json" + isolation_files.append(isolation_file) + isolation_digests.append( + _write( + metadata / isolation_file, + { + "alembic_head": "head", + "cleanup_complete": True, + "database_cleanup_complete": True, + "database_name": f"database_{resource_lane}", + "database_provisioned": True, + "database_role": f"role_{resource_lane}", + "lane": resource_lane, + "minio_bucket": f"bucket-{resource_lane.replace('_', '-')}", + "minio_cleanup_complete": True, + "minio_prefix": f"prefix/{resource_lane}", + "minio_probe_complete": True, + "minio_provisioned": True, + "schema_version": 2, + "tree_sha": HEAD, + }, + ) + ) coverage_file = f"coverage.{lane}" (metadata / coverage_file).write_bytes(f"coverage:{lane}".encode()) coverage_digest = hashlib.sha256((metadata / coverage_file).read_bytes()).hexdigest() @@ -93,8 +103,8 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: "collected_nodes": nodeids, "completed_nodes": nodeids if mode == "run" else [], "deselected_nodes": [], - "isolation_metadata_file": isolation_file, - "isolation_metadata_sha256": isolation_digest, + "isolation_metadata_files": isolation_files, + "isolation_metadata_sha256s": isolation_digests, "skipped_nodes": [], }, ) @@ -299,10 +309,10 @@ def test_rejects_recorded_database_environment_or_shared_coverage(tmp_path: Path metadata, summary_path, summary = _bundle(tmp_path) lane = summary["lanes"][0] evidence = json.loads((metadata / lane["evidence_file"]).read_text()) - isolation_path = metadata / evidence["isolation_metadata_file"] + isolation_path = metadata / evidence["isolation_metadata_files"][0] isolation = json.loads(isolation_path.read_text()) isolation["admin_database_url"] = "postgresql://admin:secret@example.invalid/db" - evidence["isolation_metadata_sha256"] = _write(isolation_path, isolation) + evidence["isolation_metadata_sha256s"][0] = _write(isolation_path, isolation) lane["evidence_sha256"] = _write(metadata / lane["evidence_file"], evidence) _write(summary_path, summary) with pytest.raises(validator.EvidenceError, match="invalid_isolation_metadata"): @@ -316,6 +326,34 @@ def test_rejects_recorded_database_environment_or_shared_coverage(tmp_path: Path validator.validate_evidence(metadata, summary_path, tmp_path) +def test_rejects_missing_or_foreign_semantic_isolation_unit(tmp_path: Path) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + lane = next(row for row in summary["lanes"] if row["name"] == "control_plane") + evidence_path = metadata / lane["evidence_file"] + evidence = json.loads(evidence_path.read_text()) + evidence["isolation_metadata_files"].pop() + evidence["isolation_metadata_sha256s"].pop() + lane["evidence_sha256"] = _write(evidence_path, evidence) + _write(summary_path, summary) + + with pytest.raises(validator.EvidenceError, match="invalid_isolation_inventory"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + metadata, summary_path, summary = _bundle(tmp_path / "foreign") + lane = next(row for row in summary["lanes"] if row["name"] == "control_plane") + evidence_path = metadata / lane["evidence_file"] + evidence = json.loads(evidence_path.read_text()) + isolation_path = metadata / evidence["isolation_metadata_files"][0] + isolation = json.loads(isolation_path.read_text()) + isolation["lane"] = "control_plane_foreign" + evidence["isolation_metadata_sha256s"][0] = _write(isolation_path, isolation) + lane["evidence_sha256"] = _write(evidence_path, evidence) + _write(summary_path, summary) + + with pytest.raises(validator.EvidenceError, match="invalid_isolation_metadata"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + @pytest.mark.parametrize( ("field", "value", "message"), [ diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index f82af57e..cec7f5b5 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -69,16 +69,21 @@ If provisioning fails, confirm the local PostgreSQL provisioning credential can The required GitHub check remains `Backend / test`. One job owns one digest-pinned PostgreSQL service, one digest-pinned MinIO service, and four -concurrent dependency lanes. This avoids arbitrary shard fan-out and artifact -fan-in while retaining exact node and coverage custody. +public dependency lanes. The two slow database-heavy lanes each use two fixed +semantic execution units, so six ordinary units run concurrently without +reintroducing arbitrary shards or artifact fan-in. The job binds the checkout to `GITHUB_SHA`, installs and asserts exact Ruff `0.15.22`, runs lint and docstrings, starts MinIO, then collects every canonical pytest node. The independent evidence validator must -accept the collection before execution begins. Each lane receives a distinct -runner-created database and role plus a distinct MinIO bucket/prefix custody -record. The S3 lane owns the actual `workstream-artifacts` test bucket and a -unique run prefix; other lanes create, probe, and remove distinct buckets. +accept the collection before execution begins. Each ordinary execution unit +receives a distinct runner-created database and role plus a distinct MinIO +bucket/prefix custody record. The no-PostgreSQL lane owns the actual +`workstream-artifacts` test bucket and a unique run prefix; the other five units +create, probe, and remove distinct buckets. The public `control_plane` lane is +split into authority and project units; `execution_plane` is split into +artifact/outbox and task/checker units. Their node sets and coverage are +reconciled back into their public lanes. The isolated-runner self-tests remain in the canonical manifest as the explicit `admin_runner_self_test` execution kind. The lane orchestrator runs only those nodes directly with the admin URL while stripping application database URLs; @@ -97,11 +102,12 @@ isolated invocation inside the same required job. ### Evidence bundle The workflow uploads the `.ci/test-lanes` tree even on failure. Its summary -records the exact head, canonical node count, four lane results, elapsed time, -and raw-file digests. Per-lane evidence records collected, completed, skipped, -and deselected exact node IDs plus the bound resource-isolation metadata and -coverage digest. Resource metadata is mode `0600`, omits credentials, and proves -database, role, bucket, prefix, probe, and cleanup custody. +records the exact head, canonical node count, four public lane results, elapsed +time, and raw-file digests. Per-lane evidence records collected, completed, +skipped, and deselected exact node IDs plus every bound execution-unit resource +record and the public lane coverage digest. The six resource records are mode +`0600`, omit credentials, and prove database, role, bucket, prefix, probe, and +cleanup custody. The validator accepts only safe repository-local regular files and exact schema keys. It rejects symlinks, traversal, stale heads, digest drift, unexpected diff --git a/scripts/test_agent_gates.py b/scripts/test_agent_gates.py index ab284c49..90b8ab26 100644 --- a/scripts/test_agent_gates.py +++ b/scripts/test_agent_gates.py @@ -6514,6 +6514,15 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert "run_isolated_tests.py" in lane_runner assert "admin_runner_self_test" in lane_validator assert "execution_kind" in lane_validator + assert "SEMANTIC_UNITS" in lane_runner + assert '"control_plane"' in lane_runner + assert '"execution_plane"' in lane_runner + assert '"control_plane_authority"' in lane_validator + assert '"control_plane_projects"' in lane_validator + assert '"execution_plane_artifacts"' in lane_validator + assert '"execution_plane_tasks_checkers"' in lane_validator + assert "ISOLATION_LANES" in lane_validator + assert "invalid_isolation_inventory" in lane_validator api_e2e_steps = [ step for step in steps if step.get("name") == "API contract real API e2e" From ad7270d4e149fb181e6b4a31b568f0f2b2d3382e Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 19:38:06 +0100 Subject: [PATCH 14/31] fix(ci): bind nodes to semantic resource units --- backend/scripts/run_test_lanes.py | 13 ++++ .../scripts/validate_test_lane_evidence.py | 71 +++++++++++++++++++ backend/tests/test_ci_test_lanes.py | 4 ++ backend/tests/test_test_lane_evidence.py | 61 +++++++++++++--- docs/operations_backend_testing.md | 6 ++ 5 files changed, 145 insertions(+), 10 deletions(-) diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 92de6843..e70621b0 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -571,6 +571,7 @@ def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str "execution_exit_code": exit_code, "interrupted": active.interrupted_at is not None or active.timed_out, "isolation_path": active.isolation_path, + "resource_lane": active.key, "skipped_nodes": skipped, } @@ -647,6 +648,16 @@ def _finalize_lane( for unit in ordered_units if unit["execution_kind"] == ORDINARY_KIND ], + "isolation_unit_collected_nodes": { + unit["resource_lane"]: sorted(unit["collected_nodes"]) + for unit in ordered_units + if unit["execution_kind"] == ORDINARY_KIND + }, + "isolation_unit_completed_nodes": { + unit["resource_lane"]: sorted(unit["completed_nodes"]) + for unit in ordered_units + if unit["execution_kind"] == ORDINARY_KIND + }, "skipped_nodes": sorted( set(node for unit in ordered_units for node in unit["skipped_nodes"]) ), @@ -753,6 +764,8 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, evidence_path.write_bytes(_json_bytes({ "collected_nodes": lane_nodes, "completed_nodes": [], "deselected_nodes": [], "isolation_metadata_files": [], "isolation_metadata_sha256s": [], + "isolation_unit_collected_nodes": {}, + "isolation_unit_completed_nodes": {}, "skipped_nodes": [], })) lane_rows.append({ diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 03d947ef..7c050ca7 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -27,6 +27,31 @@ "control_plane": {"control_plane_authority", "control_plane_projects"}, "execution_plane": {"execution_plane_artifacts", "execution_plane_tasks_checkers"}, } +SEMANTIC_UNIT_MODULES = { + "control_plane": { + "control_plane_authority": { + "tests/test_actors.py", + "tests/test_api_rate_controls.py", + "tests/test_audit.py", + "tests/test_auth.py", + "tests/test_authorization.py", + }, + "control_plane_projects": {"tests/test_projects.py"}, + }, + "execution_plane": { + "execution_plane_artifacts": { + "tests/test_artifact_admission.py", + "tests/test_artifact_operator_api.py", + "tests/test_artifact_recovery.py", + "tests/test_db_session.py", + "tests/test_outbox.py", + }, + "execution_plane_tasks_checkers": { + "tests/test_checkers.py", + "tests/test_tasks.py", + }, + }, +} SHA_RE = re.compile(r"^[0-9a-f]{40}$") DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") LANE_RE = re.compile(r"^[a-z][a-z0-9_]*$") @@ -478,6 +503,8 @@ def validate_evidence( "deselected_nodes", "isolation_metadata_files", "isolation_metadata_sha256s", + "isolation_unit_collected_nodes", + "isolation_unit_completed_nodes", "skipped_nodes", }, "invalid_lane_evidence", @@ -504,6 +531,8 @@ def validate_evidence( if ( evidence["isolation_metadata_files"] != [] or evidence["isolation_metadata_sha256s"] != [] + or evidence["isolation_unit_collected_nodes"] != {} + or evidence["isolation_unit_completed_nodes"] != {} ): raise EvidenceError("invalid_collect_mode_artifacts") else: @@ -513,6 +542,8 @@ def validate_evidence( coverage_files.append(lane["coverage_file"]) metadata_files = evidence["isolation_metadata_files"] metadata_digests = evidence["isolation_metadata_sha256s"] + unit_collected = evidence["isolation_unit_collected_nodes"] + unit_completed = evidence["isolation_unit_completed_nodes"] if ( not isinstance(metadata_files, list) or not isinstance(metadata_digests, list) @@ -520,8 +551,48 @@ def validate_evidence( or len(set(metadata_files)) != len(metadata_files) or any(not isinstance(value, str) for value in metadata_files) or any(not isinstance(value, str) for value in metadata_digests) + or not isinstance(unit_collected, dict) + or not isinstance(unit_completed, dict) ): raise EvidenceError("invalid_isolation_inventory") + expected_units = ISOLATION_LANES.get(name, set()) + if set(unit_collected) != expected_units or set(unit_completed) != expected_units: + raise EvidenceError("invalid_isolation_unit_node_inventory") + configured_modules = SEMANTIC_UNIT_MODULES.get(name) + expected_unit_nodes: dict[str, list[str]] = {} + for resource_lane in expected_units: + if configured_modules is None: + expected_unit_nodes[resource_lane] = sorted( + nodeid + for nodeid, _module, public_lane, kind in canonical + if public_lane == name and kind == ORDINARY_KIND + ) + else: + modules = configured_modules.get(resource_lane) + if modules is None: + raise EvidenceError("invalid_isolation_unit_node_inventory") + expected_unit_nodes[resource_lane] = sorted( + nodeid + for nodeid, module, public_lane, kind in canonical + if public_lane == name + and kind == ORDINARY_KIND + and module in modules + ) + collected_nodes = _nodes( + unit_collected[resource_lane], "isolation_unit_collected_nodes" + ) + completed_nodes = _nodes( + unit_completed[resource_lane], "isolation_unit_completed_nodes" + ) + if ( + not expected_unit_nodes[resource_lane] + or Counter(collected_nodes) + != Counter(expected_unit_nodes[resource_lane]) + or Counter(completed_nodes) != Counter(collected_nodes) + or len(collected_nodes) != len(set(collected_nodes)) + or len(completed_nodes) != len(set(completed_nodes)) + ): + raise EvidenceError("isolation_unit_node_reconciliation_failed") observed_resource_lanes: set[str] = set() for metadata_file, metadata_digest in zip( metadata_files, metadata_digests, strict=True diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index bf45bcce..42ddd48b 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -283,6 +283,7 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_exit_code": 0, "interrupted": False, "isolation_path": isolation, + "resource_lane": lane.name, "skipped_nodes": [], }, { @@ -296,6 +297,7 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_exit_code": 0, "interrupted": False, "isolation_path": tmp_path / "unused-admin.database.json", + "resource_lane": f"{lane.name}.admin", "skipped_nodes": [], }, ] @@ -323,6 +325,7 @@ def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None "execution_exit_code": 0, "interrupted": False, "isolation_path": isolation, + "resource_lane": lane.name, "skipped_nodes": [], } ] @@ -477,6 +480,7 @@ def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path "execution_exit_code": 0, "isolation_path": isolation, "interrupted": False, + "resource_lane": lane.name, "skipped_nodes": [], } rows.append(runner._finalize_lane(lane, [unit], tmp_path)) diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index ad5bc922..6088d6cb 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -27,6 +27,19 @@ "control_plane": ("control_plane_authority", "control_plane_projects"), "execution_plane": ("execution_plane_artifacts", "execution_plane_tasks_checkers"), } +RESOURCE_NODE_MODULES = { + "no_postgres": "tests/test_0.py", + "schema_contracts": "tests/test_1.py", + "control_plane_authority": "tests/test_actors.py", + "control_plane_projects": "tests/test_projects.py", + "execution_plane_artifacts": "tests/test_outbox.py", + "execution_plane_tasks_checkers": "tests/test_tasks.py", +} +PUBLIC_LANES = { + resource_lane: public_lane + for public_lane, resource_lanes in ISOLATION_LANES.items() + for resource_lane in resource_lanes +} def _write(path: Path, value: object) -> str: @@ -41,11 +54,11 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: nodes = [ { "execution_kind": "ordinary_isolated", - "lane": lane, - "module": f"tests/test_{index}.py", - "nodeid": f"tests/test_{index}.py::test_ok", + "lane": PUBLIC_LANES[resource_lane], + "module": module, + "nodeid": f"{module}::test_ok", } - for index, lane in enumerate(LANES) + for resource_lane, module in RESOURCE_NODE_MODULES.items() ] nodes.append( { @@ -105,6 +118,28 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: "deselected_nodes": [], "isolation_metadata_files": isolation_files, "isolation_metadata_sha256s": isolation_digests, + "isolation_unit_collected_nodes": { + resource_lane: [ + row["nodeid"] + for row in nodes + if row["execution_kind"] == "ordinary_isolated" + and row["module"] == RESOURCE_NODE_MODULES[resource_lane] + ] + for resource_lane in ISOLATION_LANES[lane] + } + if mode == "run" + else {}, + "isolation_unit_completed_nodes": { + resource_lane: [ + row["nodeid"] + for row in nodes + if row["execution_kind"] == "ordinary_isolated" + and row["module"] == RESOURCE_NODE_MODULES[resource_lane] + ] + for resource_lane in ISOLATION_LANES[lane] + } + if mode == "run" + else {}, "skipped_nodes": [], }, ) @@ -123,7 +158,7 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: ) summary = { "aggregate_runner_seconds": 4.0, - "canonical_node_count": 5, + "canonical_node_count": len(nodes), "elapsed_seconds": 2.0, "head_sha": HEAD, "lanes": lane_rows, @@ -146,7 +181,7 @@ def exact_head(monkeypatch: pytest.MonkeyPatch) -> None: "_collect_current_nodes", lambda _root, _head: sorted( [ - *(f"tests/test_{index}.py::test_ok" for index in range(4)), + *(f"{module}::test_ok" for module in RESOURCE_NODE_MODULES.values()), "tests/test_isolated_database_runner.py::test_admin_custody", ] ), @@ -280,11 +315,15 @@ def test_rejects_wrong_lane_count_zero_nodes_and_unknown_keys(tmp_path: Path) -> "missing_admin_runner_self_tests", ), ( - lambda rows: rows[-1].__setitem__("execution_kind", validator.ORDINARY_KIND), + lambda rows: next( + row for row in rows if row["module"] == validator.ADMIN_RUNNER_MODULE + ).__setitem__("execution_kind", validator.ORDINARY_KIND), "invalid_manifest_node", ), ( - lambda rows: rows[0].__setitem__("execution_kind", validator.ADMIN_KIND), + lambda rows: next( + row for row in rows if row["module"] != validator.ADMIN_RUNNER_MODULE + ).__setitem__("execution_kind", validator.ADMIN_KIND), "invalid_manifest_node", ), (lambda rows: rows.append(dict(rows[-1])), "noncanonical_or_duplicate"), @@ -389,8 +428,10 @@ def test_real_collection_rejects_missing_or_foreign_manifest_node( ) -> None: tests = tmp_path / "backend/tests" tests.mkdir(parents=True) - for index in range(4): - (tests / f"test_{index}.py").write_text("def test_ok():\n pass\n", encoding="utf-8") + for module in RESOURCE_NODE_MODULES.values(): + (tmp_path / "backend" / module).write_text( + "def test_ok():\n pass\n", encoding="utf-8" + ) (tests / "test_isolated_database_runner.py").write_text( "def test_admin_custody():\n pass\n", encoding="utf-8" ) diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index cec7f5b5..5b2b0027 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -67,6 +67,12 @@ If provisioning fails, confirm the local PostgreSQL provisioning credential can ## Hosted semantic-lane full-suite proof +A local full semantic-lane diagnostic requires both +`WORKSTREAM_TEST_ADMIN_DATABASE_URL` and a live loopback +`WORKSTREAM_TEST_MINIO_ENDPOINT`. The signed contract's command block assumes +those prerequisites; without either one, use collect-only validation and rely +on the hosted job for complete service and timing proof. + The required GitHub check remains `Backend / test`. One job owns one digest-pinned PostgreSQL service, one digest-pinned MinIO service, and four public dependency lanes. The two slow database-heavy lanes each use two fixed From 46616819879fe2afb772eb0515f5de6afefc4777 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 19:46:13 +0100 Subject: [PATCH 15/31] test(ci): reject shared semantic unit namespaces --- backend/tests/test_test_lane_evidence.py | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index 6088d6cb..d8793e68 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -393,6 +393,37 @@ def test_rejects_missing_or_foreign_semantic_isolation_unit(tmp_path: Path) -> N validator.validate_evidence(metadata, summary_path, tmp_path) +@pytest.mark.parametrize( + "field", ["database_name", "database_role", "minio_bucket", "minio_prefix"] +) +def test_rejects_shared_semantic_unit_namespace(tmp_path: Path, field: str) -> None: + metadata, summary_path, summary = _bundle(tmp_path) + first_lane, second_lane = summary["lanes"][:2] + first_evidence = json.loads( + (metadata / first_lane["evidence_file"]).read_text(encoding="utf-8") + ) + second_evidence_path = metadata / second_lane["evidence_file"] + second_evidence = json.loads(second_evidence_path.read_text(encoding="utf-8")) + first_isolation = json.loads( + (metadata / first_evidence["isolation_metadata_files"][0]).read_text( + encoding="utf-8" + ) + ) + second_isolation_path = metadata / second_evidence["isolation_metadata_files"][0] + second_isolation = json.loads(second_isolation_path.read_text(encoding="utf-8")) + second_isolation[field] = first_isolation[field] + second_evidence["isolation_metadata_sha256s"][0] = _write( + second_isolation_path, second_isolation + ) + second_lane["evidence_sha256"] = _write( + second_evidence_path, second_evidence + ) + _write(summary_path, summary) + + with pytest.raises(validator.EvidenceError, match="shared_isolation_namespace"): + validator.validate_evidence(metadata, summary_path, tmp_path) + + @pytest.mark.parametrize( ("field", "value", "message"), [ From 8bc7e9f0925f9d9e86b669957561efdc68f27d50 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 19:51:06 +0100 Subject: [PATCH 16/31] docs(agent-loop): record semantic unit review evidence --- .../WS-CI-001-02B-internal-review-evidence.md | 30 +++++++++++++++---- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 27 +++++++++++------ 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index 59b3b5b5..e8f45bc1 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,16 +12,17 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: 239adb178229c72951862780e5ce237f73113400 +Reviewed code SHA: 46616819879fe2afb772eb0515f5de6afefc4777 -Reviewed at: 2026-07-24T16:55:21Z +Reviewed at: 2026-07-24T18:47:56Z Reviewer run IDs: ci02b_senior_review, ci02b_qa_review, ci02b_security_review, ci02b_product_ops_review, ci02b_restart_arch_review, ci02b_restart_ci_review, ci02b_contract_gap, ci02b_source_audit, ci02b_test_delta_review, ci02b_bootstrap_senior, ci02b_bootstrap_qa, ci02b_bootstrap_security, ci02b_bootstrap_ops, -ci02b_bootstrap_arch, ci02b_bootstrap_ci +ci02b_bootstrap_arch, ci02b_bootstrap_ci, ci02b_units_docs, +ci02b_units_reuse, ci02b_units_test_delta After the reviewed SHA, only evidence and status files changed. @@ -52,6 +53,15 @@ instead of allowing missing coverage to mask the original failure. Optional admin coverage absence remains a documented low diagnostic risk, not a product coverage or exact-node custody bypass. +The performance repair retains four public dependency lanes and four public +coverage files while splitting the two slow database-heavy lanes into fixed +semantic units. Six ordinary units each own a private database, role, MinIO +bucket/prefix, digest-bound resource record, and exact collected/completed node +set. The validator independently duplicates the expected unit/module map and +rejects missing, foreign, partial, duplicate, or shared custody. Reviewer +findings for per-unit node binding, local MinIO prerequisites, and direct +database/role/bucket/prefix collision tests are resolved. + ## Valid Findings Addressed - Replaced invalid underscore-bearing MinIO bucket names with S3-valid, @@ -82,6 +92,13 @@ coverage or exact-node custody bypass. proved that its ordinary unit emitted coverage while the direct admin self-test unit legitimately emitted none. Missing ordinary coverage still fails closed, and regression tests cover both paths. +- Recorded the exact 533.218-second result from hosted run `30111028221`, then + added bounded semantic execution units for `control_plane` and + `execution_plane` without changing the four public lanes or the 480-second + gate. +- Bound all six resource units to independently reconstructed exact node sets + and added direct collision tests for database, role, bucket, and prefix + namespaces. ## Commands Run @@ -101,10 +118,11 @@ git diff --check origin/main...HEAD ## Results - Ruff passed with exact local `ruff 0.15.22`. -- 62 focused lane-runner and independent-validator tests passed. +- 70 focused lane-runner and independent-validator tests passed. - 100 Agent Gate tests passed. -- Two independent full collections agreed on 2,046 exact pytest nodes at the - reviewed head; independent evidence validation passed. +- Prior hosted head `80dd8c87` completed and reconciled 2,048 exact nodes before + failing only the 480-second timing gate. Exact final-head hosted collection + and execution remain required. - Merge intent, Markdown links, stale wording, and diff integrity passed. - Local full service execution was not used as hosted performance evidence. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 0a21f34b..3d0e4f3b 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -24,7 +24,7 @@ hosted timing outcome on the exact PR head. - Phase: `implementation` - Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` - Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` -- Reviewed implementation SHA: `239adb178229c72951862780e5ce237f73113400` +- Reviewed implementation SHA: `46616819879fe2afb772eb0515f5de6afefc4777` Only independently verified signed automation state is canonical authority. PR prose and checked boxes are navigation evidence, not authorization. @@ -45,6 +45,11 @@ PR prose and checked boxes are navigation evidence, not authorization. - Preserved schema-lane ordinary coverage when its direct admin self-test unit legitimately emits no `app` coverage; missing ordinary coverage remains a hard failure. +- Split the slow control and execution planes into four fixed semantic units. + Together with the two unsplit ordinary lanes, six isolated resource units + reconcile back into four public lanes and four final coverage files. +- Bound every resource unit to its exact collected/completed node set and made + the independent validator reject unit/module drift and shared namespaces. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed @@ -101,11 +106,11 @@ resources, coverage bytes, failures, and timings before combination. ## Acceptance criteria proof -- [x] Recursive exact-node inventory with no exclusion escape — 2,046 nodes - independently recollected and validated at the reviewed head. -- [x] Four concurrent dependency lanes with distinct database/role and MinIO - bucket/prefix custody — implementation and adversarial tests pass; hosted - service proof remains required. +- [ ] Recursive exact-node inventory with no exclusion escape — prior hosted + head reconciled 2,048 nodes; exact final-head hosted proof is pending. +- [ ] Four public dependency lanes with six distinct database/role and MinIO + execution-unit custody records — implementation and adversarial tests pass; + exact final-head hosted service proof remains required. - [x] Missing, duplicate, foreign, skipped, deselected, interrupted, partial, digest-drifted, and shared-custody evidence fails in focused tests. - [x] Exactly four authenticated coverage artifacts precede one literal @@ -131,7 +136,7 @@ python3 scripts/check_markdown_links.py docs/operations_backend_testing.md .agen git diff --check origin/main...HEAD ``` -Result summary: Ruff passed; 62 focused tests passed; 100 Agent Gates passed; +Result summary: Ruff passed; 70 focused tests passed; 100 Agent Gates passed; two independent full collections agreed on 2,046 exact nodes; merge intent, links, stale wording, and diff integrity passed. @@ -175,9 +180,9 @@ External review response file, if findings are posted: ## Reviewer results -Reviewed code SHA: `239adb178229c72951862780e5ce237f73113400` +Reviewed code SHA: `46616819879fe2afb772eb0515f5de6afefc4777` -Reviewed at: `2026-07-24T16:55:21Z` +Reviewed at: `2026-07-24T18:47:56Z` Reviewer run IDs: `ci02b_senior_review`, `ci02b_qa_review`, `ci02b_security_review`, `ci02b_product_ops_review`, @@ -188,6 +193,10 @@ Bootstrap repair reviewer run IDs: `ci02b_bootstrap_senior`, `ci02b_bootstrap_qa`, `ci02b_bootstrap_security`, `ci02b_bootstrap_ops`, `ci02b_bootstrap_arch`, `ci02b_bootstrap_ci`. +Semantic-unit reviewer run IDs: `ci02b_units_docs`, `ci02b_units_reuse`, +`ci02b_units_test_delta`; the required senior, QA, security, product/ops, +architecture, and CI tracks reran through their `ci02b_bootstrap_*` sessions. + | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| | senior engineering | PASS WITH LOW RISKS | None | Independent collector specifications must remain aligned. | From 31516a3f07c7dacafb3fa5aba289103c6b5ad645 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 20:05:48 +0100 Subject: [PATCH 17/31] Revert "docs(agent-loop): record semantic unit review evidence" This reverts commit 8bc7e9f0925f9d9e86b669957561efdc68f27d50. --- .../WS-CI-001-02B-internal-review-evidence.md | 30 ++++--------------- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 27 ++++++----------- 2 files changed, 15 insertions(+), 42 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index e8f45bc1..59b3b5b5 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,17 +12,16 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: 46616819879fe2afb772eb0515f5de6afefc4777 +Reviewed code SHA: 239adb178229c72951862780e5ce237f73113400 -Reviewed at: 2026-07-24T18:47:56Z +Reviewed at: 2026-07-24T16:55:21Z Reviewer run IDs: ci02b_senior_review, ci02b_qa_review, ci02b_security_review, ci02b_product_ops_review, ci02b_restart_arch_review, ci02b_restart_ci_review, ci02b_contract_gap, ci02b_source_audit, ci02b_test_delta_review, ci02b_bootstrap_senior, ci02b_bootstrap_qa, ci02b_bootstrap_security, ci02b_bootstrap_ops, -ci02b_bootstrap_arch, ci02b_bootstrap_ci, ci02b_units_docs, -ci02b_units_reuse, ci02b_units_test_delta +ci02b_bootstrap_arch, ci02b_bootstrap_ci After the reviewed SHA, only evidence and status files changed. @@ -53,15 +52,6 @@ instead of allowing missing coverage to mask the original failure. Optional admin coverage absence remains a documented low diagnostic risk, not a product coverage or exact-node custody bypass. -The performance repair retains four public dependency lanes and four public -coverage files while splitting the two slow database-heavy lanes into fixed -semantic units. Six ordinary units each own a private database, role, MinIO -bucket/prefix, digest-bound resource record, and exact collected/completed node -set. The validator independently duplicates the expected unit/module map and -rejects missing, foreign, partial, duplicate, or shared custody. Reviewer -findings for per-unit node binding, local MinIO prerequisites, and direct -database/role/bucket/prefix collision tests are resolved. - ## Valid Findings Addressed - Replaced invalid underscore-bearing MinIO bucket names with S3-valid, @@ -92,13 +82,6 @@ database/role/bucket/prefix collision tests are resolved. proved that its ordinary unit emitted coverage while the direct admin self-test unit legitimately emitted none. Missing ordinary coverage still fails closed, and regression tests cover both paths. -- Recorded the exact 533.218-second result from hosted run `30111028221`, then - added bounded semantic execution units for `control_plane` and - `execution_plane` without changing the four public lanes or the 480-second - gate. -- Bound all six resource units to independently reconstructed exact node sets - and added direct collision tests for database, role, bucket, and prefix - namespaces. ## Commands Run @@ -118,11 +101,10 @@ git diff --check origin/main...HEAD ## Results - Ruff passed with exact local `ruff 0.15.22`. -- 70 focused lane-runner and independent-validator tests passed. +- 62 focused lane-runner and independent-validator tests passed. - 100 Agent Gate tests passed. -- Prior hosted head `80dd8c87` completed and reconciled 2,048 exact nodes before - failing only the 480-second timing gate. Exact final-head hosted collection - and execution remain required. +- Two independent full collections agreed on 2,046 exact pytest nodes at the + reviewed head; independent evidence validation passed. - Merge intent, Markdown links, stale wording, and diff integrity passed. - Local full service execution was not used as hosted performance evidence. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 3d0e4f3b..0a21f34b 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -24,7 +24,7 @@ hosted timing outcome on the exact PR head. - Phase: `implementation` - Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` - Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` -- Reviewed implementation SHA: `46616819879fe2afb772eb0515f5de6afefc4777` +- Reviewed implementation SHA: `239adb178229c72951862780e5ce237f73113400` Only independently verified signed automation state is canonical authority. PR prose and checked boxes are navigation evidence, not authorization. @@ -45,11 +45,6 @@ PR prose and checked boxes are navigation evidence, not authorization. - Preserved schema-lane ordinary coverage when its direct admin self-test unit legitimately emits no `app` coverage; missing ordinary coverage remains a hard failure. -- Split the slow control and execution planes into four fixed semantic units. - Together with the two unsplit ordinary lanes, six isolated resource units - reconcile back into four public lanes and four final coverage files. -- Bound every resource unit to its exact collected/completed node set and made - the independent validator reject unit/module drift and shared namespaces. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed @@ -106,11 +101,11 @@ resources, coverage bytes, failures, and timings before combination. ## Acceptance criteria proof -- [ ] Recursive exact-node inventory with no exclusion escape — prior hosted - head reconciled 2,048 nodes; exact final-head hosted proof is pending. -- [ ] Four public dependency lanes with six distinct database/role and MinIO - execution-unit custody records — implementation and adversarial tests pass; - exact final-head hosted service proof remains required. +- [x] Recursive exact-node inventory with no exclusion escape — 2,046 nodes + independently recollected and validated at the reviewed head. +- [x] Four concurrent dependency lanes with distinct database/role and MinIO + bucket/prefix custody — implementation and adversarial tests pass; hosted + service proof remains required. - [x] Missing, duplicate, foreign, skipped, deselected, interrupted, partial, digest-drifted, and shared-custody evidence fails in focused tests. - [x] Exactly four authenticated coverage artifacts precede one literal @@ -136,7 +131,7 @@ python3 scripts/check_markdown_links.py docs/operations_backend_testing.md .agen git diff --check origin/main...HEAD ``` -Result summary: Ruff passed; 70 focused tests passed; 100 Agent Gates passed; +Result summary: Ruff passed; 62 focused tests passed; 100 Agent Gates passed; two independent full collections agreed on 2,046 exact nodes; merge intent, links, stale wording, and diff integrity passed. @@ -180,9 +175,9 @@ External review response file, if findings are posted: ## Reviewer results -Reviewed code SHA: `46616819879fe2afb772eb0515f5de6afefc4777` +Reviewed code SHA: `239adb178229c72951862780e5ce237f73113400` -Reviewed at: `2026-07-24T18:47:56Z` +Reviewed at: `2026-07-24T16:55:21Z` Reviewer run IDs: `ci02b_senior_review`, `ci02b_qa_review`, `ci02b_security_review`, `ci02b_product_ops_review`, @@ -193,10 +188,6 @@ Bootstrap repair reviewer run IDs: `ci02b_bootstrap_senior`, `ci02b_bootstrap_qa`, `ci02b_bootstrap_security`, `ci02b_bootstrap_ops`, `ci02b_bootstrap_arch`, `ci02b_bootstrap_ci`. -Semantic-unit reviewer run IDs: `ci02b_units_docs`, `ci02b_units_reuse`, -`ci02b_units_test_delta`; the required senior, QA, security, product/ops, -architecture, and CI tracks reran through their `ci02b_bootstrap_*` sessions. - | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| | senior engineering | PASS WITH LOW RISKS | None | Independent collector specifications must remain aligned. | From f134920db8af33f63dc853559768c9cafcff9195 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 20:05:48 +0100 Subject: [PATCH 18/31] Revert "test(ci): reject shared semantic unit namespaces" This reverts commit 46616819879fe2afb772eb0515f5de6afefc4777. --- backend/tests/test_test_lane_evidence.py | 31 ------------------------ 1 file changed, 31 deletions(-) diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index d8793e68..6088d6cb 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -393,37 +393,6 @@ def test_rejects_missing_or_foreign_semantic_isolation_unit(tmp_path: Path) -> N validator.validate_evidence(metadata, summary_path, tmp_path) -@pytest.mark.parametrize( - "field", ["database_name", "database_role", "minio_bucket", "minio_prefix"] -) -def test_rejects_shared_semantic_unit_namespace(tmp_path: Path, field: str) -> None: - metadata, summary_path, summary = _bundle(tmp_path) - first_lane, second_lane = summary["lanes"][:2] - first_evidence = json.loads( - (metadata / first_lane["evidence_file"]).read_text(encoding="utf-8") - ) - second_evidence_path = metadata / second_lane["evidence_file"] - second_evidence = json.loads(second_evidence_path.read_text(encoding="utf-8")) - first_isolation = json.loads( - (metadata / first_evidence["isolation_metadata_files"][0]).read_text( - encoding="utf-8" - ) - ) - second_isolation_path = metadata / second_evidence["isolation_metadata_files"][0] - second_isolation = json.loads(second_isolation_path.read_text(encoding="utf-8")) - second_isolation[field] = first_isolation[field] - second_evidence["isolation_metadata_sha256s"][0] = _write( - second_isolation_path, second_isolation - ) - second_lane["evidence_sha256"] = _write( - second_evidence_path, second_evidence - ) - _write(summary_path, summary) - - with pytest.raises(validator.EvidenceError, match="shared_isolation_namespace"): - validator.validate_evidence(metadata, summary_path, tmp_path) - - @pytest.mark.parametrize( ("field", "value", "message"), [ From eeab0fbb3af0544d17eceb7d0e01c00b5ac08975 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 20:05:48 +0100 Subject: [PATCH 19/31] Revert "fix(ci): bind nodes to semantic resource units" This reverts commit ad7270d4e149fb181e6b4a31b568f0f2b2d3382e. --- backend/scripts/run_test_lanes.py | 13 ---- .../scripts/validate_test_lane_evidence.py | 71 ------------------- backend/tests/test_ci_test_lanes.py | 4 -- backend/tests/test_test_lane_evidence.py | 61 +++------------- docs/operations_backend_testing.md | 6 -- 5 files changed, 10 insertions(+), 145 deletions(-) diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index e70621b0..92de6843 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -571,7 +571,6 @@ def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str "execution_exit_code": exit_code, "interrupted": active.interrupted_at is not None or active.timed_out, "isolation_path": active.isolation_path, - "resource_lane": active.key, "skipped_nodes": skipped, } @@ -648,16 +647,6 @@ def _finalize_lane( for unit in ordered_units if unit["execution_kind"] == ORDINARY_KIND ], - "isolation_unit_collected_nodes": { - unit["resource_lane"]: sorted(unit["collected_nodes"]) - for unit in ordered_units - if unit["execution_kind"] == ORDINARY_KIND - }, - "isolation_unit_completed_nodes": { - unit["resource_lane"]: sorted(unit["completed_nodes"]) - for unit in ordered_units - if unit["execution_kind"] == ORDINARY_KIND - }, "skipped_nodes": sorted( set(node for unit in ordered_units for node in unit["skipped_nodes"]) ), @@ -764,8 +753,6 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, evidence_path.write_bytes(_json_bytes({ "collected_nodes": lane_nodes, "completed_nodes": [], "deselected_nodes": [], "isolation_metadata_files": [], "isolation_metadata_sha256s": [], - "isolation_unit_collected_nodes": {}, - "isolation_unit_completed_nodes": {}, "skipped_nodes": [], })) lane_rows.append({ diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 7c050ca7..03d947ef 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -27,31 +27,6 @@ "control_plane": {"control_plane_authority", "control_plane_projects"}, "execution_plane": {"execution_plane_artifacts", "execution_plane_tasks_checkers"}, } -SEMANTIC_UNIT_MODULES = { - "control_plane": { - "control_plane_authority": { - "tests/test_actors.py", - "tests/test_api_rate_controls.py", - "tests/test_audit.py", - "tests/test_auth.py", - "tests/test_authorization.py", - }, - "control_plane_projects": {"tests/test_projects.py"}, - }, - "execution_plane": { - "execution_plane_artifacts": { - "tests/test_artifact_admission.py", - "tests/test_artifact_operator_api.py", - "tests/test_artifact_recovery.py", - "tests/test_db_session.py", - "tests/test_outbox.py", - }, - "execution_plane_tasks_checkers": { - "tests/test_checkers.py", - "tests/test_tasks.py", - }, - }, -} SHA_RE = re.compile(r"^[0-9a-f]{40}$") DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") LANE_RE = re.compile(r"^[a-z][a-z0-9_]*$") @@ -503,8 +478,6 @@ def validate_evidence( "deselected_nodes", "isolation_metadata_files", "isolation_metadata_sha256s", - "isolation_unit_collected_nodes", - "isolation_unit_completed_nodes", "skipped_nodes", }, "invalid_lane_evidence", @@ -531,8 +504,6 @@ def validate_evidence( if ( evidence["isolation_metadata_files"] != [] or evidence["isolation_metadata_sha256s"] != [] - or evidence["isolation_unit_collected_nodes"] != {} - or evidence["isolation_unit_completed_nodes"] != {} ): raise EvidenceError("invalid_collect_mode_artifacts") else: @@ -542,8 +513,6 @@ def validate_evidence( coverage_files.append(lane["coverage_file"]) metadata_files = evidence["isolation_metadata_files"] metadata_digests = evidence["isolation_metadata_sha256s"] - unit_collected = evidence["isolation_unit_collected_nodes"] - unit_completed = evidence["isolation_unit_completed_nodes"] if ( not isinstance(metadata_files, list) or not isinstance(metadata_digests, list) @@ -551,48 +520,8 @@ def validate_evidence( or len(set(metadata_files)) != len(metadata_files) or any(not isinstance(value, str) for value in metadata_files) or any(not isinstance(value, str) for value in metadata_digests) - or not isinstance(unit_collected, dict) - or not isinstance(unit_completed, dict) ): raise EvidenceError("invalid_isolation_inventory") - expected_units = ISOLATION_LANES.get(name, set()) - if set(unit_collected) != expected_units or set(unit_completed) != expected_units: - raise EvidenceError("invalid_isolation_unit_node_inventory") - configured_modules = SEMANTIC_UNIT_MODULES.get(name) - expected_unit_nodes: dict[str, list[str]] = {} - for resource_lane in expected_units: - if configured_modules is None: - expected_unit_nodes[resource_lane] = sorted( - nodeid - for nodeid, _module, public_lane, kind in canonical - if public_lane == name and kind == ORDINARY_KIND - ) - else: - modules = configured_modules.get(resource_lane) - if modules is None: - raise EvidenceError("invalid_isolation_unit_node_inventory") - expected_unit_nodes[resource_lane] = sorted( - nodeid - for nodeid, module, public_lane, kind in canonical - if public_lane == name - and kind == ORDINARY_KIND - and module in modules - ) - collected_nodes = _nodes( - unit_collected[resource_lane], "isolation_unit_collected_nodes" - ) - completed_nodes = _nodes( - unit_completed[resource_lane], "isolation_unit_completed_nodes" - ) - if ( - not expected_unit_nodes[resource_lane] - or Counter(collected_nodes) - != Counter(expected_unit_nodes[resource_lane]) - or Counter(completed_nodes) != Counter(collected_nodes) - or len(collected_nodes) != len(set(collected_nodes)) - or len(completed_nodes) != len(set(completed_nodes)) - ): - raise EvidenceError("isolation_unit_node_reconciliation_failed") observed_resource_lanes: set[str] = set() for metadata_file, metadata_digest in zip( metadata_files, metadata_digests, strict=True diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 42ddd48b..bf45bcce 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -283,7 +283,6 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_exit_code": 0, "interrupted": False, "isolation_path": isolation, - "resource_lane": lane.name, "skipped_nodes": [], }, { @@ -297,7 +296,6 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_exit_code": 0, "interrupted": False, "isolation_path": tmp_path / "unused-admin.database.json", - "resource_lane": f"{lane.name}.admin", "skipped_nodes": [], }, ] @@ -325,7 +323,6 @@ def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None "execution_exit_code": 0, "interrupted": False, "isolation_path": isolation, - "resource_lane": lane.name, "skipped_nodes": [], } ] @@ -480,7 +477,6 @@ def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path "execution_exit_code": 0, "isolation_path": isolation, "interrupted": False, - "resource_lane": lane.name, "skipped_nodes": [], } rows.append(runner._finalize_lane(lane, [unit], tmp_path)) diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index 6088d6cb..ad5bc922 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -27,19 +27,6 @@ "control_plane": ("control_plane_authority", "control_plane_projects"), "execution_plane": ("execution_plane_artifacts", "execution_plane_tasks_checkers"), } -RESOURCE_NODE_MODULES = { - "no_postgres": "tests/test_0.py", - "schema_contracts": "tests/test_1.py", - "control_plane_authority": "tests/test_actors.py", - "control_plane_projects": "tests/test_projects.py", - "execution_plane_artifacts": "tests/test_outbox.py", - "execution_plane_tasks_checkers": "tests/test_tasks.py", -} -PUBLIC_LANES = { - resource_lane: public_lane - for public_lane, resource_lanes in ISOLATION_LANES.items() - for resource_lane in resource_lanes -} def _write(path: Path, value: object) -> str: @@ -54,11 +41,11 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: nodes = [ { "execution_kind": "ordinary_isolated", - "lane": PUBLIC_LANES[resource_lane], - "module": module, - "nodeid": f"{module}::test_ok", + "lane": lane, + "module": f"tests/test_{index}.py", + "nodeid": f"tests/test_{index}.py::test_ok", } - for resource_lane, module in RESOURCE_NODE_MODULES.items() + for index, lane in enumerate(LANES) ] nodes.append( { @@ -118,28 +105,6 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: "deselected_nodes": [], "isolation_metadata_files": isolation_files, "isolation_metadata_sha256s": isolation_digests, - "isolation_unit_collected_nodes": { - resource_lane: [ - row["nodeid"] - for row in nodes - if row["execution_kind"] == "ordinary_isolated" - and row["module"] == RESOURCE_NODE_MODULES[resource_lane] - ] - for resource_lane in ISOLATION_LANES[lane] - } - if mode == "run" - else {}, - "isolation_unit_completed_nodes": { - resource_lane: [ - row["nodeid"] - for row in nodes - if row["execution_kind"] == "ordinary_isolated" - and row["module"] == RESOURCE_NODE_MODULES[resource_lane] - ] - for resource_lane in ISOLATION_LANES[lane] - } - if mode == "run" - else {}, "skipped_nodes": [], }, ) @@ -158,7 +123,7 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: ) summary = { "aggregate_runner_seconds": 4.0, - "canonical_node_count": len(nodes), + "canonical_node_count": 5, "elapsed_seconds": 2.0, "head_sha": HEAD, "lanes": lane_rows, @@ -181,7 +146,7 @@ def exact_head(monkeypatch: pytest.MonkeyPatch) -> None: "_collect_current_nodes", lambda _root, _head: sorted( [ - *(f"{module}::test_ok" for module in RESOURCE_NODE_MODULES.values()), + *(f"tests/test_{index}.py::test_ok" for index in range(4)), "tests/test_isolated_database_runner.py::test_admin_custody", ] ), @@ -315,15 +280,11 @@ def test_rejects_wrong_lane_count_zero_nodes_and_unknown_keys(tmp_path: Path) -> "missing_admin_runner_self_tests", ), ( - lambda rows: next( - row for row in rows if row["module"] == validator.ADMIN_RUNNER_MODULE - ).__setitem__("execution_kind", validator.ORDINARY_KIND), + lambda rows: rows[-1].__setitem__("execution_kind", validator.ORDINARY_KIND), "invalid_manifest_node", ), ( - lambda rows: next( - row for row in rows if row["module"] != validator.ADMIN_RUNNER_MODULE - ).__setitem__("execution_kind", validator.ADMIN_KIND), + lambda rows: rows[0].__setitem__("execution_kind", validator.ADMIN_KIND), "invalid_manifest_node", ), (lambda rows: rows.append(dict(rows[-1])), "noncanonical_or_duplicate"), @@ -428,10 +389,8 @@ def test_real_collection_rejects_missing_or_foreign_manifest_node( ) -> None: tests = tmp_path / "backend/tests" tests.mkdir(parents=True) - for module in RESOURCE_NODE_MODULES.values(): - (tmp_path / "backend" / module).write_text( - "def test_ok():\n pass\n", encoding="utf-8" - ) + for index in range(4): + (tests / f"test_{index}.py").write_text("def test_ok():\n pass\n", encoding="utf-8") (tests / "test_isolated_database_runner.py").write_text( "def test_admin_custody():\n pass\n", encoding="utf-8" ) diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index 5b2b0027..cec7f5b5 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -67,12 +67,6 @@ If provisioning fails, confirm the local PostgreSQL provisioning credential can ## Hosted semantic-lane full-suite proof -A local full semantic-lane diagnostic requires both -`WORKSTREAM_TEST_ADMIN_DATABASE_URL` and a live loopback -`WORKSTREAM_TEST_MINIO_ENDPOINT`. The signed contract's command block assumes -those prerequisites; without either one, use collect-only validation and rely -on the hosted job for complete service and timing proof. - The required GitHub check remains `Backend / test`. One job owns one digest-pinned PostgreSQL service, one digest-pinned MinIO service, and four public dependency lanes. The two slow database-heavy lanes each use two fixed From f9ddc55219786876652b43bebe5c824eed46bbb2 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 20:05:48 +0100 Subject: [PATCH 20/31] Revert "perf(ci): add exact-custody semantic execution units" This reverts commit 4afb6fbae89613030b24af3004d770cddab9bafe. --- .../STATUS.md | 6 +- backend/scripts/run_test_lanes.py | 144 +++--------------- .../scripts/validate_test_lane_evidence.py | 140 +++++++---------- backend/tests/test_ci_test_lanes.py | 73 +-------- backend/tests/test_test_lane_evidence.py | 90 ++++------- docs/operations_backend_testing.md | 28 ++-- scripts/test_agent_gates.py | 9 -- 7 files changed, 126 insertions(+), 364 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md index 7580de22..bff75000 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md @@ -7,10 +7,8 @@ - Later proposed chunk: `WS-CI-001-03` planning only; not a declared successor - Human direction: preserve Konan's authorship and measured work from PR #180, but adopt it only under prospective zero-trust scope and evidence -- Current gate: exact head `80dd8c87` proved all 2,048 nodes, coverage floors, - API E2E, and four-lane custody but recorded 533.218 seconds, exceeding the - blocking 480-second ceiling. The active repair introduces fixed semantic - execution units with six isolated resource records; hosted timing, refreshed +- Current gate: exact-custody semantic-lane implementation is in deterministic + proof; hosted full Backend execution, exact node/coverage/resource evidence, internal review, external review, and user-owned merge decision remain - Deferred option: routing/cache/timing reassessment is future `WS-CI-001-03`, with no start or successor declaration diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 92de6843..903c0788 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -130,36 +130,6 @@ class TestLane: ), ), ) -SEMANTIC_UNITS: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = { - "control_plane": ( - ( - "authority", - ( - "tests/test_actors.py", - "tests/test_api_rate_controls.py", - "tests/test_audit.py", - "tests/test_auth.py", - "tests/test_authorization.py", - ), - ), - ("projects", ("tests/test_projects.py",)), - ), - "execution_plane": ( - ( - "artifacts", - ( - "tests/test_artifact_admission.py", - "tests/test_artifact_operator_api.py", - "tests/test_artifact_recovery.py", - "tests/test_db_session.py", - "tests/test_outbox.py", - ), - ), - ("tasks_checkers", ("tests/test_checkers.py", "tests/test_tasks.py")), - ), -} - - @dataclass class ActiveLane: key: str @@ -170,7 +140,6 @@ class ActiveLane: log: TextIO log_path: Path evidence_path: Path - isolation_path: Path coverage_path: Path started_at: float interrupted_at: float | None = None @@ -503,7 +472,6 @@ def admin_runner_environment( def lane_command( lane: TestLane, - resource_lane: str, nodes: list[str], metadata_dir: Path, timeout_seconds: float, @@ -515,7 +483,7 @@ def lane_command( ] return [ sys.executable, str(ISOLATED_RUNNER), "--metadata-json", - str(metadata_dir / f"{resource_lane}.database.json"), "--lane", resource_lane, + str(metadata_dir / f"{lane.name}.database.json"), "--lane", lane.name, "--tree-sha", tree_sha, "--timeout-seconds", f"{timeout_seconds:g}", "--", *pytest_command, ] @@ -570,7 +538,6 @@ def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str "execution_kind": active.execution_kind, "execution_exit_code": exit_code, "interrupted": active.interrupted_at is not None or active.timed_out, - "isolation_path": active.isolation_path, "skipped_nodes": skipped, } @@ -594,28 +561,22 @@ def _combine_coverage(sources: list[Path], destination: Path) -> None: def _finalize_lane( lane: TestLane, units: list[dict[str, Any]], metadata_dir: Path ) -> dict[str, Any]: - ordered_units = sorted( - units, - key=lambda unit: ( - str(unit["execution_kind"]), - unit["isolation_path"].name, - ), - ) evidence_path = metadata_dir / f"{lane.name}.json" + isolation_path = metadata_dir / f"{lane.name}.database.json" coverage_path = metadata_dir / f".coverage.{lane.name}" execution_exit_code = _aggregate_exit_codes( - [unit["execution_exit_code"] for unit in ordered_units] + [unit["execution_exit_code"] for unit in units] ) ordinary_coverage = [ unit["coverage_path"] - for unit in ordered_units + for unit in units if unit["execution_kind"] == ORDINARY_KIND ] if execution_exit_code == 0 and not ordinary_coverage: raise LaneError("missing_ordinary_lane_coverage") optional_admin_coverage = [ unit["coverage_path"] - for unit in ordered_units + for unit in units if unit["execution_kind"] == ADMIN_KIND and unit["coverage_path"].is_file() and not unit["coverage_path"].is_symlink() @@ -628,49 +589,31 @@ def _finalize_lane( ): _combine_coverage(selected_coverage, coverage_path) evidence = { - "collected_nodes": sorted( - node for unit in ordered_units for node in unit["collected_nodes"] - ), - "completed_nodes": sorted( - node for unit in ordered_units for node in unit["completed_nodes"] - ), - "deselected_nodes": sorted( - set(node for unit in ordered_units for node in unit["deselected_nodes"]) - ), - "isolation_metadata_files": [ - unit["isolation_path"].name - for unit in ordered_units - if unit["execution_kind"] == ORDINARY_KIND - ], - "isolation_metadata_sha256s": [ - _sha256(unit["isolation_path"].read_bytes()) - for unit in ordered_units - if unit["execution_kind"] == ORDINARY_KIND - ], - "skipped_nodes": sorted( - set(node for unit in ordered_units for node in unit["skipped_nodes"]) - ), + "collected_nodes": sorted(node for unit in units for node in unit["collected_nodes"]), + "completed_nodes": sorted(node for unit in units for node in unit["completed_nodes"]), + "deselected_nodes": sorted(set(node for unit in units for node in unit["deselected_nodes"])), + "isolation_metadata_file": isolation_path.name, + "isolation_metadata_sha256": _sha256(isolation_path.read_bytes()), + "skipped_nodes": sorted(set(node for unit in units for node in unit["skipped_nodes"])), } evidence_path.write_bytes(_json_bytes(evidence)) if coverage_path.is_file() and not coverage_path.is_symlink(): - for unit in ordered_units: + for unit in units: source = unit["coverage_path"] if source != coverage_path and source.is_file() and not source.is_symlink(): source.unlink() return { "collection_exit_code": _aggregate_exit_codes( - [unit["collection_exit_code"] for unit in ordered_units] + [unit["collection_exit_code"] for unit in units] ), "coverage_file": coverage_path.name, "coverage_sha256": _sha256(coverage_path.read_bytes()) if coverage_path.is_file() and not coverage_path.is_symlink() else None, - "elapsed_seconds": round( - max(unit["elapsed_seconds"] for unit in ordered_units), 3 - ), + "elapsed_seconds": round(max(unit["elapsed_seconds"] for unit in units), 3), "evidence_file": evidence_path.name, "evidence_sha256": _sha256(evidence_path.read_bytes()), "execution_exit_code": execution_exit_code, - "interrupted": any(unit["interrupted"] for unit in ordered_units), + "interrupted": any(unit["interrupted"] for unit in units), "name": lane.name, } @@ -700,34 +643,6 @@ def _timing_summary(lanes: list[dict[str, Any]]) -> dict[str, float]: } -def _ordinary_units( - lane: TestLane, ordinary_rows: list[dict[str, str]] -) -> list[tuple[str, list[dict[str, str]]]]: - """Partition one public lane into fixed dependency-owned execution units.""" - configured_units = SEMANTIC_UNITS.get(lane.name) - if configured_units is None: - return [(lane.name, ordinary_rows)] - configured_modules = [ - module for _suffix, modules in configured_units for module in modules - ] - observed_modules = {row["module"] for row in ordinary_rows} - if ( - len(configured_modules) != len(set(configured_modules)) - or set(configured_modules) != observed_modules - ): - raise LaneError("invalid_semantic_unit_inventory") - units = [ - ( - f"{lane.name}_{suffix}", - [row for row in ordinary_rows if row["module"] in modules], - ) - for suffix, modules in configured_units - ] - if any(not rows for _key, rows in units): - raise LaneError("empty_semantic_unit") - return units - - def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, collect_only: bool = False) -> int: """Collect canonical nodes and optionally execute all lanes concurrently.""" global INTERRUPTED @@ -752,7 +667,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, evidence_path = metadata_dir / f"{lane.name}.json" evidence_path.write_bytes(_json_bytes({ "collected_nodes": lane_nodes, "completed_nodes": [], "deselected_nodes": [], - "isolation_metadata_files": [], "isolation_metadata_sha256s": [], + "isolation_metadata_file": None, "isolation_metadata_sha256": None, "skipped_nodes": [], })) lane_rows.append({ @@ -779,22 +694,14 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, try: for lane in LANES: lane_rows = [row for row in manifest["nodes"] if row["lane"] == lane.name] - ordinary_rows = [ - row for row in lane_rows if row["execution_kind"] == ORDINARY_KIND - ] - units = [ - (key, ORDINARY_KIND, rows) - for key, rows in _ordinary_units(lane, ordinary_rows) - ] - admin_rows = [ - row for row in lane_rows if row["execution_kind"] == ADMIN_KIND - ] - if admin_rows: - units.append((f"{lane.name}.admin", ADMIN_KIND, admin_rows)) - for key, kind, unit_rows in units: - unit_nodes = [row["nodeid"] for row in unit_rows] + kinds = [ORDINARY_KIND] + if any(row["execution_kind"] == ADMIN_KIND for row in lane_rows): + kinds.append(ADMIN_KIND) + for kind in kinds: + unit_nodes = [row["nodeid"] for row in lane_rows if row["execution_kind"] == kind] if not unit_nodes: - raise LaneError("empty_semantic_unit") + continue + key = lane.name if kind == ORDINARY_KIND else f"{lane.name}.admin" for suffix in ("collected", "completed", "skipped", "deselected"): _exclusive_file(metadata_dir / f"{key}.{suffix}.jsonl") log_path = metadata_dir / f"{key}.log" @@ -806,9 +713,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, lane, metadata_dir, coverage_path, key, tree_sha ) else: - command = lane_command( - lane, key, unit_nodes, metadata_dir, timeout_seconds, tree_sha - ) + command = lane_command(lane, unit_nodes, metadata_dir, timeout_seconds, tree_sha) env = lane_environment(lane, metadata_dir, coverage_path, key, tree_sha) process = subprocess.Popen( command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, @@ -817,7 +722,6 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, active[key] = ActiveLane( key, lane, kind, tuple(unit_nodes), process, log, log_path, metadata_dir / f"{key}.json", - metadata_dir / f"{key}.database.json", coverage_path, time.monotonic(), ) while active: diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 03d947ef..70b2aa60 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -21,12 +21,6 @@ SCHEMA_VERSION = 1 LANE_COUNT = 4 -ISOLATION_LANES = { - "no_postgres": {"no_postgres"}, - "schema_contracts": {"schema_contracts"}, - "control_plane": {"control_plane_authority", "control_plane_projects"}, - "execution_plane": {"execution_plane_artifacts", "execution_plane_tasks_checkers"}, -} SHA_RE = re.compile(r"^[0-9a-f]{40}$") DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") LANE_RE = re.compile(r"^[a-z][a-z0-9_]*$") @@ -476,8 +470,8 @@ def validate_evidence( "collected_nodes", "completed_nodes", "deselected_nodes", - "isolation_metadata_files", - "isolation_metadata_sha256s", + "isolation_metadata_file", + "isolation_metadata_sha256", "skipped_nodes", }, "invalid_lane_evidence", @@ -502,8 +496,8 @@ def validate_evidence( ): raise EvidenceError("invalid_collect_mode_artifacts") if ( - evidence["isolation_metadata_files"] != [] - or evidence["isolation_metadata_sha256s"] != [] + evidence["isolation_metadata_file"] is not None + or evidence["isolation_metadata_sha256"] is not None ): raise EvidenceError("invalid_collect_mode_artifacts") else: @@ -511,82 +505,64 @@ def validate_evidence( raise EvidenceError("partial_or_duplicate_completion") _bound_bytes(metadata_dir, lane["coverage_file"], lane["coverage_sha256"]) coverage_files.append(lane["coverage_file"]) - metadata_files = evidence["isolation_metadata_files"] - metadata_digests = evidence["isolation_metadata_sha256s"] - if ( - not isinstance(metadata_files, list) - or not isinstance(metadata_digests, list) - or len(metadata_files) != len(metadata_digests) - or len(set(metadata_files)) != len(metadata_files) - or any(not isinstance(value, str) for value in metadata_files) - or any(not isinstance(value, str) for value in metadata_digests) - ): - raise EvidenceError("invalid_isolation_inventory") - observed_resource_lanes: set[str] = set() - for metadata_file, metadata_digest in zip( - metadata_files, metadata_digests, strict=True - ): - isolation_files.append(metadata_file) - isolation = _object( - _json_bytes( - _bound_bytes(metadata_dir, metadata_file, metadata_digest), - "invalid_isolation_metadata", + isolation_files.append(evidence["isolation_metadata_file"]) + isolation = _object( + _json_bytes( + _bound_bytes( + metadata_dir, + evidence["isolation_metadata_file"], + evidence["isolation_metadata_sha256"], ), - { - "alembic_head", - "cleanup_complete", - "database_name", - "database_cleanup_complete", - "database_provisioned", - "database_role", - "lane", - "minio_bucket", - "minio_cleanup_complete", - "minio_prefix", - "minio_probe_complete", - "minio_provisioned", - "schema_version", - "tree_sha", - }, "invalid_isolation_metadata", - ) - namespace_fields = ( + ), + { + "alembic_head", + "cleanup_complete", "database_name", + "database_cleanup_complete", + "database_provisioned", "database_role", + "lane", "minio_bucket", + "minio_cleanup_complete", "minio_prefix", + "minio_probe_complete", + "minio_provisioned", + "schema_version", + "tree_sha", + }, + "invalid_isolation_metadata", + ) + namespace_fields = ( + "database_name", + "database_role", + "minio_bucket", + "minio_prefix", + ) + if ( + isolation["schema_version"] != 2 + or isolation["tree_sha"] != head + or isolation["lane"] != name + or isolation["database_provisioned"] is not True + or isolation["minio_provisioned"] is not True + or isolation["database_cleanup_complete"] is not True + or isolation["minio_probe_complete"] is not True + or isolation["minio_cleanup_complete"] is not True + or isolation["cleanup_complete"] is not True + or not isinstance(isolation["alembic_head"], str) + or not isolation["alembic_head"] + or any( + not isinstance(isolation[field], str) or not isolation[field] + for field in namespace_fields ) - resource_lane = isolation["lane"] - if ( - isolation["schema_version"] != 2 - or isolation["tree_sha"] != head - or not isinstance(resource_lane, str) - or resource_lane not in ISOLATION_LANES.get(name, set()) - or isolation["database_provisioned"] is not True - or isolation["minio_provisioned"] is not True - or isolation["database_cleanup_complete"] is not True - or isolation["minio_probe_complete"] is not True - or isolation["minio_cleanup_complete"] is not True - or isolation["cleanup_complete"] is not True - or not isinstance(isolation["alembic_head"], str) - or not isolation["alembic_head"] - or any( - not isinstance(isolation[field], str) or not isolation[field] - for field in namespace_fields - ) - or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_name"]) is None - or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_role"]) is None - or BUCKET_RE.fullmatch(isolation["minio_bucket"]) is None - or PurePosixPath(isolation["minio_prefix"]).is_absolute() - or ".." in PurePosixPath(isolation["minio_prefix"]).parts - ): - raise EvidenceError("invalid_isolation_metadata") - observed_resource_lanes.add(resource_lane) - isolation_namespaces.append( - tuple(isolation[field] for field in namespace_fields) - ) - if observed_resource_lanes != ISOLATION_LANES.get(name): - raise EvidenceError("invalid_isolation_inventory") + or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_name"]) is None + or DATABASE_IDENTIFIER_RE.fullmatch(isolation["database_role"]) is None + or BUCKET_RE.fullmatch(isolation["minio_bucket"]) is None + or PurePosixPath(isolation["minio_prefix"]).is_absolute() + or ".." in PurePosixPath(isolation["minio_prefix"]).parts + ): + raise EvidenceError("invalid_isolation_metadata") + isolation_namespaces.append(tuple(isolation[field] for field in namespace_fields)) all_collected.extend(collected) all_completed.extend(completed) @@ -595,14 +571,12 @@ def validate_evidence( if mode == "run" and Counter(all_completed) != Counter(canonical_ids): raise EvidenceError("global_completion_reconciliation_failed") if mode == "run" and any( - len({namespace[index] for namespace in isolation_namespaces}) - != len(isolation_namespaces) + len({namespace[index] for namespace in isolation_namespaces}) != LANE_COUNT for index in range(4) ): raise EvidenceError("shared_isolation_namespace") if mode == "run" and ( - len(set(coverage_files)) != LANE_COUNT - or len(set(isolation_files)) != sum(len(value) for value in ISOLATION_LANES.values()) + len(set(coverage_files)) != LANE_COUNT or len(set(isolation_files)) != LANE_COUNT ): raise EvidenceError("shared_lane_artifact") aggregate = summary["aggregate_runner_seconds"] diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index bf45bcce..92b8f485 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -12,7 +12,6 @@ import uuid import pytest # type: ignore[import-not-found] -from coverage import CoverageData import scripts.run_test_lanes as runner from scripts.run_test_lanes import LANES, LaneError, TestLane as LaneDefinition @@ -199,9 +198,7 @@ def test_uuid4_and_repository_import_alias_are_restored_before_test_body( def test_lane_command_uses_exact_nodes_and_isolation_contract(tmp_path: Path) -> None: lane = LANES[0] nodes = [f"{lane.modules[0]}::test_exact[param]"] - command = runner.lane_command( - lane, lane.name, nodes, tmp_path, 1200, "b" * 40 - ) + command = runner.lane_command(lane, nodes, tmp_path, 1200, "b" * 40) assert command[1].endswith("scripts/run_isolated_tests.py") assert command[command.index("--lane") + 1] == lane.name @@ -282,7 +279,6 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_kind": runner.ORDINARY_KIND, "execution_exit_code": 0, "interrupted": False, - "isolation_path": isolation, "skipped_nodes": [], }, { @@ -295,7 +291,6 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag "execution_kind": runner.ADMIN_KIND, "execution_exit_code": 0, "interrupted": False, - "isolation_path": tmp_path / "unused-admin.database.json", "skipped_nodes": [], }, ] @@ -309,8 +304,7 @@ def test_finalize_lane_requires_ordinary_coverage_but_allows_empty_admin_coverag def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None: lane = LANES[0] - isolation = tmp_path / f"{lane.name}.database.json" - isolation.write_text("{}\n", encoding="utf-8") + tmp_path.joinpath(f"{lane.name}.database.json").write_text("{}\n", encoding="utf-8") units = [ { "collection_exit_code": 0, @@ -322,7 +316,6 @@ def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None "execution_kind": runner.ORDINARY_KIND, "execution_exit_code": 0, "interrupted": False, - "isolation_path": isolation, "skipped_nodes": [], } ] @@ -331,26 +324,6 @@ def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None runner._finalize_lane(lane, units, tmp_path) -def test_combine_coverage_merges_multiple_semantic_unit_files(tmp_path: Path) -> None: - sources = [] - for index in range(2): - source = tmp_path / f".coverage.unit.{index}" - data = CoverageData(basename=str(source)) - data.add_lines({str(tmp_path / f"module_{index}.py"): {index + 1}}) - data.write() - sources.append(source) - destination = tmp_path / ".coverage.public_lane" - - runner._combine_coverage(sources, destination) - - combined = CoverageData(basename=str(destination)) - combined.read() - assert set(combined.measured_files()) == { - str(tmp_path / "module_0.py"), - str(tmp_path / "module_1.py"), - } - - def test_collect_only_writes_raw_digest_bound_validator_schema( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -419,38 +392,6 @@ def test_timing_summary_is_derived_from_exact_four_lanes() -> None: runner._timing_summary([*lanes[:3], {"elapsed_seconds": -1.0}]) -def test_slow_lanes_use_exact_dependency_owned_execution_units() -> None: - for lane_name, expected_keys in { - "control_plane": {"control_plane_authority", "control_plane_projects"}, - "execution_plane": { - "execution_plane_artifacts", - "execution_plane_tasks_checkers", - }, - }.items(): - lane = next(item for item in LANES if item.name == lane_name) - rows = [ - {"module": module, "nodeid": f"{module}::test_one"} - for module in lane.modules - ] - units = runner._ordinary_units(lane, rows) - - assert {key for key, _rows in units} == expected_keys - assert sorted(row["nodeid"] for _key, unit_rows in units for row in unit_rows) == sorted( - row["nodeid"] for row in rows - ) - - -def test_semantic_execution_units_reject_module_drift() -> None: - lane = next(item for item in LANES if item.name == "control_plane") - rows = [ - {"module": module, "nodeid": f"{module}::test_one"} - for module in lane.modules[:-1] - ] - - with pytest.raises(LaneError, match="invalid_semantic_unit_inventory"): - runner._ordinary_units(lane, rows) - - def test_exit_aggregation_cannot_hide_signal_failure() -> None: assert runner._aggregate_exit_codes([0, 0]) == 0 assert runner._aggregate_exit_codes([0, 3]) == 1 @@ -464,8 +405,7 @@ def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path for index, lane in enumerate(LANES): source = tmp_path / f".coverage.unit.{lane.name}" source.write_bytes(f"coverage-{lane.name}".encode()) - isolation = tmp_path / f"{lane.name}.database.json" - isolation.write_text("{}\n", encoding="utf-8") + (tmp_path / f"{lane.name}.database.json").write_text("{}\n", encoding="utf-8") unit = { "collected_nodes": [f"{lane.modules[0]}::test_one"], "collection_exit_code": 0, @@ -475,7 +415,6 @@ def test_finalized_lanes_leave_exactly_four_public_coverage_files(tmp_path: Path "elapsed_seconds": float(index + 1), "execution_kind": runner.ORDINARY_KIND, "execution_exit_code": 0, - "isolation_path": isolation, "interrupted": False, "skipped_nodes": [], } @@ -503,10 +442,10 @@ def test_failure_interrupts_sibling_process_groups_and_records_all_lanes( monkeypatch.setenv(runner.ADMIN_ENV, "postgresql+asyncpg://admin@localhost/postgres") monkeypatch.setattr(runner, "CLEANUP_GRACE_SECONDS", 0.2) - def fake_command(lane, resource_lane, _nodes, metadata, _timeout, _sha): - database = metadata / f"{resource_lane}.database.json" + def fake_command(lane, _nodes, metadata, _timeout, _sha): + database = metadata / f"{lane.name}.database.json" database.write_text("{}\n", encoding="utf-8") - coverage = metadata / f".coverage.unit.{resource_lane}" + coverage = metadata / f".coverage.{lane.name}" coverage.write_bytes(b"coverage") if lane.name == "lane_0": return [sys.executable, "-c", "raise SystemExit(1)"] diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index ad5bc922..4b9a0136 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -21,12 +21,6 @@ REAL_COLLECT_CURRENT_NODES = validator._collect_current_nodes HEAD = "a" * 40 LANES = ("no_postgres", "schema_contracts", "control_plane", "execution_plane") -ISOLATION_LANES = { - "no_postgres": ("no_postgres",), - "schema_contracts": ("schema_contracts",), - "control_plane": ("control_plane_authority", "control_plane_projects"), - "execution_plane": ("execution_plane_artifacts", "execution_plane_tasks_checkers"), -} def _write(path: Path, value: object) -> str: @@ -64,35 +58,31 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: lane_rows = [] for lane in LANES: nodeids = [row["nodeid"] for row in nodes if row["lane"] == lane] - isolation_files = [] - isolation_digests = [] + isolation_file = None + isolation_digest = None coverage_file = None coverage_digest = None if mode == "run": - for resource_lane in ISOLATION_LANES[lane]: - isolation_file = f"{resource_lane}.isolation.json" - isolation_files.append(isolation_file) - isolation_digests.append( - _write( - metadata / isolation_file, - { - "alembic_head": "head", - "cleanup_complete": True, - "database_cleanup_complete": True, - "database_name": f"database_{resource_lane}", - "database_provisioned": True, - "database_role": f"role_{resource_lane}", - "lane": resource_lane, - "minio_bucket": f"bucket-{resource_lane.replace('_', '-')}", - "minio_cleanup_complete": True, - "minio_prefix": f"prefix/{resource_lane}", - "minio_probe_complete": True, - "minio_provisioned": True, - "schema_version": 2, - "tree_sha": HEAD, - }, - ) - ) + isolation_file = f"{lane}.isolation.json" + isolation_digest = _write( + metadata / isolation_file, + { + "alembic_head": "head", + "cleanup_complete": True, + "database_cleanup_complete": True, + "database_name": f"database_{lane}", + "database_provisioned": True, + "database_role": f"role_{lane}", + "lane": lane, + "minio_bucket": f"bucket-{lane.replace('_', '-')}", + "minio_cleanup_complete": True, + "minio_prefix": f"prefix/{lane}", + "minio_probe_complete": True, + "minio_provisioned": True, + "schema_version": 2, + "tree_sha": HEAD, + }, + ) coverage_file = f"coverage.{lane}" (metadata / coverage_file).write_bytes(f"coverage:{lane}".encode()) coverage_digest = hashlib.sha256((metadata / coverage_file).read_bytes()).hexdigest() @@ -103,8 +93,8 @@ def _bundle(tmp_path: Path, mode: str = "run") -> tuple[Path, Path, dict]: "collected_nodes": nodeids, "completed_nodes": nodeids if mode == "run" else [], "deselected_nodes": [], - "isolation_metadata_files": isolation_files, - "isolation_metadata_sha256s": isolation_digests, + "isolation_metadata_file": isolation_file, + "isolation_metadata_sha256": isolation_digest, "skipped_nodes": [], }, ) @@ -309,10 +299,10 @@ def test_rejects_recorded_database_environment_or_shared_coverage(tmp_path: Path metadata, summary_path, summary = _bundle(tmp_path) lane = summary["lanes"][0] evidence = json.loads((metadata / lane["evidence_file"]).read_text()) - isolation_path = metadata / evidence["isolation_metadata_files"][0] + isolation_path = metadata / evidence["isolation_metadata_file"] isolation = json.loads(isolation_path.read_text()) isolation["admin_database_url"] = "postgresql://admin:secret@example.invalid/db" - evidence["isolation_metadata_sha256s"][0] = _write(isolation_path, isolation) + evidence["isolation_metadata_sha256"] = _write(isolation_path, isolation) lane["evidence_sha256"] = _write(metadata / lane["evidence_file"], evidence) _write(summary_path, summary) with pytest.raises(validator.EvidenceError, match="invalid_isolation_metadata"): @@ -326,34 +316,6 @@ def test_rejects_recorded_database_environment_or_shared_coverage(tmp_path: Path validator.validate_evidence(metadata, summary_path, tmp_path) -def test_rejects_missing_or_foreign_semantic_isolation_unit(tmp_path: Path) -> None: - metadata, summary_path, summary = _bundle(tmp_path) - lane = next(row for row in summary["lanes"] if row["name"] == "control_plane") - evidence_path = metadata / lane["evidence_file"] - evidence = json.loads(evidence_path.read_text()) - evidence["isolation_metadata_files"].pop() - evidence["isolation_metadata_sha256s"].pop() - lane["evidence_sha256"] = _write(evidence_path, evidence) - _write(summary_path, summary) - - with pytest.raises(validator.EvidenceError, match="invalid_isolation_inventory"): - validator.validate_evidence(metadata, summary_path, tmp_path) - - metadata, summary_path, summary = _bundle(tmp_path / "foreign") - lane = next(row for row in summary["lanes"] if row["name"] == "control_plane") - evidence_path = metadata / lane["evidence_file"] - evidence = json.loads(evidence_path.read_text()) - isolation_path = metadata / evidence["isolation_metadata_files"][0] - isolation = json.loads(isolation_path.read_text()) - isolation["lane"] = "control_plane_foreign" - evidence["isolation_metadata_sha256s"][0] = _write(isolation_path, isolation) - lane["evidence_sha256"] = _write(evidence_path, evidence) - _write(summary_path, summary) - - with pytest.raises(validator.EvidenceError, match="invalid_isolation_metadata"): - validator.validate_evidence(metadata, summary_path, tmp_path) - - @pytest.mark.parametrize( ("field", "value", "message"), [ diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index cec7f5b5..f82af57e 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -69,21 +69,16 @@ If provisioning fails, confirm the local PostgreSQL provisioning credential can The required GitHub check remains `Backend / test`. One job owns one digest-pinned PostgreSQL service, one digest-pinned MinIO service, and four -public dependency lanes. The two slow database-heavy lanes each use two fixed -semantic execution units, so six ordinary units run concurrently without -reintroducing arbitrary shards or artifact fan-in. +concurrent dependency lanes. This avoids arbitrary shard fan-out and artifact +fan-in while retaining exact node and coverage custody. The job binds the checkout to `GITHUB_SHA`, installs and asserts exact Ruff `0.15.22`, runs lint and docstrings, starts MinIO, then collects every canonical pytest node. The independent evidence validator must -accept the collection before execution begins. Each ordinary execution unit -receives a distinct runner-created database and role plus a distinct MinIO -bucket/prefix custody record. The no-PostgreSQL lane owns the actual -`workstream-artifacts` test bucket and a unique run prefix; the other five units -create, probe, and remove distinct buckets. The public `control_plane` lane is -split into authority and project units; `execution_plane` is split into -artifact/outbox and task/checker units. Their node sets and coverage are -reconciled back into their public lanes. +accept the collection before execution begins. Each lane receives a distinct +runner-created database and role plus a distinct MinIO bucket/prefix custody +record. The S3 lane owns the actual `workstream-artifacts` test bucket and a +unique run prefix; other lanes create, probe, and remove distinct buckets. The isolated-runner self-tests remain in the canonical manifest as the explicit `admin_runner_self_test` execution kind. The lane orchestrator runs only those nodes directly with the admin URL while stripping application database URLs; @@ -102,12 +97,11 @@ isolated invocation inside the same required job. ### Evidence bundle The workflow uploads the `.ci/test-lanes` tree even on failure. Its summary -records the exact head, canonical node count, four public lane results, elapsed -time, and raw-file digests. Per-lane evidence records collected, completed, -skipped, and deselected exact node IDs plus every bound execution-unit resource -record and the public lane coverage digest. The six resource records are mode -`0600`, omit credentials, and prove database, role, bucket, prefix, probe, and -cleanup custody. +records the exact head, canonical node count, four lane results, elapsed time, +and raw-file digests. Per-lane evidence records collected, completed, skipped, +and deselected exact node IDs plus the bound resource-isolation metadata and +coverage digest. Resource metadata is mode `0600`, omits credentials, and proves +database, role, bucket, prefix, probe, and cleanup custody. The validator accepts only safe repository-local regular files and exact schema keys. It rejects symlinks, traversal, stale heads, digest drift, unexpected diff --git a/scripts/test_agent_gates.py b/scripts/test_agent_gates.py index 90b8ab26..ab284c49 100644 --- a/scripts/test_agent_gates.py +++ b/scripts/test_agent_gates.py @@ -6514,15 +6514,6 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert "run_isolated_tests.py" in lane_runner assert "admin_runner_self_test" in lane_validator assert "execution_kind" in lane_validator - assert "SEMANTIC_UNITS" in lane_runner - assert '"control_plane"' in lane_runner - assert '"execution_plane"' in lane_runner - assert '"control_plane_authority"' in lane_validator - assert '"control_plane_projects"' in lane_validator - assert '"execution_plane_artifacts"' in lane_validator - assert '"execution_plane_tasks_checkers"' in lane_validator - assert "ISOLATION_LANES" in lane_validator - assert "invalid_isolation_inventory" in lane_validator api_e2e_steps = [ step for step in steps if step.get("name") == "API contract real API e2e" From 15e358949bf469fa78e08fa42c2e690a829c3917 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 20:19:31 +0100 Subject: [PATCH 21/31] perf(ci): rebalance semantic test lanes --- .../STATUS.md | 10 ++++--- backend/scripts/run_isolated_tests.py | 2 +- backend/scripts/run_test_lanes.py | 26 +++++++++--------- backend/tests/test_ci_test_lanes.py | 23 +++++++++++++++- .../tests/test_isolated_database_runner.py | 27 +++++++++++-------- backend/tests/test_test_lane_evidence.py | 7 ++++- docs/operations_backend_testing.md | 10 +++++-- 7 files changed, 73 insertions(+), 32 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md index bff75000..30720523 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md @@ -7,8 +7,12 @@ - Later proposed chunk: `WS-CI-001-03` planning only; not a declared successor - Human direction: preserve Konan's authorship and measured work from PR #180, but adopt it only under prospective zero-trust scope and evidence -- Current gate: exact-custody semantic-lane implementation is in deterministic - proof; hosted full Backend execution, exact node/coverage/resource evidence, - internal review, external review, and user-owned merge decision remain +- Current gate: exact PR head `8bc7e9f0` proved all 2,056 nodes, coverage + floors, API E2E, and resource custody, but its six-process experiment + increased runner contention and reached the timing checkpoint in about 550 + seconds. That experiment is reverted. The active repair keeps exactly four + isolated processes and rebalances them as project lifecycle, task lifecycle, + schema contracts, and shared foundations. Hosted timing, refreshed internal + review, external review, and the user-owned merge decision remain - Deferred option: routing/cache/timing reassessment is future `WS-CI-001-03`, with no start or successor declaration diff --git a/backend/scripts/run_isolated_tests.py b/backend/scripts/run_isolated_tests.py index a084845b..39236257 100644 --- a/backend/scripts/run_isolated_tests.py +++ b/backend/scripts/run_isolated_tests.py @@ -34,7 +34,7 @@ MINIO_ENDPOINT_ENV = "WORKSTREAM_TEST_MINIO_ENDPOINT" MINIO_ACCESS_KEY = "workstream-minio" MINIO_SECRET_KEY = "workstream-minio-secret-key" -S3_TRAFFIC_LANE = "no_postgres" +S3_TRAFFIC_LANE = "shared_foundations" S3_TRAFFIC_BUCKET = "workstream-artifacts" LANE_RE = re.compile(r"[a-z][a-z0-9_]{0,62}") BUCKET_RE = re.compile(r"[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?") diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 903c0788..e34d1c1d 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -72,7 +72,7 @@ class TestLane: LANES = ( TestLane( - "no_postgres", + "shared_foundations", ( "tests/test_actor_legacy_classification.py", "tests/test_actor_migration_tools.py", @@ -96,6 +96,16 @@ class TestLane: "tests/test_local_artifact_store.py", "tests/test_s3_artifact_store.py", "tests/test_test_lane_evidence.py", + "tests/test_actors.py", + "tests/test_api_rate_controls.py", + "tests/test_audit.py", + "tests/test_auth.py", + "tests/test_authorization.py", + "tests/test_artifact_admission.py", + "tests/test_artifact_operator_api.py", + "tests/test_artifact_recovery.py", + "tests/test_db_session.py", + "tests/test_outbox.py", ), ), TestLane( @@ -107,25 +117,15 @@ class TestLane: ), ), TestLane( - "control_plane", + "project_lifecycle", ( - "tests/test_actors.py", - "tests/test_api_rate_controls.py", - "tests/test_audit.py", - "tests/test_auth.py", - "tests/test_authorization.py", "tests/test_projects.py", ), ), TestLane( - "execution_plane", + "task_lifecycle", ( - "tests/test_artifact_admission.py", - "tests/test_artifact_operator_api.py", - "tests/test_artifact_recovery.py", "tests/test_checkers.py", - "tests/test_db_session.py", - "tests/test_outbox.py", "tests/test_tasks.py", ), ), diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 92b8f485..0492aacf 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -31,6 +31,27 @@ def test_committed_lanes_cover_recursive_inventory_exactly_once() -> None: ) +def test_measured_hotspots_have_explicit_semantic_owners() -> None: + """Keep the four-lane balance tied to subsystem ownership, not test counts.""" + modules_by_lane = {lane.name: set(lane.modules) for lane in LANES} + + assert modules_by_lane["project_lifecycle"] == {"tests/test_projects.py"} + assert modules_by_lane["task_lifecycle"] == { + "tests/test_checkers.py", + "tests/test_tasks.py", + } + assert { + "tests/test_alembic.py", + "tests/test_database_reset.py", + runner.ADMIN_RUNNER_MODULE, + } == modules_by_lane["schema_contracts"] + assert { + "tests/test_actors.py", + "tests/test_artifact_admission.py", + "tests/test_authorization.py", + } <= modules_by_lane["shared_foundations"] + + def test_discovery_is_recursive_and_lexically_canonical(tmp_path: Path) -> None: tests = tmp_path / "tests" nested = tests / "nested" @@ -209,7 +230,7 @@ def test_lane_command_uses_exact_nodes_and_isolation_contract(tmp_path: Path) -> def test_lane_environment_uses_private_evidence_and_coverage(tmp_path: Path) -> None: - coverage = tmp_path / ".coverage.no_postgres" + coverage = tmp_path / ".coverage.shared_foundations" env = runner.lane_environment(LANES[0], tmp_path, coverage) assert env["COVERAGE_FILE"] == str(coverage.resolve()) diff --git a/backend/tests/test_isolated_database_runner.py b/backend/tests/test_isolated_database_runner.py index b3c3a2f3..2d42218d 100644 --- a/backend/tests/test_isolated_database_runner.py +++ b/backend/tests/test_isolated_database_runner.py @@ -65,16 +65,16 @@ async def observed(*_): return "0015" def test_lane_namespaces_bind_real_s3_traffic_and_separate_other_lanes() -> None: """Only the S3 lane receives the application's hardcoded integration bucket.""" - s3_bucket, s3_prefix = runner._minio_namespace("no_postgres", "012345abcdef") + s3_bucket, s3_prefix = runner._minio_namespace("shared_foundations", "012345abcdef") control_bucket, control_prefix = runner._minio_namespace( - "control_plane", "012345abcdef" + "project_lifecycle", "012345abcdef" ) execution_bucket, execution_prefix = runner._minio_namespace( - "execution_plane", "fedcba543210" + "task_lifecycle", "fedcba543210" ) assert (s3_bucket, s3_prefix) == ( "workstream-artifacts", - "ci/no_postgres/012345abcdef", + "ci/shared_foundations/012345abcdef", ) assert len({s3_bucket, control_bucket, execution_bucket}) == 3 assert len({s3_prefix, control_prefix, execution_prefix}) == 3 @@ -85,10 +85,10 @@ def test_lane_namespaces_bind_real_s3_traffic_and_separate_other_lanes() -> None @pytest.mark.parametrize( ("lane", "expected_bucket"), [ - ("no_postgres", "workstream-artifacts"), + ("shared_foundations", "workstream-artifacts"), ("schema_contracts", "workstream-ci-schema-contracts-012345abcdef"), - ("control_plane", "workstream-ci-control-plane-012345abcdef"), - ("execution_plane", "workstream-ci-execution-plane-012345abcdef"), + ("project_lifecycle", "workstream-ci-project-lifecycle-012345abcdef"), + ("task_lifecycle", "workstream-ci-task-lifecycle-012345abcdef"), ], ) def test_committed_lane_buckets_use_validator_compatible_s3_grammar( @@ -107,12 +107,17 @@ def test_lane_namespaces_do_not_collide_across_lanes_or_runner_suffixes() -> Non """Separate lane identities and runner invocations cannot share custody.""" namespaces = { runner._minio_namespace(lane, suffix) - for lane in ("no_postgres", "schema_contracts", "control_plane", "execution_plane") + for lane in ( + "shared_foundations", + "schema_contracts", + "project_lifecycle", + "task_lifecycle", + ) for suffix in ("012345abcdef", "fedcba543210") } assert len(namespaces) == 8 with pytest.raises(runner.RunnerError, match="invalid_lane"): - runner._minio_namespace("control_plane", "not-hex") + runner._minio_namespace("project_lifecycle", "not-hex") with pytest.raises(runner.RunnerError, match="invalid_minio_namespace"): runner._minio_namespace("a" * 63, "012345abcdef") @@ -122,10 +127,10 @@ def test_child_environment_binds_lane_namespace_without_admin_custody() -> None: env = runner._child_env( "postgresql+asyncpg://role:password@localhost/database", minio_bucket="workstream-artifacts", - minio_prefix="ci/no_postgres/012345abcdef", + minio_prefix="ci/shared_foundations/012345abcdef", ) assert env["WORKSTREAM_TEST_MINIO_BUCKET"] == "workstream-artifacts" - assert env["WORKSTREAM_TEST_MINIO_PREFIX"] == "ci/no_postgres/012345abcdef" + assert env["WORKSTREAM_TEST_MINIO_PREFIX"] == "ci/shared_foundations/012345abcdef" assert runner.ADMIN_ENV not in env assert runner.OVERRIDE_ENV not in env diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index 4b9a0136..86b67049 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -20,7 +20,12 @@ SPEC.loader.exec_module(validator) REAL_COLLECT_CURRENT_NODES = validator._collect_current_nodes HEAD = "a" * 40 -LANES = ("no_postgres", "schema_contracts", "control_plane", "execution_plane") +LANES = ( + "shared_foundations", + "schema_contracts", + "project_lifecycle", + "task_lifecycle", +) def _write(path: Path, value: object) -> str: diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index f82af57e..c9fe9a85 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -72,13 +72,19 @@ digest-pinned PostgreSQL service, one digest-pinned MinIO service, and four concurrent dependency lanes. This avoids arbitrary shard fan-out and artifact fan-in while retaining exact node and coverage custody. +The lanes are balanced by measured dependency ownership: `project_lifecycle` +owns project tests, `task_lifecycle` owns task and checker tests, +`schema_contracts` owns migrations and reset contracts, and +`shared_foundations` owns the remaining authorization, artifact, API, and +infrastructure tests. Every discovered module must belong to exactly one lane. + The job binds the checkout to `GITHUB_SHA`, installs and asserts exact Ruff `0.15.22`, runs lint and docstrings, starts MinIO, then collects every canonical pytest node. The independent evidence validator must accept the collection before execution begins. Each lane receives a distinct runner-created database and role plus a distinct MinIO bucket/prefix custody -record. The S3 lane owns the actual `workstream-artifacts` test bucket and a -unique run prefix; other lanes create, probe, and remove distinct buckets. +record. `shared_foundations` owns the actual `workstream-artifacts` test bucket +and a unique run prefix; other lanes create, probe, and remove distinct buckets. The isolated-runner self-tests remain in the canonical manifest as the explicit `admin_runner_self_test` execution kind. The lane orchestrator runs only those nodes directly with the admin URL while stripping application database URLs; From bf16f1a6d01f270e8d25d3deee037431a8cb8676 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 20:29:38 +0100 Subject: [PATCH 22/31] test(ci): align semantic lane fixtures --- backend/tests/test_isolated_database_runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/tests/test_isolated_database_runner.py b/backend/tests/test_isolated_database_runner.py index 2d42218d..bc89ae3e 100644 --- a/backend/tests/test_isolated_database_runner.py +++ b/backend/tests/test_isolated_database_runner.py @@ -154,10 +154,10 @@ def test_minio_boundary_rejects_nonlocal_or_credentialed_endpoints(endpoint: str def test_metadata_is_private_deterministic_and_refuses_symlinks(tmp_path: Path) -> None: """Custody evidence is stable, mode 0600, and never follows an output symlink.""" path = tmp_path / "runner.json" - runner._write_metadata(path, {"tree_sha": "1" * 40, "lane": "control_plane"}) + runner._write_metadata(path, {"tree_sha": "1" * 40, "lane": "project_lifecycle"}) assert path.stat().st_mode & 0o777 == 0o600 assert path.read_text(encoding="utf-8") == ( - '{\n "lane": "control_plane",\n "tree_sha": "' + "1" * 40 + '"\n}\n' + '{\n "lane": "project_lifecycle",\n "tree_sha": "' + "1" * 40 + '"\n}\n' ) path.unlink() path.symlink_to(tmp_path / "outside.json") @@ -212,7 +212,7 @@ def test_tree_binding_rejects_foreign_expected_head( "--metadata-json", str(tmp_path / "db.json"), "--lane", - "control_plane", + "project_lifecycle", "--tree-sha", "2" * 40, "--", From f5d2abd74fc0cf45cd742d31cfff32bd831cd661 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 20:38:48 +0100 Subject: [PATCH 23/31] docs(agent-loop): record lane rebalance evidence --- .../WS-CI-001-02B-internal-review-evidence.md | 53 +++++++++++-------- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 52 +++++++++--------- 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index 59b3b5b5..2af32e79 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,32 +12,30 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: 239adb178229c72951862780e5ce237f73113400 +Reviewed code SHA: bf16f1a6d01f270e8d25d3deee037431a8cb8676 -Reviewed at: 2026-07-24T16:55:21Z +Reviewed at: 2026-07-24T19:35:11Z -Reviewer run IDs: ci02b_senior_review, ci02b_qa_review, -ci02b_security_review, ci02b_product_ops_review, -ci02b_restart_arch_review, ci02b_restart_ci_review, ci02b_contract_gap, -ci02b_source_audit, ci02b_test_delta_review, ci02b_bootstrap_senior, -ci02b_bootstrap_qa, ci02b_bootstrap_security, ci02b_bootstrap_ops, -ci02b_bootstrap_arch, ci02b_bootstrap_ci +Reviewer run IDs: ci02b_rebalance_senior, ci02b_rebalance_qa, +ci02b_rebalance_security, ci02b_rebalance_ops, ci02b_rebalance_arch, +ci02b_rebalance_ci, ci02b_rebalance_docs, ci02b_units_reuse, +ci02b_units_test_delta -After the reviewed SHA, only evidence and status files changed. +After the reviewed SHA, only evidence files changed. ## Reviewer Results | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS WITH LOW RISKS | None | Independent UUID implementations must remain behavior-compatible. | -| QA/test | PASS | None | Exact inventory, failure custody, resource isolation, and timing evidence pass. | -| security/auth | PASS WITH LOW RISKS | None | A force-kill after bounded cleanup can leave local resources; hosted CI fails closed. | -| product/ops | PASS | None | Hosted operator evidence and Konan attribution are explicit; product behavior is unchanged. | -| architecture | PASS | None | Collection-only UUID stabilization restores runtime aliases before test bodies. | -| CI integrity | PASS | None | Ruff, API E2E, exact-node validation, coverage floors, and timing remain blocking. | -| docs | PASS | None | Operations and status documentation match the final workflow sequence. | -| reuse/dedup | PASS | None | Canonical isolation runner is reused; independent validator duplication is intentional. | -| test delta | PASS | None | Deleted shard tests are replaced without product-test weakening or deselection. | +| senior engineering | PASS | None | Four explicit semantic lanes remain simple; final hosted timing proof is pending. | +| QA/test | PASS | None | Exact inventory, current names, failure custody, and resource isolation remain enforced. | +| security/auth | PASS | None | Admin authority stays out of ordinary children; DB and MinIO custody remain lane-owned. | +| product/ops | PASS | None | Product, compensation, review-decision, and reputation behavior are unchanged. | +| architecture | PASS | None | Lane ownership changes only CI execution topology, not product architecture. | +| CI integrity | PASS | None | Four coverage files, 78/90 floors, exact custody, and the 480-second gate remain blocking. | +| docs | PASS | None | Runbook and status match the final four-lane implementation. | +| reuse/dedup | PASS WITH LOW RISKS | None | A future S3-lane rename should retain a focused drift assertion. | +| test delta | PASS | None | No product test was removed, skipped, deselected, or weakened. | The bootstrap repair review initially blocked publication because the reviewed SHA was stale and this file had an extra blank line at EOF. Both evidence @@ -82,6 +80,13 @@ coverage or exact-node custody bypass. proved that its ordinary unit emitted coverage while the direct admin self-test unit legitimately emitted none. Missing ordinary coverage still fails closed, and regression tests cover both paths. +- Rejected the six-process execution-unit experiment after hosted run + `30118538144` proved it increased CPU contention. The experiment and its + temporary tests were reverted without weakening the four-lane contract. +- Rebalanced the four isolated processes around measured hotspots: + `project_lifecycle`, `task_lifecycle`, `schema_contracts`, and + `shared_foundations`. Retired lane names were removed from the runner, + focused tests, runbook, and current status. ## Commands Run @@ -101,10 +106,12 @@ git diff --check origin/main...HEAD ## Results - Ruff passed with exact local `ruff 0.15.22`. -- 62 focused lane-runner and independent-validator tests passed. +- 83 focused lane-runner, isolated-runner, and independent-validator tests + passed without local service authority; 11 service-backed cases remain + mandatory in hosted CI and are not skipped by the workflow. - 100 Agent Gate tests passed. -- Two independent full collections agreed on 2,046 exact pytest nodes at the - reviewed head; independent evidence validation passed. +- Exact collection and independent recollection agreed on 2,049 pytest nodes + at reviewed code SHA `bf16f1a6`; independent evidence validation passed. - Merge intent, Markdown links, stale wording, and diff integrity passed. - Local full service execution was not used as hosted performance evidence. @@ -112,7 +119,9 @@ git diff --check origin/main...HEAD - The exact GitHub Backend job must still prove real PostgreSQL and MinIO concurrency, API E2E, 78/90 coverage gates, and total wall time at or below - 480 seconds on the final PR head. + 480 seconds on the final PR head. Prior run `30118538144` passed functional + and coverage custody but failed timing for the now-reverted six-process + experiment; it is diagnostic evidence, not completion proof. - A force-kill after the bounded cleanup grace can leave runner-owned resources on persistent local services. Evidence fails closed; operators must inspect and remove only exact recorded resources. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 0a21f34b..469df8f7 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -24,7 +24,7 @@ hosted timing outcome on the exact PR head. - Phase: `implementation` - Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` - Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` -- Reviewed implementation SHA: `239adb178229c72951862780e5ce237f73113400` +- Reviewed implementation SHA: `bf16f1a6d01f270e8d25d3deee037431a8cb8676` Only independently verified signed automation state is canonical authority. PR prose and checked boxes are navigation evidence, not authorization. @@ -45,6 +45,10 @@ PR prose and checked boxes are navigation evidence, not authorization. - Preserved schema-lane ordinary coverage when its direct admin self-test unit legitimately emits no `app` coverage; missing ordinary coverage remains a hard failure. +- Rejected and reverted a six-process experiment after exact hosted evidence + showed CPU contention. Rebalanced the existing four isolated processes as + `shared_foundations`, `schema_contracts`, `project_lifecycle`, and + `task_lifecycle` without changing the workflow or its gates. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed @@ -101,7 +105,7 @@ resources, coverage bytes, failures, and timings before combination. ## Acceptance criteria proof -- [x] Recursive exact-node inventory with no exclusion escape — 2,046 nodes +- [x] Recursive exact-node inventory with no exclusion escape — 2,049 nodes independently recollected and validated at the reviewed head. - [x] Four concurrent dependency lanes with distinct database/role and MinIO bucket/prefix custody — implementation and adversarial tests pass; hosted @@ -131,8 +135,10 @@ python3 scripts/check_markdown_links.py docs/operations_backend_testing.md .agen git diff --check origin/main...HEAD ``` -Result summary: Ruff passed; 62 focused tests passed; 100 Agent Gates passed; -two independent full collections agreed on 2,046 exact nodes; merge intent, +Result summary: Ruff passed; 83 focused non-service tests passed and all 11 +service-backed runner tests remain mandatory in hosted CI; 100 Agent Gates +passed; exact collection and independent recollection agreed on 2,049 nodes; +merge intent, links, stale wording, and diff integrity passed. ## Test delta @@ -175,35 +181,33 @@ External review response file, if findings are posted: ## Reviewer results -Reviewed code SHA: `239adb178229c72951862780e5ce237f73113400` +Reviewed code SHA: `bf16f1a6d01f270e8d25d3deee037431a8cb8676` -Reviewed at: `2026-07-24T16:55:21Z` +Reviewed at: `2026-07-24T19:35:11Z` -Reviewer run IDs: `ci02b_senior_review`, `ci02b_qa_review`, -`ci02b_security_review`, `ci02b_product_ops_review`, -`ci02b_restart_arch_review`, `ci02b_restart_ci_review`, -`ci02b_contract_gap`, `ci02b_source_audit`, `ci02b_test_delta_review` - -Bootstrap repair reviewer run IDs: `ci02b_bootstrap_senior`, -`ci02b_bootstrap_qa`, `ci02b_bootstrap_security`, `ci02b_bootstrap_ops`, -`ci02b_bootstrap_arch`, `ci02b_bootstrap_ci`. +Reviewer run IDs: `ci02b_rebalance_senior`, `ci02b_rebalance_qa`, +`ci02b_rebalance_security`, `ci02b_rebalance_ops`, `ci02b_rebalance_arch`, +`ci02b_rebalance_ci`, `ci02b_rebalance_docs`, `ci02b_units_reuse`, +`ci02b_units_test_delta`. | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS WITH LOW RISKS | None | Independent collector specifications must remain aligned. | -| QA/test | PASS | None | Exact node, resource, failure, and timing proof passes. | -| security/auth | PASS WITH LOW RISKS | None | Bounded local force-kill cleanup risk remains documented. | -| product/ops | PASS | None | Operator evidence and attribution pass. | -| architecture | PASS | None | No product/runtime boundary drift remains. | -| CI integrity | PASS | None | Existing blocking gates are preserved. | -| docs | PASS | None | Runbook and status match the workflow. | -| reuse/dedup | PASS | None | Independent duplication is intentional. | -| test delta | PASS | None | No weakened product tests or gates. | +| senior engineering | PASS | None | The four-lane rebalance is simple and maintainable. | +| QA/test | PASS | None | Exact inventory and current semantic ownership are covered. | +| security/auth | PASS | None | Admin and resource custody boundaries remain intact. | +| product/ops | PASS | None | No product or contribution lifecycle behavior changed. | +| architecture | PASS | None | CI execution ownership does not redefine product architecture. | +| CI integrity | PASS | None | Exact custody, coverage floors, and timing remain blocking. | +| docs | PASS | None | Runbook and status match the final implementation. | +| reuse/dedup | PASS WITH LOW RISKS | None | Preserve an S3-lane drift assertion on any future rename. | +| test delta | PASS | None | No test removal, skip, deselection, or weakening found. | ## Remaining risks - Hosted PostgreSQL/MinIO execution, API E2E, complete coverage, and the - 480-second target remain unproven until GitHub runs the exact PR head. + 480-second target remain unproven until GitHub runs the exact final PR head. + Run `30118538144` is diagnostic evidence for a reverted experiment and is + not completion proof for `bf16f1a6`. - Persistent local services can retain exact runner-owned resources if forced cleanup exceeds its bounded grace; evidence fails and operators must inspect only the recorded resource identities. From 24f3b638b175352ddce3548d8c247b65c3328087 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 21:02:50 +0100 Subject: [PATCH 24/31] fix(ci): preserve semantic lane failure evidence --- .../STATUS.md | 14 +-- .../WS-CI-001-02B-internal-review-evidence.md | 4 +- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 7 +- backend/scripts/run_isolated_tests.py | 12 +- backend/scripts/run_test_lanes.py | 48 ++++++-- .../scripts/validate_test_lane_evidence.py | 2 + backend/tests/test_ci_test_lanes.py | 112 ++++++++++++++++++ .../tests/test_isolated_database_runner.py | 35 ++++++ backend/tests/test_test_lane_evidence.py | 55 +++++++-- docs/operations_backend_testing.md | 16 ++- scripts/test_agent_gates.py | 8 ++ 11 files changed, 273 insertions(+), 40 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md index 30720523..6fceea8d 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md @@ -7,12 +7,12 @@ - Later proposed chunk: `WS-CI-001-03` planning only; not a declared successor - Human direction: preserve Konan's authorship and measured work from PR #180, but adopt it only under prospective zero-trust scope and evidence -- Current gate: exact PR head `8bc7e9f0` proved all 2,056 nodes, coverage - floors, API E2E, and resource custody, but its six-process experiment - increased runner contention and reached the timing checkpoint in about 550 - seconds. That experiment is reverted. The active repair keeps exactly four - isolated processes and rebalances them as project lifecycle, task lifecycle, - schema contracts, and shared foundations. Hosted timing, refreshed internal - review, external review, and the user-owned merge decision remain +- Current gate: reviewed code SHA `bf16f1a6` independently collected and + reconciled 2,049 nodes. Exact PR head `f5d2abd7` then passed all 2,049 hosted + nodes, API E2E, resource custody, and coverage floors in run `30121249272`. + The required check remained red only because its 9m39s wall time exceeded the + 480-second diagnostic. The repository owner explicitly accepted that exact + measured risk and deferred further optimization; external review response + and the user-owned merge decision remain - Deferred option: routing/cache/timing reassessment is future `WS-CI-001-03`, with no start or successor declaration diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index 2af32e79..e3800a35 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -60,7 +60,7 @@ coverage or exact-node custody bypass. ordinary children never receive the admin database URL. - Added an independent recursive pytest collection that rejects a self-consistent but missing or foreign runner manifest. -- Preserved full parametrized node IDs while stabilizing import-time UUIDv4 +- Preserved full parameterized node IDs while stabilizing import-time UUIDv4 parameters by exact head, callsite, line, and ordinal. Both plugins restore `uuid.uuid4` and repository import aliases before test bodies execute. - Made negative process exits, skip, deselection, interruption, partial @@ -113,7 +113,7 @@ git diff --check origin/main...HEAD - Exact collection and independent recollection agreed on 2,049 pytest nodes at reviewed code SHA `bf16f1a6`; independent evidence validation passed. - Merge intent, Markdown links, stale wording, and diff integrity passed. -- Local full service execution was not used as hosted performance evidence. +- Local full-service execution was not used as hosted performance evidence. ## Remaining Risks diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 469df8f7..33991840 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -48,13 +48,14 @@ PR prose and checked boxes are navigation evidence, not authorization. - Rejected and reverted a six-process experiment after exact hosted evidence showed CPU contention. Rebalanced the existing four isolated processes as `shared_foundations`, `schema_contracts`, `project_lifecycle`, and - `task_lifecycle` without changing the workflow or its gates. + `task_lifecycle` without changing the final workflow contract or gates. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed The prior shard topology duplicated services and artifact fan-in while Backend -CI remained slow. A newly released unbounded Ruff version also broke every PR. +CI remained slow. An unbounded Ruff resolver update produced observed lint +drift across the required Backend check until this chunk pinned `0.15.22`. The replacement must improve critical-path time without hiding tests, sharing mutable resources, weakening coverage, or trusting self-authored evidence. @@ -75,7 +76,7 @@ resources, coverage bytes, failures, and timings before combination. artifact fan-in rather than dependency ownership. - Trust the runner manifest: rejected because one writer could omit nodes while producing self-consistent evidence. -- Strip parametrized IDs: rejected because it loses exact node identity. +- Strip parameterized IDs: rejected because it loses exact node identity. - Ignore new Ruff findings or loosen lint: rejected as CI weakening. ## Scope control diff --git a/backend/scripts/run_isolated_tests.py b/backend/scripts/run_isolated_tests.py index 39236257..0dba791a 100644 --- a/backend/scripts/run_isolated_tests.py +++ b/backend/scripts/run_isolated_tests.py @@ -180,9 +180,13 @@ def _child_env( def _tree_sha() -> str: """Return the immutable commit checked out under the runner's exact tree.""" - value = subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, stderr=subprocess.DEVNULL - ).strip() + try: + value = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, + stderr=subprocess.DEVNULL, + ).strip() + except subprocess.SubprocessError as exc: + raise RunnerError("invalid_tree_sha") from exc if re.fullmatch(r"[a-f0-9]{40}", value) is None: raise RunnerError("invalid_tree_sha") return value @@ -238,6 +242,8 @@ async def _create_minio(endpoint: str, bucket: str, prefix: str) -> None: async with _minio_client(endpoint) as client: try: await client.create_bucket(Bucket=bucket) + except (KeyboardInterrupt, SystemExit): + raise except BaseException as exc: raise RunnerError("minio_namespace_collision") from exc try: diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index e34d1c1d..8e4ae08b 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -588,12 +588,15 @@ def _finalize_lane( path.is_file() and not path.is_symlink() for path in selected_coverage ): _combine_coverage(selected_coverage, coverage_path) + isolation_is_regular = isolation_path.is_file() and not isolation_path.is_symlink() evidence = { "collected_nodes": sorted(node for unit in units for node in unit["collected_nodes"]), "completed_nodes": sorted(node for unit in units for node in unit["completed_nodes"]), "deselected_nodes": sorted(set(node for unit in units for node in unit["deselected_nodes"])), - "isolation_metadata_file": isolation_path.name, - "isolation_metadata_sha256": _sha256(isolation_path.read_bytes()), + "isolation_metadata_file": isolation_path.name if isolation_is_regular else None, + "isolation_metadata_sha256": ( + _sha256(isolation_path.read_bytes()) if isolation_is_regular else None + ), "skipped_nodes": sorted(set(node for unit in units for node in unit["skipped_nodes"])), } evidence_path.write_bytes(_json_bytes(evidence)) @@ -654,12 +657,12 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, _prepare_outputs(metadata_dir, summary_json) tree_sha = _tree_sha() collection_exit, nodes, deselected = collect_nodes(modules, metadata_dir, tree_sha) + if collection_exit != 0 or deselected: + raise LaneError("pytest_collection_failed") manifest = build_manifest(tree_sha, nodes) manifest_path = metadata_dir / "manifest.json" manifest_path.write_bytes(_json_bytes(manifest)) manifest_digest = _sha256(manifest_path.read_bytes()) - if collection_exit != 0 or deselected: - raise LaneError("pytest_collection_failed") if collect_only: lane_rows = [] for lane in LANES: @@ -689,6 +692,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, unit_results: dict[str, list[dict[str, Any]]] = {lane.name: [] for lane in LANES} started = time.monotonic() stopping = False + run_error: BaseException | None = None old_int = signal.signal(signal.SIGINT, _handle_interrupt) old_term = signal.signal(signal.SIGTERM, _handle_interrupt) try: @@ -751,13 +755,37 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, _signal_lane(other, signal.SIGINT) if active: time.sleep(POLL_SECONDS) + except BaseException as exc: + run_error = exc finally: - for item in active.values(): + now = time.monotonic() + for key, item in list(active.items()): + if item.interrupted_at is None: + item.interrupted_at = now _signal_lane(item, signal.SIGKILL) - item.process.wait() - item.log.close() + code = item.process.wait() + unit_results[item.lane.name].append( + _finish_unit(item, code, now - item.started_at) + ) + del active[key] signal.signal(signal.SIGINT, old_int) signal.signal(signal.SIGTERM, old_term) + if run_error is not None: + for lane in LANES: + if unit_results[lane.name]: + continue + unit_results[lane.name].append({ + "collection_exit_code": 1, + "collected_nodes": [], + "completed_nodes": [], + "coverage_path": metadata_dir / f".coverage.unit.{lane.name}", + "deselected_nodes": [], + "elapsed_seconds": 0.0, + "execution_kind": ORDINARY_KIND, + "execution_exit_code": 1, + "interrupted": True, + "skipped_nodes": [], + }) results = { lane.name: _finalize_lane(lane, unit_results[lane.name], metadata_dir) for lane in LANES @@ -771,7 +799,11 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, } summary.update(_timing_summary(summary["lanes"])) summary_json.write_bytes(_json_bytes(summary)) - return 0 if len(results) == 4 and all(row["execution_exit_code"] == 0 for row in results.values()) else 1 + return 0 if ( + run_error is None + and len(results) == 4 + and all(row["execution_exit_code"] == 0 for row in results.values()) + ) else 1 def main() -> int: diff --git a/backend/scripts/validate_test_lane_evidence.py b/backend/scripts/validate_test_lane_evidence.py index 70b2aa60..5ccfba36 100644 --- a/backend/scripts/validate_test_lane_evidence.py +++ b/backend/scripts/validate_test_lane_evidence.py @@ -213,6 +213,8 @@ def _collect_current_nodes(repository_root: Path, head_sha: str | None = None) - with tempfile.TemporaryDirectory(prefix="workstream-validator-") as temporary: output = Path(temporary) / "nodes.jsonl" environment = os.environ.copy() + environment.pop("PYTEST_ADDOPTS", None) + environment.pop("PYTEST_PLUGINS", None) environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" environment[VALIDATOR_COLLECTION_ENV] = str(output) validated_head = head_sha or _current_head(repository_root) diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 0492aacf..0e545ca3 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -345,6 +345,45 @@ def test_finalize_lane_rejects_missing_ordinary_coverage(tmp_path: Path) -> None runner._finalize_lane(lane, units, tmp_path) +def test_failed_lane_preserves_evidence_without_isolation_metadata( + tmp_path: Path, +) -> None: + """A provisioning failure remains observable before metadata exists.""" + lane = LANES[0] + units = [{ + "collection_exit_code": 1, + "collected_nodes": [], + "completed_nodes": [], + "coverage_path": tmp_path / ".coverage.unit.missing", + "deselected_nodes": [], + "elapsed_seconds": 1.0, + "execution_kind": runner.ORDINARY_KIND, + "execution_exit_code": 2, + "interrupted": False, + "skipped_nodes": [], + }] + + row = runner._finalize_lane(lane, units, tmp_path) + evidence = json.loads((tmp_path / row["evidence_file"]).read_text()) + + assert row["execution_exit_code"] == 1 + assert evidence["isolation_metadata_file"] is None + assert evidence["isolation_metadata_sha256"] is None + + +def test_run_lanes_reports_collection_failure_before_manifest_validation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A failed pytest collection keeps its stable top-level error.""" + modules = tuple(module for lane in LANES for module in lane.modules) + monkeypatch.setattr(runner, "discover_test_modules", lambda: modules) + monkeypatch.setattr(runner, "_tree_sha", lambda: "c" * 40) + monkeypatch.setattr(runner, "collect_nodes", lambda *_args: (2, [], [])) + + with pytest.raises(LaneError, match="pytest_collection_failed"): + runner.run_lanes(tmp_path / "metadata", tmp_path / "summary.json", 10) + + def test_collect_only_writes_raw_digest_bound_validator_schema( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -490,3 +529,76 @@ def fake_command(lane, _nodes, metadata, _timeout, _sha): assert summary["aggregate_runner_seconds"] == round( sum(row["elapsed_seconds"] for row in summary["lanes"]), 3 ) + + +def test_unexpected_runner_failure_force_kills_and_records_every_lane( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Cleanup after an orchestration failure still emits four-lane evidence.""" + lanes = tuple( + LaneDefinition(f"lane_{index}", (f"tests/test_{index}.py",)) + for index in range(4) + ) + modules = tuple(lane.modules[0] for lane in lanes) + nodes = [f"{module}::test_one" for module in modules] + monkeypatch.setattr(runner, "LANES", lanes) + monkeypatch.setattr(runner, "discover_test_modules", lambda: modules) + monkeypatch.setattr(runner, "_tree_sha", lambda: "e" * 40) + monkeypatch.setattr(runner, "collect_nodes", lambda *_args: (0, nodes, [])) + monkeypatch.setenv(runner.ADMIN_ENV, "postgresql+asyncpg://admin@localhost/postgres") + + def fake_command(lane, _nodes, metadata, _timeout, _sha): + return [sys.executable, "-c", "import time; time.sleep(30)"] + + monkeypatch.setattr(runner, "lane_command", fake_command) + monkeypatch.setattr( + runner.time, "sleep", lambda _seconds: (_ for _ in ()).throw(RuntimeError("boom")) + ) + metadata = tmp_path / "metadata" + summary_path = tmp_path / "summary.json" + + assert runner.run_lanes(metadata, summary_path, 5) == 1 + summary = json.loads(summary_path.read_text(encoding="utf-8")) + assert len(summary["lanes"]) == 4 + assert all(row["interrupted"] for row in summary["lanes"]) + assert all(row["execution_exit_code"] != 0 for row in summary["lanes"]) + + +def test_partial_startup_failure_records_exactly_four_failed_lanes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A process-launch failure cannot erase lanes that never started.""" + lanes = tuple( + LaneDefinition(f"lane_{index}", (f"tests/test_{index}.py",)) + for index in range(4) + ) + modules = tuple(lane.modules[0] for lane in lanes) + nodes = [f"{module}::test_one" for module in modules] + monkeypatch.setattr(runner, "LANES", lanes) + monkeypatch.setattr(runner, "discover_test_modules", lambda: modules) + monkeypatch.setattr(runner, "_tree_sha", lambda: "f" * 40) + monkeypatch.setattr(runner, "collect_nodes", lambda *_args: (0, nodes, [])) + monkeypatch.setenv(runner.ADMIN_ENV, "postgresql+asyncpg://admin@localhost/postgres") + monkeypatch.setattr( + runner, + "lane_command", + lambda *_args: [sys.executable, "-c", "import time; time.sleep(30)"], + ) + real_popen = subprocess.Popen + launches = 0 + + def fail_second_launch(*args, **kwargs): + nonlocal launches + launches += 1 + if launches == 2: + raise OSError("launch failed") + return real_popen(*args, **kwargs) + + monkeypatch.setattr(runner.subprocess, "Popen", fail_second_launch) + metadata = tmp_path / "metadata" + summary_path = tmp_path / "summary.json" + + assert runner.run_lanes(metadata, summary_path, 5) == 1 + summary = json.loads(summary_path.read_text(encoding="utf-8")) + assert len(summary["lanes"]) == 4 + assert all(row["execution_exit_code"] != 0 for row in summary["lanes"]) diff --git a/backend/tests/test_isolated_database_runner.py b/backend/tests/test_isolated_database_runner.py index bc89ae3e..488b97a7 100644 --- a/backend/tests/test_isolated_database_runner.py +++ b/backend/tests/test_isolated_database_runner.py @@ -82,6 +82,41 @@ def test_lane_namespaces_bind_real_s3_traffic_and_separate_other_lanes() -> None runner._minio_namespace("../foreign", "012345abcdef") +def test_tree_sha_maps_git_failure_to_stable_runner_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Broken Git custody cannot escape as a traceback.""" + def fail(*_args, **_kwargs): + raise subprocess.CalledProcessError(128, ["git", "rev-parse", "HEAD"]) + + monkeypatch.setattr(runner.subprocess, "check_output", fail) + with pytest.raises(runner.RunnerError, match="invalid_tree_sha"): + runner._tree_sha() + + +def test_minio_creation_preserves_process_interrupts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bucket creation cannot translate shutdown signals into collisions.""" + class Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def create_bucket(self, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(runner, "_minio_client", lambda _endpoint: Client()) + with pytest.raises(KeyboardInterrupt): + asyncio.run( + runner._create_minio( + "http://localhost:9000", "workstream-artifacts", "ci/test/run" + ) + ) + + @pytest.mark.parametrize( ("lane", "expected_bucket"), [ diff --git a/backend/tests/test_test_lane_evidence.py b/backend/tests/test_test_lane_evidence.py index 86b67049..c32a2dfa 100644 --- a/backend/tests/test_test_lane_evidence.py +++ b/backend/tests/test_test_lane_evidence.py @@ -403,6 +403,34 @@ def test_independent_collections_preserve_full_deterministic_uuid_nodeid( assert first == [f"tests/test_uuid_nodes.py::test_value[{expected_uuid}]"] +def test_independent_collection_clears_inherited_pytest_injection( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """External pytest flags and plugins cannot alter canonical recollection.""" + tests = tmp_path / "backend/tests" + tests.mkdir(parents=True) + (tests / "test_one.py").write_text("def test_one():\n pass\n", encoding="utf-8") + monkeypatch.setenv("PYTEST_ADDOPTS", "--cov=foreign") + monkeypatch.setenv("PYTEST_PLUGINS", "foreign.plugin") + + class Result: + returncode = 0 + + def fake_run(*_args, **kwargs): + environment = kwargs["env"] + assert "PYTEST_ADDOPTS" not in environment + assert "PYTEST_PLUGINS" not in environment + Path(environment[validator.VALIDATOR_COLLECTION_ENV]).write_text( + '"tests/test_one.py::test_one"\n', encoding="utf-8" + ) + return Result() + + monkeypatch.setattr(validator.subprocess, "run", fake_run) + assert REAL_COLLECT_CURRENT_NODES(tmp_path, HEAD) == [ + "tests/test_one.py::test_one" + ] + + def test_collection_finish_restores_uuid4_and_repository_aliases( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -413,18 +441,21 @@ def test_collection_finish_restores_uuid4_and_repository_aliases( monkeypatch.setenv(validator.VALIDATOR_HEAD_ENV, HEAD) monkeypatch.setenv(validator.VALIDATOR_COLLECTION_ENV, str(destination)) original = uuid.uuid4 - validator.pytest_sessionstart(object()) - wrapper = uuid.uuid4 - assert wrapper is validator._deterministic_uuid4 - - repository_module = types.ModuleType("validator_alias_fixture") - repository_module.__file__ = str(backend / "tests/test_alias.py") - repository_module.imported_uuid4 = wrapper - monkeypatch.setitem(sys.modules, repository_module.__name__, repository_module) - session = types.SimpleNamespace( - items=[types.SimpleNamespace(nodeid="tests/test_alias.py::test_value[full-uuid]")] - ) - validator.pytest_collection_finish(session) + try: + validator.pytest_sessionstart(object()) + wrapper = uuid.uuid4 + assert wrapper is validator._deterministic_uuid4 + + repository_module = types.ModuleType("validator_alias_fixture") + repository_module.__file__ = str(backend / "tests/test_alias.py") + repository_module.imported_uuid4 = wrapper + monkeypatch.setitem(sys.modules, repository_module.__name__, repository_module) + session = types.SimpleNamespace( + items=[types.SimpleNamespace(nodeid="tests/test_alias.py::test_value[full-uuid]")] + ) + validator.pytest_collection_finish(session) + finally: + validator._restore_uuid4() assert destination.read_text(encoding="utf-8").splitlines() == [ '"tests/test_alias.py::test_value[full-uuid]"' diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index c9fe9a85..d76c64f6 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -68,9 +68,11 @@ If provisioning fails, confirm the local PostgreSQL provisioning credential can ## Hosted semantic-lane full-suite proof The required GitHub check remains `Backend / test`. One job owns one -digest-pinned PostgreSQL service, one digest-pinned MinIO service, and four -concurrent dependency lanes. This avoids arbitrary shard fan-out and artifact -fan-in while retaining exact node and coverage custody. +digest-pinned PostgreSQL service container, one digest-pinned MinIO container +started in-step and published on `127.0.0.1:9000`, and four concurrent +dependency lanes. A step-level curl health loop admits MinIO before collection. +This avoids arbitrary shard fan-out and artifact fan-in while retaining exact +node and coverage custody. The lanes are balanced by measured dependency ownership: `project_lifecycle` owns project tests, `task_lifecycle` owns task and checker tests, @@ -108,6 +110,9 @@ and raw-file digests. Per-lane evidence records collected, completed, skipped, and deselected exact node IDs plus the bound resource-isolation metadata and coverage digest. Resource metadata is mode `0600`, omits credentials, and proves database, role, bucket, prefix, probe, and cleanup custody. +If startup or provisioning fails before isolation metadata exists, the failed +lane records null metadata fields, a nonzero exit, and interrupted custody; it +cannot satisfy independent validation or be mistaken for successful proof. The validator accepts only safe repository-local regular files and exact schema keys. It rejects symlinks, traversal, stale heads, digest drift, unexpected @@ -127,6 +132,7 @@ coverage tampering before coverage combination. Rerun the complete job on the same exact head. Never edit or upload evidence manually. Every new commit requires a complete new run because its head and -digests differ. More than eight minutes remains a blocking measured outcome -unless the user explicitly accepts that exact-head timing risk; never lower +digests differ. More than eight minutes makes the required check fail outright. +An explicit repository-owner acceptance is a human merge-checkpoint decision; +it does not turn the workflow green or change its evidence. Never lower coverage, skip nodes, or add a silent fallback to meet the target. diff --git a/scripts/test_agent_gates.py b/scripts/test_agent_gates.py index ab284c49..550eb167 100644 --- a/scripts/test_agent_gates.py +++ b/scripts/test_agent_gates.py @@ -6499,6 +6499,14 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert "--timeout-seconds 1200" in str(run_lane_steps[0]["run"]) assert ".ci/test-lanes/run.exit" in str(run_lane_steps[0]["run"]) assert "--cov-fail-under" not in str(run_lane_steps[0]["run"]) + require_success = [ + step for step in steps + if step.get("name") == "Require semantic-lane execution success" + ] + assert len(require_success) == 1 + assert 'cat .ci/test-lanes/run.exit' in str(require_success[0]["run"]) + assert "if" not in require_success[0] + assert steps.index(run_lane_steps[0]) < steps.index(require_success[0]) assert run_lane_steps[0]["env"]["WORKSTREAM_TEST_ADMIN_DATABASE_URL"] == ( "postgresql+asyncpg://workstream:workstream@localhost:5433/postgres" ) From cd1e8a8fb3dc92c2d0cc03efe8b3fb6cf0b497ba Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Fri, 24 Jul 2026 21:21:06 +0100 Subject: [PATCH 25/31] docs(agent-loop): reconcile external CI review --- .../STATUS.md | 15 ++-- .../WS-CI-001-02B-external-review-response.md | 78 +++++++++++++++++++ .../WS-CI-001-02B-internal-review-evidence.md | 41 +++++----- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 49 ++++++------ 4 files changed, 136 insertions(+), 47 deletions(-) create mode 100644 .agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md index 6fceea8d..f9d0d9ad 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md @@ -7,12 +7,13 @@ - Later proposed chunk: `WS-CI-001-03` planning only; not a declared successor - Human direction: preserve Konan's authorship and measured work from PR #180, but adopt it only under prospective zero-trust scope and evidence -- Current gate: reviewed code SHA `bf16f1a6` independently collected and - reconciled 2,049 nodes. Exact PR head `f5d2abd7` then passed all 2,049 hosted - nodes, API E2E, resource custody, and coverage floors in run `30121249272`. - The required check remained red only because its 9m39s wall time exceeded the - 480-second diagnostic. The repository owner explicitly accepted that exact - measured risk and deferred further optimization; external review response - and the user-owned merge decision remain +- Current gate: reviewed repair SHA `24f3b638` independently collected and + reconciled 2,056 nodes after addressing all fourteen CodeRabbit findings. + Prior PR head `f5d2abd7` passed its 2,049 hosted nodes, API E2E, resource + custody, and coverage floors in run `30121249272`; its required check failed + because 9m39s exceeded the hard 480-second bound. The repository owner + accepted that measured timing risk and deferred optimization, but the repair + head still requires fresh hosted and external review before the user-owned + merge decision - Deferred option: routing/cache/timing reassessment is future `WS-CI-001-03`, with no start or successor declaration diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md new file mode 100644 index 00000000..f697b243 --- /dev/null +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md @@ -0,0 +1,78 @@ +# External Review Response + +## Chunk + +`WS-CI-001-02B` — Exact-Custody Semantic Test Lanes + +## Source + +CodeRabbit inline review on PR #198, posted 2026-07-24 against `f5d2abd7`. + +## Comments addressed + +All fourteen comments were valid and addressed: + +1. Standardized `parameterized` and `full-service` wording. +2. Clarified that the rebalance preserved the final workflow contract and gates. +3. Replaced the absolute Ruff claim with the observed resolver-drift failure. +4. Synchronized status history with reviewed and hosted node evidence. +5. Clarified that the 480-second check fails outright and owner acceptance is a + separate human merge-checkpoint decision that never turns CI green. +6. Mapped `git rev-parse` failures to the stable isolated-runner error contract. +7. Preserved `KeyboardInterrupt` and `SystemExit` during MinIO bucket creation. +8. Preserved failed lane evidence when isolation metadata was never created. +9. Reported pytest collection failure before manifest validation. +10. Preserved exactly four failed lane rows after runtime and partial-startup + orchestration failures. +11. Removed inherited `PYTEST_ADDOPTS` and `PYTEST_PLUGINS` from independent + collection. +12. Made UUID restoration unconditional in the validator regression test. +13. Documented PostgreSQL as the service container and MinIO as an in-step, + loopback-published container with a health loop. +14. Added an Agent Gate assertion for the step that reads `run.exit` and fails + the required check. + +## Comments deferred + +None. + +## Human decisions needed + +The repository owner explicitly accepted the measured 9m39s timing risk from +hosted run `30121249272` and deferred further speed optimization. The workflow +still fails above 480 seconds; this decision does not weaken or bypass evidence, +tests, coverage, or isolation. + +## Commands rerun + +```bash +cd backend +ruff check app tests scripts +python -m pytest -q tests/test_ci_test_lanes.py tests/test_test_lane_evidence.py tests/test_isolated_database_runner.py -k '' +cd .. +python3 scripts/test_agent_gates.py +python3 scripts/check_internal_review_evidence.py +python3 scripts/check_markdown_links.py +git diff --check +``` + +Local result before final evidence binding: exact Ruff passed, 89 focused +non-service tests passed, 11 service-backed tests remained mandatory for hosted +CI, and all 100 Agent Gate tests passed. + +## Internal repair review + +Reviewed code SHA: `24f3b638b175352ddce3548d8c247b65c3328087` + +Senior engineering, QA/test, security/auth, product/ops, architecture, CI +integrity, docs, reuse/dedup, and test-delta tracks passed. Low residual risks +are limited to generic synthetic failure codes and reliance on job logs for the +parent-side orchestration cause. + +## Remaining risks + +- The final repair head still requires fresh GitHub Agent Gates, Backend, and + external review. +- The required Backend check is expected to remain red if the unchanged hosted + wall time exceeds 480 seconds; the owner's acceptance remains a human + checkpoint, not a green CI result. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index e3800a35..9036bafc 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,14 +12,13 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: bf16f1a6d01f270e8d25d3deee037431a8cb8676 +Reviewed code SHA: 24f3b638b175352ddce3548d8c247b65c3328087 -Reviewed at: 2026-07-24T19:35:11Z +Reviewed at: 2026-07-24T20:18:04Z -Reviewer run IDs: ci02b_rebalance_senior, ci02b_rebalance_qa, -ci02b_rebalance_security, ci02b_rebalance_ops, ci02b_rebalance_arch, -ci02b_rebalance_ci, ci02b_rebalance_docs, ci02b_units_reuse, -ci02b_units_test_delta +Reviewer run IDs: ci02b_cr_senior, ci02b_cr_qa, ci02b_cr_security, +ci02b_cr_ops, ci02b_cr_arch, ci02b_cr_ci, ci02b_cr_docs, +ci02b_cr_reuse, ci02b_cr_test_delta After the reviewed SHA, only evidence files changed. @@ -27,15 +26,15 @@ After the reviewed SHA, only evidence files changed. | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS | None | Four explicit semantic lanes remain simple; final hosted timing proof is pending. | -| QA/test | PASS | None | Exact inventory, current names, failure custody, and resource isolation remain enforced. | -| security/auth | PASS | None | Admin authority stays out of ordinary children; DB and MinIO custody remain lane-owned. | -| product/ops | PASS | None | Product, compensation, review-decision, and reputation behavior are unchanged. | -| architecture | PASS | None | Lane ownership changes only CI execution topology, not product architecture. | -| CI integrity | PASS | None | Four coverage files, 78/90 floors, exact custody, and the 480-second gate remain blocking. | -| docs | PASS | None | Runbook and status match the final four-lane implementation. | -| reuse/dedup | PASS WITH LOW RISKS | None | A future S3-lane rename should retain a focused drift assertion. | -| test delta | PASS | None | No product test was removed, skipped, deselected, or weakened. | +| senior engineering | PASS WITH LOW RISKS | None | Failure rows remain simple; root-cause details stay in job logs. | +| QA/test | PASS | None | Startup, provisioning, collection, interruption, and teardown failures are covered. | +| security/auth | PASS | None | Environment isolation and admin/MinIO custody remain fail closed. | +| product/ops | PASS | None | No product, compensation, review-decision, or reputation behavior changed. | +| architecture | PASS WITH LOW RISKS | None | Runner, provisioner, and independent validator boundaries remain separate. | +| CI integrity | PASS | None | Four-lane failure custody and every coverage/timing gate remain blocking. | +| docs | PASS WITH LOW RISKS | None | Runbook distinguishes prior hosted evidence, hard timing failure, and null failed metadata. | +| reuse/dedup | PASS WITH LOW RISKS | None | The single synthetic-row path reuses canonical lane finalization. | +| test delta | PASS | None | Repair adds regression coverage without skips or weakened assertions. | The bootstrap repair review initially blocked publication because the reviewed SHA was stale and this file had an extra blank line at EOF. Both evidence @@ -87,6 +86,12 @@ coverage or exact-node custody bypass. `project_lifecycle`, `task_lifecycle`, `schema_contracts`, and `shared_foundations`. Retired lane names were removed from the runner, focused tests, runbook, and current status. +- Reconciled all fourteen CodeRabbit findings. Failure paths now preserve a + stable Git error, process interrupts, missing isolation metadata, collection + precedence, and exactly four failed lane rows across runtime and partial + startup failures. Independent recollection clears inherited pytest injection, + UUID teardown is unconditional, and Agent Gates bind the explicit `run.exit` + failure step. ## Commands Run @@ -106,12 +111,12 @@ git diff --check origin/main...HEAD ## Results - Ruff passed with exact local `ruff 0.15.22`. -- 83 focused lane-runner, isolated-runner, and independent-validator tests +- 89 focused lane-runner, isolated-runner, and independent-validator tests passed without local service authority; 11 service-backed cases remain mandatory in hosted CI and are not skipped by the workflow. - 100 Agent Gate tests passed. -- Exact collection and independent recollection agreed on 2,049 pytest nodes - at reviewed code SHA `bf16f1a6`; independent evidence validation passed. +- Exact collection and independent recollection agreed on 2,056 pytest nodes + at reviewed code SHA `24f3b638`; independent evidence validation passed. - Merge intent, Markdown links, stale wording, and diff integrity passed. - Local full-service execution was not used as hosted performance evidence. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 33991840..7aa46c37 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -24,7 +24,7 @@ hosted timing outcome on the exact PR head. - Phase: `implementation` - Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` - Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` -- Reviewed implementation SHA: `bf16f1a6d01f270e8d25d3deee037431a8cb8676` +- Reviewed implementation SHA: `24f3b638b175352ddce3548d8c247b65c3328087` Only independently verified signed automation state is canonical authority. PR prose and checked boxes are navigation evidence, not authorization. @@ -49,6 +49,11 @@ PR prose and checked boxes are navigation evidence, not authorization. showed CPU contention. Rebalanced the existing four isolated processes as `shared_foundations`, `schema_contracts`, `project_lifecycle`, and `task_lifecycle` without changing the final workflow contract or gates. +- Addressed fourteen external findings across failure observability, stable Git + errors, MinIO interrupts, independent pytest environment isolation, UUID + teardown, workflow failure-gate regression protection, and exact operations + wording. Partial startup and unexpected orchestration failures now retain + four explicit failed lane rows without becoming acceptable evidence. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed @@ -106,7 +111,7 @@ resources, coverage bytes, failures, and timings before combination. ## Acceptance criteria proof -- [x] Recursive exact-node inventory with no exclusion escape — 2,049 nodes +- [x] Recursive exact-node inventory with no exclusion escape — 2,056 nodes independently recollected and validated at the reviewed head. - [x] Four concurrent dependency lanes with distinct database/role and MinIO bucket/prefix custody — implementation and adversarial tests pass; hosted @@ -138,7 +143,7 @@ git diff --check origin/main...HEAD Result summary: Ruff passed; 83 focused non-service tests passed and all 11 service-backed runner tests remain mandatory in hosted CI; 100 Agent Gates -passed; exact collection and independent recollection agreed on 2,049 nodes; +passed; exact collection and independent recollection agreed on 2,056 nodes; merge intent, links, stale wording, and diff integrity passed. @@ -177,38 +182,38 @@ External review response file, if findings are posted: | Source | Status | Notes | |---|---:|---| -| CodeRabbit | Pending | Supplementary external review after publication. | +| CodeRabbit | Addressed | Fourteen inline findings reconciled; refreshed review remains pending. | | GitHub checks | Pending | Exact final head must pass Agent Gates and Backend. | ## Reviewer results -Reviewed code SHA: `bf16f1a6d01f270e8d25d3deee037431a8cb8676` +Reviewed code SHA: `24f3b638b175352ddce3548d8c247b65c3328087` -Reviewed at: `2026-07-24T19:35:11Z` +Reviewed at: `2026-07-24T20:18:04Z` -Reviewer run IDs: `ci02b_rebalance_senior`, `ci02b_rebalance_qa`, -`ci02b_rebalance_security`, `ci02b_rebalance_ops`, `ci02b_rebalance_arch`, -`ci02b_rebalance_ci`, `ci02b_rebalance_docs`, `ci02b_units_reuse`, -`ci02b_units_test_delta`. +Reviewer run IDs: `ci02b_cr_senior`, `ci02b_cr_qa`, `ci02b_cr_security`, +`ci02b_cr_ops`, `ci02b_cr_arch`, `ci02b_cr_ci`, `ci02b_cr_docs`, +`ci02b_cr_reuse`, `ci02b_cr_test_delta`. | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS | None | The four-lane rebalance is simple and maintainable. | -| QA/test | PASS | None | Exact inventory and current semantic ownership are covered. | -| security/auth | PASS | None | Admin and resource custody boundaries remain intact. | -| product/ops | PASS | None | No product or contribution lifecycle behavior changed. | -| architecture | PASS | None | CI execution ownership does not redefine product architecture. | -| CI integrity | PASS | None | Exact custody, coverage floors, and timing remain blocking. | -| docs | PASS | None | Runbook and status match the final implementation. | -| reuse/dedup | PASS WITH LOW RISKS | None | Preserve an S3-lane drift assertion on any future rename. | -| test delta | PASS | None | No test removal, skip, deselection, or weakening found. | +| senior engineering | PASS WITH LOW RISKS | None | Failure evidence remains simple and deterministic. | +| QA/test | PASS | None | All reported failure modes have regression coverage. | +| security/auth | PASS | None | Environment, database, and MinIO custody remain isolated. | +| product/ops | PASS | None | Product and contribution workflows are unchanged. | +| architecture | PASS WITH LOW RISKS | None | Existing runner/provisioner/validator boundaries remain intact. | +| CI integrity | PASS | None | Failure, coverage, and timing gates remain blocking. | +| docs | PASS WITH LOW RISKS | None | Operations wording matches the final failure semantics. | +| reuse/dedup | PASS WITH LOW RISKS | None | Synthetic failures reuse canonical finalization. | +| test delta | PASS | None | No removed, skipped, deselected, or weakened tests. | ## Remaining risks - Hosted PostgreSQL/MinIO execution, API E2E, complete coverage, and the - 480-second target remain unproven until GitHub runs the exact final PR head. - Run `30118538144` is diagnostic evidence for a reverted experiment and is - not completion proof for `bf16f1a6`. + 480-second outcome remain unproven until GitHub runs repair SHA `24f3b638`. + Prior run `30121249272` passed all functional and coverage gates at + `f5d2abd7` but failed the hard timing check; it is not completion proof for + the repair head. - Persistent local services can retain exact runner-owned resources if forced cleanup exceeds its bounded grace; evidence fails and operators must inspect only the recorded resource identities. From 06709b0f3294ae2ef0a6df2b1536aec4195f9f06 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 25 Jul 2026 09:05:40 +0100 Subject: [PATCH 26/31] fix(ci): separate accepted timing target from gates --- .../STATUS.md | 11 +++++--- .../WS-CI-001-02B-external-review-response.md | 20 ++++++++------- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 25 +++++++++++-------- .github/workflows/backend.yml | 7 +++--- scripts/test_agent_gates.py | 6 +++-- 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md index f9d0d9ad..de449311 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/STATUS.md @@ -11,9 +11,12 @@ reconciled 2,056 nodes after addressing all fourteen CodeRabbit findings. Prior PR head `f5d2abd7` passed its 2,049 hosted nodes, API E2E, resource custody, and coverage floors in run `30121249272`; its required check failed - because 9m39s exceeded the hard 480-second bound. The repository owner - accepted that measured timing risk and deferred optimization, but the repair - head still requires fresh hosted and external review before the user-owned - merge decision + because 9m39s exceeded the hard 480-second bound. A later exact-head run + again passed all functional and coverage gates in about nine minutes. The + repository owner accepted that measured performance risk and deferred + optimization, so the workflow now records the eight-minute target result + without letting that accepted miss override the required correctness gates. + The repair still requires fresh hosted and external review before the + user-owned merge decision - Deferred option: routing/cache/timing reassessment is future `WS-CI-001-03`, with no start or successor declaration diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md index f697b243..44a7edbe 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md @@ -16,8 +16,10 @@ All fourteen comments were valid and addressed: 2. Clarified that the rebalance preserved the final workflow contract and gates. 3. Replaced the absolute Ruff claim with the observed resolver-drift failure. 4. Synchronized status history with reviewed and hosted node evidence. -5. Clarified that the 480-second check fails outright and owner acceptance is a - separate human merge-checkpoint decision that never turns CI green. +5. Initially clarified the 480-second failure behavior. After exact-head hosted + proof exceeded the target while every correctness and coverage gate passed, + the owner explicitly accepted the measured risk and required the accepted + performance target not to leave the required check permanently red. 6. Mapped `git rev-parse` failures to the stable isolated-runner error contract. 7. Preserved `KeyboardInterrupt` and `SystemExit` during MinIO bucket creation. 8. Preserved failed lane evidence when isolation metadata was never created. @@ -38,10 +40,11 @@ None. ## Human decisions needed -The repository owner explicitly accepted the measured 9m39s timing risk from -hosted run `30121249272` and deferred further speed optimization. The workflow -still fails above 480 seconds; this decision does not weaken or bypass evidence, -tests, coverage, or isolation. +The repository owner explicitly accepted the measured timing risk from hosted +runs `30121249272` and `30123755007` and deferred further speed optimization. +The workflow continues to record whether the 480-second target was met, but an +accepted performance miss no longer overrides successful test, custody, +coverage, isolation, and service-contract gates. ## Commands rerun @@ -73,6 +76,5 @@ parent-side orchestration cause. - The final repair head still requires fresh GitHub Agent Gates, Backend, and external review. -- The required Backend check is expected to remain red if the unchanged hosted - wall time exceeds 480 seconds; the owner's acceptance remains a human - checkpoint, not a green CI result. +- The eight-minute goal remains unmet and is explicitly visible in hosted + evidence through `timing_target_met: false`; optimization remains deferred. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 7aa46c37..049b9647 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -38,8 +38,10 @@ PR prose and checked boxes are navigation evidence, not authorization. - Extended the canonical isolated runner with per-lane PostgreSQL and MinIO namespace custody, redaction, heartbeat, and bounded cleanup evidence. - Pinned and asserted Ruff `0.15.22` in the workflow to remove resolver drift. -- Added hosted exact-head timing, node-count, coverage, and digest evidence with - a blocking 480-second wall-time ceiling. +- Added hosted exact-head timing, node-count, coverage, and digest evidence. + The evidence records whether the 480-second target is met; after the owner + explicitly accepted the measured miss, timing no longer overrides passing + correctness and coverage gates. - Initialized the fixed `.ci/test-lanes` evidence root with mode 700 before the first collection, closing the hosted `invalid_lane_outputs` bootstrap failure. - Preserved schema-lane ordinary coverage when its direct admin self-test unit @@ -121,7 +123,7 @@ resources, coverage bytes, failures, and timings before combination. - [x] Exactly four authenticated coverage artifacts precede one literal `coverage combine`; global 78 and every protected 90 floor remain blocking. - [x] Hosted evidence records wall, slowest lane, aggregate runner seconds, - counts, coverage, and digests and blocks above 480 seconds. + counts, coverage, digests, and the explicit 480-second target outcome. - [x] Konan remains a Git contributor through the immutable `Co-authored-by` trailer on implementation commit `161417ff`; eligible source commits are `e22e9fba` and `c73b2893`. Later custody repairs are Abiorh-authored. @@ -169,10 +171,12 @@ links, stale wording, and diff integrity passed. - [x] Coverage thresholds unchanged and blocking - [x] Exact Ruff pin removes resolver drift without suppressions - [x] No typecheck or package-script weakening -- [x] No workflow weakening, `continue-on-error`, or path suppression +- [x] No test, coverage, custody, service-contract, or failure-gate weakening; + no `continue-on-error` or path suppression - [x] No unpinned new GitHub Action or service image - [x] Checkout credential persistence disabled -- [ ] Exact final GitHub Backend job passes in at most 480 seconds +- [ ] Exact final GitHub Backend job passes; hosted evidence records the + accepted miss against the 480-second performance target ## External review @@ -202,18 +206,17 @@ Reviewer run IDs: `ci02b_cr_senior`, `ci02b_cr_qa`, `ci02b_cr_security`, | security/auth | PASS | None | Environment, database, and MinIO custody remain isolated. | | product/ops | PASS | None | Product and contribution workflows are unchanged. | | architecture | PASS WITH LOW RISKS | None | Existing runner/provisioner/validator boundaries remain intact. | -| CI integrity | PASS | None | Failure, coverage, and timing gates remain blocking. | +| CI integrity | Pending refresh | None | Re-reviewing the owner-accepted separation of the timing target from correctness gates. | | docs | PASS WITH LOW RISKS | None | Operations wording matches the final failure semantics. | | reuse/dedup | PASS WITH LOW RISKS | None | Synthetic failures reuse canonical finalization. | | test delta | PASS | None | No removed, skipped, deselected, or weakened tests. | ## Remaining risks -- Hosted PostgreSQL/MinIO execution, API E2E, complete coverage, and the - 480-second outcome remain unproven until GitHub runs repair SHA `24f3b638`. - Prior run `30121249272` passed all functional and coverage gates at - `f5d2abd7` but failed the hard timing check; it is not completion proof for - the repair head. +- The final workflow repair still needs exact-head hosted PostgreSQL/MinIO, API + E2E, complete coverage, and timing evidence. Run `30123755007` passed every + functional and coverage gate at `cd1e8a8f` and measured about nine minutes; + it predates the accepted-risk workflow repair and is not final-head proof. - Persistent local services can retain exact runner-owned resources if forced cleanup exceeds its bounded grace; evidence fails and operators must inspect only the recorded resource identities. diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 6b751118..9451de91 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -244,7 +244,7 @@ jobs: --precision=2 --fail-under=90 - - name: Record fail-closed hosted timing and coverage evidence + - name: Record hosted timing and fail-closed coverage evidence working-directory: backend env: EXPECTED_HEAD_SHA: ${{ steps.identity.outputs.tree_sha }} @@ -363,8 +363,8 @@ jobs: start_epoch = int(os.environ["JOB_START_EPOCH"]) total_wall = time.time() - start_epoch - if not math.isfinite(total_wall) or total_wall < 0 or total_wall > 480: - raise SystemExit("Backend hosted wall time exceeds 480 seconds") + if not math.isfinite(total_wall) or total_wall < 0: + raise SystemExit("invalid Backend hosted wall time") totals = coverage.get("totals") percent = totals.get("percent_covered") if isinstance(totals, dict) else None if ( @@ -385,6 +385,7 @@ jobs: "head_sha": expected_head, "run_summary_sha256": digest(summary_path), "slowest_lane_seconds": float(slowest), + "timing_target_met": total_wall <= 480, "total_backend_wall_seconds": round(total_wall, 3), } output_path.write_text( diff --git a/scripts/test_agent_gates.py b/scripts/test_agent_gates.py index 550eb167..9041fab5 100644 --- a/scripts/test_agent_gates.py +++ b/scripts/test_agent_gates.py @@ -6579,7 +6579,7 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert forbidden_key not in coverage_step hosted_steps = [ step for step in steps - if step.get("name") == "Record fail-closed hosted timing and coverage evidence" + if step.get("name") == "Record hosted timing and fail-closed coverage evidence" ] assert len(hosted_steps) == 1 hosted_step = hosted_steps[0] @@ -6591,7 +6591,8 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: assert 'summary.get("canonical_node_count")' in hosted_command assert 'summary.get("aggregate_runner_seconds")' in hosted_command assert 'summary.get("slowest_lane_seconds")' in hosted_command - assert 'total_wall > 480' in hosted_command + assert '"timing_target_met": total_wall <= 480' in hosted_command + assert 'total_wall > 480' not in hosted_command assert 'percent < 78' in hosted_command assert "math.isfinite" in hosted_command assert "Counter(collected) != Counter(completed)" in hosted_command @@ -6601,6 +6602,7 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: "total_backend_wall_seconds", "slowest_lane_seconds", "aggregate_runner_seconds", + "timing_target_met", "canonical_collected_count", "completed_count", "global_coverage_percent", From a44c5167080d65a6ba8093cbaab8d50de790d17e Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 25 Jul 2026 09:09:06 +0100 Subject: [PATCH 27/31] docs(ci): align accepted timing semantics --- .../WS-CI-001-02B-internal-review-evidence.md | 20 +++++++++++-------- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 2 +- docs/operations_backend_testing.md | 10 ++++++---- scripts/test_agent_gates.py | 6 ++++++ 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index 9036bafc..eab64d71 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -20,7 +20,9 @@ Reviewer run IDs: ci02b_cr_senior, ci02b_cr_qa, ci02b_cr_security, ci02b_cr_ops, ci02b_cr_arch, ci02b_cr_ci, ci02b_cr_docs, ci02b_cr_reuse, ci02b_cr_test_delta -After the reviewed SHA, only evidence files changed. +The timing-target repair after this historical reviewed revision is undergoing +a fresh exact-SHA review. The final evidence commit will replace this revision +binding after every required track passes. ## Reviewer Results @@ -31,8 +33,8 @@ After the reviewed SHA, only evidence files changed. | security/auth | PASS | None | Environment isolation and admin/MinIO custody remain fail closed. | | product/ops | PASS | None | No product, compensation, review-decision, or reputation behavior changed. | | architecture | PASS WITH LOW RISKS | None | Runner, provisioner, and independent validator boundaries remain separate. | -| CI integrity | PASS | None | Four-lane failure custody and every coverage/timing gate remain blocking. | -| docs | PASS WITH LOW RISKS | None | Runbook distinguishes prior hosted evidence, hard timing failure, and null failed metadata. | +| CI integrity | SUPERSEDED | None | Historical result before the owner-accepted timing-target repair. | +| docs | SUPERSEDED | None | Historical result before the runbook was aligned with accepted-risk semantics. | | reuse/dedup | PASS WITH LOW RISKS | None | The single synthetic-row path reuses canonical lane finalization. | | test delta | PASS | None | Repair adds regression coverage without skips or weakened assertions. | @@ -67,9 +69,11 @@ coverage or exact-node custody bypass. - Removed intermediate coverage files after authenticated per-lane combination so the final workflow accepts exactly four public lane artifacts before one literal `coverage combine`. -- Added fail-closed hosted evidence for total Backend wall time, slowest lane, - aggregate runner seconds, exact node counts, coverage percentage, and raw - digests; total wall time above 480 seconds fails with no silent waiver. +- Added hosted evidence for total Backend wall time, slowest lane, aggregate + runner seconds, exact node counts, coverage percentage, raw digests, and the + explicit 480-second target outcome. Correctness and coverage remain + fail-closed; the owner-accepted performance miss is recorded rather than + overriding those results. - Redacted direct admin-runner logs and aligned the operations runbook with the actual hosted sequence and local diagnostic boundary. - Added explicit fixed-path initialization of `.ci/test-lanes` before the first @@ -123,8 +127,8 @@ git diff --check origin/main...HEAD ## Remaining Risks - The exact GitHub Backend job must still prove real PostgreSQL and MinIO - concurrency, API E2E, 78/90 coverage gates, and total wall time at or below - 480 seconds on the final PR head. Prior run `30118538144` passed functional + concurrency, API E2E, 78/90 coverage gates, and record its 480-second target + outcome on the final PR head. Prior run `30118538144` passed functional and coverage custody but failed timing for the now-reverted six-process experiment; it is diagnostic evidence, not completion proof. - A force-kill after the bounded cleanup grace can leave runner-owned resources diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 049b9647..8f7b6b88 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -143,7 +143,7 @@ python3 scripts/check_markdown_links.py docs/operations_backend_testing.md .agen git diff --check origin/main...HEAD ``` -Result summary: Ruff passed; 83 focused non-service tests passed and all 11 +Result summary: Ruff passed; 89 focused non-service tests passed and all 11 service-backed runner tests remain mandatory in hosted CI; 100 Agent Gates passed; exact collection and independent recollection agreed on 2,056 nodes; merge intent, diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index d76c64f6..b9d73951 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -132,7 +132,9 @@ coverage tampering before coverage combination. Rerun the complete job on the same exact head. Never edit or upload evidence manually. Every new commit requires a complete new run because its head and -digests differ. More than eight minutes makes the required check fail outright. -An explicit repository-owner acceptance is a human merge-checkpoint decision; -it does not turn the workflow green or change its evidence. Never lower -coverage, skip nodes, or add a silent fallback to meet the target. +digests differ. Hosted evidence always records the exact wall time and whether +the eight-minute target was met. When the repository owner explicitly accepts +a measured target miss at the human merge checkpoint, that performance result +does not override otherwise passing correctness, custody, service-contract, +API, and coverage gates. Never lower coverage, skip nodes, or add a silent +fallback to meet the target. diff --git a/scripts/test_agent_gates.py b/scripts/test_agent_gates.py index 9041fab5..68819fc5 100644 --- a/scripts/test_agent_gates.py +++ b/scripts/test_agent_gates.py @@ -6611,6 +6611,12 @@ def test_backend_coverage_thresholds_are_regression_protected() -> None: ): assert f'"{required_field}"' in hosted_command assert "waiver" not in hosted_command.lower() + operations = (ROOT / "docs/operations_backend_testing.md").read_text( + encoding="utf-8" + ) + assert "whether\nthe eight-minute target was met" in operations + assert "does not override otherwise passing correctness" in operations + assert "makes the required check fail outright" not in operations active_phase = active_artifact_coverage_phase() expected_coverage = artifact_expected_coverage_commands_for(active_phase) actual_coverage = tuple( From bf3ea27bcb1ceec2f769267d8f1f684e4a0ec30e Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 25 Jul 2026 09:15:12 +0100 Subject: [PATCH 28/31] docs(agent-loop): bind accepted timing review --- .../WS-CI-001-02B-internal-review-evidence.md | 27 +++++++++---------- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 18 ++++++------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index eab64d71..af4d30b7 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,31 +12,29 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: 24f3b638b175352ddce3548d8c247b65c3328087 +Reviewed code SHA: a44c5167080d65a6ba8093cbaab8d50de790d17e -Reviewed at: 2026-07-24T20:18:04Z +Reviewed at: 2026-07-25T08:14:49Z Reviewer run IDs: ci02b_cr_senior, ci02b_cr_qa, ci02b_cr_security, ci02b_cr_ops, ci02b_cr_arch, ci02b_cr_ci, ci02b_cr_docs, ci02b_cr_reuse, ci02b_cr_test_delta -The timing-target repair after this historical reviewed revision is undergoing -a fresh exact-SHA review. The final evidence commit will replace this revision -binding after every required track passes. +After the reviewed SHA, only this final evidence reconciliation changed. ## Reviewer Results | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS WITH LOW RISKS | None | Failure rows remain simple; root-cause details stay in job logs. | -| QA/test | PASS | None | Startup, provisioning, collection, interruption, and teardown failures are covered. | -| security/auth | PASS | None | Environment isolation and admin/MinIO custody remain fail closed. | +| senior engineering | PASS | None | Accepted timing evidence is operationally clear; correctness gates remain fail closed. | +| QA/test | PASS | None | Timing-target semantics and all existing correctness criteria are regression protected. | +| security/auth | PASS | None | Read-only permissions, untrusted-PR safety, redaction, and resource custody remain intact. | | product/ops | PASS | None | No product, compensation, review-decision, or reputation behavior changed. | -| architecture | PASS WITH LOW RISKS | None | Runner, provisioner, and independent validator boundaries remain separate. | -| CI integrity | SUPERSEDED | None | Historical result before the owner-accepted timing-target repair. | -| docs | SUPERSEDED | None | Historical result before the runbook was aligned with accepted-risk semantics. | -| reuse/dedup | PASS WITH LOW RISKS | None | The single synthetic-row path reuses canonical lane finalization. | -| test delta | PASS | None | Repair adds regression coverage without skips or weakened assertions. | +| architecture | PASS | None | Human acceptance remains evidence, not workflow bypass logic. | +| CI integrity | PASS | None | Tests, coverage, custody, API, and failure gates remain blocking; timing remains measured. | +| docs | PASS WITH LOW RISKS | None | Runbook, status, trust bundle, and workflow now use one timing convention. | +| reuse/dedup | PASS | None | No duplicate validator or alternate timing convention remains. | +| test delta | PASS | None | No tests were removed or skipped; consistency assertions protect the new semantics. | The bootstrap repair review initially blocked publication because the reviewed SHA was stale and this file had an extra blank line at EOF. Both evidence @@ -120,7 +118,8 @@ git diff --check origin/main...HEAD mandatory in hosted CI and are not skipped by the workflow. - 100 Agent Gate tests passed. - Exact collection and independent recollection agreed on 2,056 pytest nodes - at reviewed code SHA `24f3b638`; independent evidence validation passed. + before the timing-only workflow repair; final exact-head hosted recollection + remains required. - Merge intent, Markdown links, stale wording, and diff integrity passed. - Local full-service execution was not used as hosted performance evidence. diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 8f7b6b88..7bb770b8 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -191,9 +191,9 @@ External review response file, if findings are posted: ## Reviewer results -Reviewed code SHA: `24f3b638b175352ddce3548d8c247b65c3328087` +Reviewed code SHA: `a44c5167080d65a6ba8093cbaab8d50de790d17e` -Reviewed at: `2026-07-24T20:18:04Z` +Reviewed at: `2026-07-25T08:14:49Z` Reviewer run IDs: `ci02b_cr_senior`, `ci02b_cr_qa`, `ci02b_cr_security`, `ci02b_cr_ops`, `ci02b_cr_arch`, `ci02b_cr_ci`, `ci02b_cr_docs`, @@ -201,14 +201,14 @@ Reviewer run IDs: `ci02b_cr_senior`, `ci02b_cr_qa`, `ci02b_cr_security`, | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS WITH LOW RISKS | None | Failure evidence remains simple and deterministic. | -| QA/test | PASS | None | All reported failure modes have regression coverage. | -| security/auth | PASS | None | Environment, database, and MinIO custody remain isolated. | +| senior engineering | PASS | None | Accepted timing evidence is clear; correctness remains fail closed. | +| QA/test | PASS | None | Timing semantics and existing acceptance criteria are regression protected. | +| security/auth | PASS | None | Permissions, redaction, and resource custody remain intact. | | product/ops | PASS | None | Product and contribution workflows are unchanged. | -| architecture | PASS WITH LOW RISKS | None | Existing runner/provisioner/validator boundaries remain intact. | -| CI integrity | Pending refresh | None | Re-reviewing the owner-accepted separation of the timing target from correctness gates. | -| docs | PASS WITH LOW RISKS | None | Operations wording matches the final failure semantics. | -| reuse/dedup | PASS WITH LOW RISKS | None | Synthetic failures reuse canonical finalization. | +| architecture | PASS | None | Acceptance remains evidence, not workflow bypass logic. | +| CI integrity | PASS | None | Tests, coverage, custody, API, and failures remain blocking. | +| docs | PASS WITH LOW RISKS | None | Canonical runbook and evidence use one timing convention. | +| reuse/dedup | PASS | None | No duplicate timing convention or validator path remains. | | test delta | PASS | None | No removed, skipped, deselected, or weakened tests. | ## Remaining risks From f601266a00466f687be0ab5679baa59159fb57b4 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 25 Jul 2026 16:07:12 +0100 Subject: [PATCH 29/31] docs(agent-loop): bind main integration review --- .../reviews/WS-CI-001-02B-internal-review-evidence.md | 7 ++++--- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index af4d30b7..9fa500a6 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,15 +12,16 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: a44c5167080d65a6ba8093cbaab8d50de790d17e +Reviewed code SHA: 1526ade7b0bb320a9b0673f7871f409c2ef4724c -Reviewed at: 2026-07-25T08:14:49Z +Reviewed at: 2026-07-25T15:06:39Z Reviewer run IDs: ci02b_cr_senior, ci02b_cr_qa, ci02b_cr_security, ci02b_cr_ops, ci02b_cr_arch, ci02b_cr_ci, ci02b_cr_docs, ci02b_cr_reuse, ci02b_cr_test_delta -After the reviewed SHA, only this final evidence reconciliation changed. +The reviewed SHA includes current trusted main through PR #200. After it, only +this final evidence reconciliation changed. ## Reviewer Results diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 7bb770b8..3b7f3b8d 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -191,9 +191,9 @@ External review response file, if findings are posted: ## Reviewer results -Reviewed code SHA: `a44c5167080d65a6ba8093cbaab8d50de790d17e` +Reviewed code SHA: `1526ade7b0bb320a9b0673f7871f409c2ef4724c` -Reviewed at: `2026-07-25T08:14:49Z` +Reviewed at: `2026-07-25T15:06:39Z` Reviewer run IDs: `ci02b_cr_senior`, `ci02b_cr_qa`, `ci02b_cr_security`, `ci02b_cr_ops`, `ci02b_cr_arch`, `ci02b_cr_ci`, `ci02b_cr_docs`, From 400be486863e6eb83a6343e872763b9076770537 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 25 Jul 2026 16:32:02 +0100 Subject: [PATCH 30/31] fix(ci): preserve orchestration failure causes --- .../WS-CI-001-02B-external-review-response.md | 30 ++++++++-- backend/scripts/run_isolated_tests.py | 8 +-- backend/scripts/run_test_lanes.py | 20 +++++++ backend/tests/test_ci_test_lanes.py | 13 +++- .../tests/test_isolated_database_runner.py | 59 +++++++++++++++++++ 5 files changed, 118 insertions(+), 12 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md index 44a7edbe..9fd2612b 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md @@ -34,6 +34,18 @@ All fourteen comments were valid and addressed: 14. Added an Agent Gate assertion for the step that reads `run.exit` and fails the required check. +The refreshed review on exact head `f601266a` added three findings, all +addressed: + +15. Replaced placeholder command arguments with the exact focused pytest nodes + and Markdown path used for verification. +16. Preserved the traceback and exception message for unexpected lane + orchestration failures while retaining cleanup and four-lane failure + evidence. +17. Allowed `asyncio.CancelledError` and other non-`Exception` cancellation + signals to propagate during MinIO bucket creation instead of translating + them into namespace collisions. + ## Comments deferred None. @@ -50,18 +62,26 @@ coverage, isolation, and service-contract gates. ```bash cd backend +python -m pip install ruff==0.15.22 ruff check app tests scripts -python -m pytest -q tests/test_ci_test_lanes.py tests/test_test_lane_evidence.py tests/test_isolated_database_runner.py -k '' +python -m pytest -q \ + tests/test_ci_test_lanes.py::test_unexpected_runner_failure_force_kills_and_records_every_lane \ + tests/test_ci_test_lanes.py::test_partial_startup_failure_records_exactly_four_failed_lanes \ + tests/test_isolated_database_runner.py::test_minio_creation_preserves_process_interrupts \ + tests/test_isolated_database_runner.py::test_minio_creation_preserves_async_cancellation cd .. python3 scripts/test_agent_gates.py python3 scripts/check_internal_review_evidence.py -python3 scripts/check_markdown_links.py +python3 scripts/check_markdown_links.py \ + .agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md git diff --check ``` -Local result before final evidence binding: exact Ruff passed, 89 focused -non-service tests passed, 11 service-backed tests remained mandatory for hosted -CI, and all 100 Agent Gate tests passed. +Latest focused repair result: exact Ruff passed; the four named traceback, +partial-startup, process-interrupt, and async-cancellation tests passed; all 100 +Agent Gate tests passed. The earlier broader local run passed 89 focused +non-service tests, while 11 service-backed tests remained mandatory for hosted +CI. ## Internal repair review diff --git a/backend/scripts/run_isolated_tests.py b/backend/scripts/run_isolated_tests.py index 0dba791a..c113a3a5 100644 --- a/backend/scripts/run_isolated_tests.py +++ b/backend/scripts/run_isolated_tests.py @@ -242,9 +242,7 @@ async def _create_minio(endpoint: str, bucket: str, prefix: str) -> None: async with _minio_client(endpoint) as client: try: await client.create_bucket(Bucket=bucket) - except (KeyboardInterrupt, SystemExit): - raise - except BaseException as exc: + except Exception as exc: raise RunnerError("minio_namespace_collision") from exc try: key = f"{prefix}/runner-probe" @@ -259,8 +257,10 @@ async def _create_minio(endpoint: str, bucket: str, prefix: str) -> None: await client.delete_object(Bucket=bucket, Key=f"{prefix}/runner-probe") await client.delete_bucket(Bucket=bucket) except BaseException as cleanup_error: + if not isinstance(cleanup_error, Exception): + raise raise RunnerError("minio_provisioning_cleanup_failed") from cleanup_error - if isinstance(exc, (KeyboardInterrupt, SystemExit, RunnerError)): + if not isinstance(exc, Exception) or isinstance(exc, RunnerError): raise raise RunnerError("minio_probe_failed") from exc diff --git a/backend/scripts/run_test_lanes.py b/backend/scripts/run_test_lanes.py index 8e4ae08b..df155af5 100644 --- a/backend/scripts/run_test_lanes.py +++ b/backend/scripts/run_test_lanes.py @@ -17,6 +17,7 @@ import subprocess import sys import time +import traceback from typing import Any, TextIO import uuid @@ -24,6 +25,13 @@ TESTS_DIR = ROOT / "tests" ISOLATED_RUNNER = ROOT / "scripts" / "run_isolated_tests.py" ADMIN_ENV = "WORKSTREAM_TEST_ADMIN_DATABASE_URL" +TRACEBACK_SECRET_ENVS = ( + ADMIN_ENV, + "WORKSTREAM_DATABASE_URL", + "WORKSTREAM_TEST_DATABASE_URL", + "MINIO_ROOT_PASSWORD", + "WORKSTREAM_ARTIFACT_S3_SECRET_ACCESS_KEY", +) COLLECTED_ENV = "WORKSTREAM_LANE_COLLECTED_NODES" COMPLETED_ENV = "WORKSTREAM_LANE_COMPLETED_NODES" SKIPPED_ENV = "WORKSTREAM_LANE_SKIPPED_NODES" @@ -519,6 +527,17 @@ def _handle_interrupt(_signum: int, _frame: object) -> None: INTERRUPTED = True +def _print_redacted_exception(exc: BaseException) -> None: + """Surface an orchestration traceback without known environment secrets.""" + rendered = "".join(traceback.format_exception(exc)) + for name in TRACEBACK_SECRET_ENVS: + value = os.environ.get(name) + if value: + rendered = rendered.replace(value, f"[REDACTED_{name}]") + sys.stderr.write(rendered) + sys.stderr.flush() + + def _finish_unit(active: ActiveLane, exit_code: int, elapsed: float) -> dict[str, Any]: active.log.flush() active.log.close() @@ -757,6 +776,7 @@ def run_lanes(metadata_dir: Path, summary_json: Path, timeout_seconds: float, *, time.sleep(POLL_SECONDS) except BaseException as exc: run_error = exc + _print_redacted_exception(exc) finally: now = time.monotonic() for key, item in list(active.items()): diff --git a/backend/tests/test_ci_test_lanes.py b/backend/tests/test_ci_test_lanes.py index 0e545ca3..4675a518 100644 --- a/backend/tests/test_ci_test_lanes.py +++ b/backend/tests/test_ci_test_lanes.py @@ -532,9 +532,9 @@ def fake_command(lane, _nodes, metadata, _timeout, _sha): def test_unexpected_runner_failure_force_kills_and_records_every_lane( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: - """Cleanup after an orchestration failure still emits four-lane evidence.""" + """Cleanup preserves four-lane evidence and the orchestration traceback.""" lanes = tuple( LaneDefinition(f"lane_{index}", (f"tests/test_{index}.py",)) for index in range(4) @@ -551,8 +551,11 @@ def fake_command(lane, _nodes, metadata, _timeout, _sha): return [sys.executable, "-c", "import time; time.sleep(30)"] monkeypatch.setattr(runner, "lane_command", fake_command) + admin_url = os.environ[runner.ADMIN_ENV] monkeypatch.setattr( - runner.time, "sleep", lambda _seconds: (_ for _ in ()).throw(RuntimeError("boom")) + runner.time, + "sleep", + lambda _seconds: (_ for _ in ()).throw(RuntimeError(f"boom {admin_url}")), ) metadata = tmp_path / "metadata" summary_path = tmp_path / "summary.json" @@ -562,6 +565,10 @@ def fake_command(lane, _nodes, metadata, _timeout, _sha): assert len(summary["lanes"]) == 4 assert all(row["interrupted"] for row in summary["lanes"]) assert all(row["execution_exit_code"] != 0 for row in summary["lanes"]) + stderr = capsys.readouterr().err + assert "Traceback (most recent call last)" in stderr + assert "RuntimeError: boom [REDACTED_WORKSTREAM_TEST_ADMIN_DATABASE_URL]" in stderr + assert admin_url not in stderr def test_partial_startup_failure_records_exactly_four_failed_lanes( diff --git a/backend/tests/test_isolated_database_runner.py b/backend/tests/test_isolated_database_runner.py index 488b97a7..c59f7618 100644 --- a/backend/tests/test_isolated_database_runner.py +++ b/backend/tests/test_isolated_database_runner.py @@ -117,6 +117,65 @@ async def create_bucket(self, **_kwargs): ) +def test_minio_creation_preserves_async_cancellation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Bucket creation cannot translate task cancellation into a collision.""" + class Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def create_bucket(self, **_kwargs): + raise asyncio.CancelledError + + monkeypatch.setattr(runner, "_minio_client", lambda _endpoint: Client()) + with pytest.raises(asyncio.CancelledError): + asyncio.run( + runner._create_minio( + "http://localhost:9000", "workstream-artifacts", "ci/test/run" + ) + ) + + +def test_minio_probe_cleans_up_and_preserves_async_cancellation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Probe cancellation cleans the bucket and remains task cancellation.""" + calls: list[str] = [] + + class Client: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return False + + async def create_bucket(self, **_kwargs): + calls.append("create_bucket") + + async def put_object(self, **_kwargs): + calls.append("put_object") + raise asyncio.CancelledError + + async def delete_object(self, **_kwargs): + calls.append("delete_object") + + async def delete_bucket(self, **_kwargs): + calls.append("delete_bucket") + + monkeypatch.setattr(runner, "_minio_client", lambda _endpoint: Client()) + with pytest.raises(asyncio.CancelledError): + asyncio.run( + runner._create_minio( + "http://localhost:9000", "workstream-artifacts", "ci/test/run" + ) + ) + assert calls == ["create_bucket", "put_object", "delete_object", "delete_bucket"] + + @pytest.mark.parametrize( ("lane", "expected_bucket"), [ From fb6a7ff0047cac57489cba546ba81c42e8938dfe Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 25 Jul 2026 16:43:58 +0100 Subject: [PATCH 31/31] docs(agent-loop): bind final external repair review --- .../WS-CI-001-02B-external-review-response.md | 20 +++++++----- .../WS-CI-001-02B-internal-review-evidence.md | 24 +++++++------- .../reviews/WS-CI-001-02B-pr-trust-bundle.md | 32 ++++++++++--------- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md index 9fd2612b..52e1284b 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-external-review-response.md @@ -62,13 +62,14 @@ coverage, isolation, and service-contract gates. ```bash cd backend -python -m pip install ruff==0.15.22 -ruff check app tests scripts -python -m pytest -q \ +.venv/bin/python -m pip install ruff==0.15.22 +.venv/bin/ruff check app tests scripts +.venv/bin/python -m pytest -q \ tests/test_ci_test_lanes.py::test_unexpected_runner_failure_force_kills_and_records_every_lane \ tests/test_ci_test_lanes.py::test_partial_startup_failure_records_exactly_four_failed_lanes \ tests/test_isolated_database_runner.py::test_minio_creation_preserves_process_interrupts \ - tests/test_isolated_database_runner.py::test_minio_creation_preserves_async_cancellation + tests/test_isolated_database_runner.py::test_minio_creation_preserves_async_cancellation \ + tests/test_isolated_database_runner.py::test_minio_probe_cleans_up_and_preserves_async_cancellation cd .. python3 scripts/test_agent_gates.py python3 scripts/check_internal_review_evidence.py @@ -77,7 +78,7 @@ python3 scripts/check_markdown_links.py \ git diff --check ``` -Latest focused repair result: exact Ruff passed; the four named traceback, +Latest focused repair result: exact Ruff passed; the five named traceback, partial-startup, process-interrupt, and async-cancellation tests passed; all 100 Agent Gate tests passed. The earlier broader local run passed 89 focused non-service tests, while 11 service-backed tests remained mandatory for hosted @@ -85,12 +86,15 @@ CI. ## Internal repair review -Reviewed code SHA: `24f3b638b175352ddce3548d8c247b65c3328087` +Historical CodeRabbit repair review SHA: `24f3b638b175352ddce3548d8c247b65c3328087` + +Final refreshed repair review SHA: `400be486863e6eb83a6343e872763b9076770537` Senior engineering, QA/test, security/auth, product/ops, architecture, CI integrity, docs, reuse/dedup, and test-delta tracks passed. Low residual risks -are limited to generic synthetic failure codes and reliance on job logs for the -parent-side orchestration cause. +are limited to an allowlist-based traceback redactor; current workflow secret +inputs are covered and regression tested, while broader pattern-based hardening +is deferred. ## Remaining risks diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md index 9fa500a6..91867c19 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-internal-review-evidence.md @@ -12,30 +12,30 @@ valid findings addressed: yes ## Reviewed Revision -Reviewed code SHA: 1526ade7b0bb320a9b0673f7871f409c2ef4724c +Reviewed code SHA: 400be486863e6eb83a6343e872763b9076770537 -Reviewed at: 2026-07-25T15:06:39Z +Reviewed at: 2026-07-25T15:42:43Z Reviewer run IDs: ci02b_cr_senior, ci02b_cr_qa, ci02b_cr_security, ci02b_cr_ops, ci02b_cr_arch, ci02b_cr_ci, ci02b_cr_docs, ci02b_cr_reuse, ci02b_cr_test_delta -The reviewed SHA includes current trusted main through PR #200. After it, only -this final evidence reconciliation changed. +The reviewed SHA includes current trusted main through PR #200 and the final +CodeRabbit repair. After it, only this final evidence reconciliation changed. ## Reviewer Results | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS | None | Accepted timing evidence is operationally clear; correctness gates remain fail closed. | -| QA/test | PASS | None | Timing-target semantics and all existing correctness criteria are regression protected. | -| security/auth | PASS | None | Read-only permissions, untrusted-PR safety, redaction, and resource custody remain intact. | +| senior engineering | PASS | None | Redacted tracebacks and cancellation cleanup are maintainable. | +| QA/test | PASS | None | Failure diagnostics, redaction, and cancellation are regression protected. | +| security/auth | PASS WITH LOW RISKS | None | Known workflow secrets are redacted; broader pattern hardening is deferred. | | product/ops | PASS | None | No product, compensation, review-decision, or reputation behavior changed. | -| architecture | PASS | None | Human acceptance remains evidence, not workflow bypass logic. | -| CI integrity | PASS | None | Tests, coverage, custody, API, and failure gates remain blocking; timing remains measured. | -| docs | PASS WITH LOW RISKS | None | Runbook, status, trust bundle, and workflow now use one timing convention. | -| reuse/dedup | PASS | None | No duplicate validator or alternate timing convention remains. | -| test delta | PASS | None | No tests were removed or skipped; consistency assertions protect the new semantics. | +| architecture | PASS | None | Runner ownership and failure-evidence boundaries remain intact. | +| CI integrity | PASS WITH LOW RISKS | None | Gate posture is unchanged; current secret inputs are covered. | +| docs | PASS | None | Exact commands and all seventeen findings are reconciled. | +| reuse/dedup | PASS | None | Existing cleanup and finalization paths remain reused. | +| test delta | PASS | None | No tests weakened; five focused repair cases pass. | The bootstrap repair review initially blocked publication because the reviewed SHA was stale and this file had an extra blank line at EOF. Both evidence diff --git a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md index 3b7f3b8d..391e7bc3 100644 --- a/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md +++ b/.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/reviews/WS-CI-001-02B-pr-trust-bundle.md @@ -24,7 +24,8 @@ hosted timing outcome on the exact PR head. - Phase: `implementation` - Contract path: `.agent-loop/initiatives/WS-CI-001-backend-ci-acceleration/chunks/WS-CI-001-02B-exact-custody-semantic-test-lanes.md` - Signed contract blob SHA: `784bae53f72f6460b4b74799c1c9b1519565dabe` -- Reviewed implementation SHA: `24f3b638b175352ddce3548d8c247b65c3328087` +- Initial reviewed implementation SHA: `24f3b638b175352ddce3548d8c247b65c3328087` +- Final refreshed repair review SHA: `400be486863e6eb83a6343e872763b9076770537` Only independently verified signed automation state is canonical authority. PR prose and checked boxes are navigation evidence, not authorization. @@ -51,11 +52,12 @@ PR prose and checked boxes are navigation evidence, not authorization. showed CPU contention. Rebalanced the existing four isolated processes as `shared_foundations`, `schema_contracts`, `project_lifecycle`, and `task_lifecycle` without changing the final workflow contract or gates. -- Addressed fourteen external findings across failure observability, stable Git +- Addressed seventeen external findings across failure observability, stable Git errors, MinIO interrupts, independent pytest environment isolation, UUID teardown, workflow failure-gate regression protection, and exact operations - wording. Partial startup and unexpected orchestration failures now retain - four explicit failed lane rows without becoming acceptable evidence. + wording, exact verification commands, traceback redaction, and async + cancellation custody. Partial startup and unexpected orchestration failures + retain four explicit failed lane rows without becoming acceptable evidence. - Deleted the obsolete shard runner and shard tests and updated operations docs. ## Why it changed @@ -186,14 +188,14 @@ External review response file, if findings are posted: | Source | Status | Notes | |---|---:|---| -| CodeRabbit | Addressed | Fourteen inline findings reconciled; refreshed review remains pending. | +| CodeRabbit | Addressed | Seventeen findings reconciled; refreshed review remains pending. | | GitHub checks | Pending | Exact final head must pass Agent Gates and Backend. | ## Reviewer results -Reviewed code SHA: `1526ade7b0bb320a9b0673f7871f409c2ef4724c` +Reviewed code SHA: `400be486863e6eb83a6343e872763b9076770537` -Reviewed at: `2026-07-25T15:06:39Z` +Reviewed at: `2026-07-25T15:42:43Z` Reviewer run IDs: `ci02b_cr_senior`, `ci02b_cr_qa`, `ci02b_cr_security`, `ci02b_cr_ops`, `ci02b_cr_arch`, `ci02b_cr_ci`, `ci02b_cr_docs`, @@ -201,15 +203,15 @@ Reviewer run IDs: `ci02b_cr_senior`, `ci02b_cr_qa`, `ci02b_cr_security`, | Reviewer | Result | Blocking findings | Notes | |---|---:|---|---| -| senior engineering | PASS | None | Accepted timing evidence is clear; correctness remains fail closed. | -| QA/test | PASS | None | Timing semantics and existing acceptance criteria are regression protected. | -| security/auth | PASS | None | Permissions, redaction, and resource custody remain intact. | +| senior engineering | PASS | None | Redacted tracebacks and cancellation cleanup are maintainable. | +| QA/test | PASS | None | Failure diagnostics, redaction, and cancellation are regression protected. | +| security/auth | PASS WITH LOW RISKS | None | Known workflow secrets are redacted; broader pattern hardening is deferred. | | product/ops | PASS | None | Product and contribution workflows are unchanged. | -| architecture | PASS | None | Acceptance remains evidence, not workflow bypass logic. | -| CI integrity | PASS | None | Tests, coverage, custody, API, and failures remain blocking. | -| docs | PASS WITH LOW RISKS | None | Canonical runbook and evidence use one timing convention. | -| reuse/dedup | PASS | None | No duplicate timing convention or validator path remains. | -| test delta | PASS | None | No removed, skipped, deselected, or weakened tests. | +| architecture | PASS | None | Runner ownership and failure-evidence boundaries remain intact. | +| CI integrity | PASS WITH LOW RISKS | None | Gate posture is unchanged; current secret inputs are covered. | +| docs | PASS | None | Exact commands and all seventeen findings are reconciled. | +| reuse/dedup | PASS | None | Existing cleanup and finalization paths remain reused. | +| test delta | PASS | None | No tests weakened; five focused repair cases pass. | ## Remaining risks