diff --git a/docs/PRD.md b/docs/PRD.md index 4ce9682..847021c 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -489,7 +489,7 @@ For the passed-on-arrival case, which occurred in 4 of 8 executed cycles in the | Command | Behaviour | |---|---| | `tdd sensitivity begin` | record `git diff` and the untracked-file set as the reference state | -| `tdd sensitivity check` | run the suite with the agent's mutation in place; require the target to now fail; record the mutation diff and the observed failure | +| `tdd sensitivity check` | run the suite with the agent's mutation in place; require the target to now fail; record the mutation diff and the observed failure; each adapter extracts a one-line `target_evidence` (the assertion line, not runner noise), stored in `sensitivity_check.evidence_line` (schema v7) and rendered as the `observed:` snippet in the friction log | | `tdd sensitivity end` | `git checkout --` the mutated tracked paths, then assert the resulting `git diff` is byte-identical to the reference; emit `restore_mismatch` on any difference | - **R8.4** A cycle that passed on arrival cannot reach `CLOSED` without a completed sensitivity @@ -802,7 +802,7 @@ depend on any of them being installed. ### 13.1 Storage - **R13.1** SQLite, single file, append-only for invocations and events. - **R13.2** Schema versioned and migrated. The schema is the long-lived asset; the transport is not. - Current schema version: **3** (v3 adds the `advance_claim` table, R9.23). + Current schema version: **7** (v7 adds `sensitivity_check.evidence_line TEXT` for per-adapter assertion-line evidence; earlier milestones: v3 adds `advance_claim`, v4–v6 are intermediate columns). ### 13.2 Location - **R13.3** **One ledger per repository**, in a per-user data directory keyed by the repository's diff --git a/src/tddcli/adapters/base.py b/src/tddcli/adapters/base.py index 7ce42b6..454fc7d 100644 --- a/src/tddcli/adapters/base.py +++ b/src/tddcli/adapters/base.py @@ -25,6 +25,7 @@ class Verdict: target: str | None = None target_outcome: str = NOT_FOUND target_failure: str = "" + target_evidence: str = "" passed: list[str] = field(default_factory=list) failed: list[str] = field(default_factory=list) duration_ms: int = 0 diff --git a/src/tddcli/adapters/exec_adapter.py b/src/tddcli/adapters/exec_adapter.py index 2642db8..48b7ace 100644 --- a/src/tddcli/adapters/exec_adapter.py +++ b/src/tddcli/adapters/exec_adapter.py @@ -168,6 +168,9 @@ def run(self, target: str | None = None) -> Verdict: if target == qualified: verdict.target_outcome = FAILED verdict.target_failure = clip_failure(combined) + verdict.target_evidence = next( + (ln for ln in reversed(combined.splitlines()) if ln.strip()), "" + ) verdict.duration_ms = int((time.monotonic() - started) * 1000) return verdict diff --git a/src/tddcli/adapters/gradle_adapter.py b/src/tddcli/adapters/gradle_adapter.py index 83b00db..f2f6c3c 100644 --- a/src/tddcli/adapters/gradle_adapter.py +++ b/src/tddcli/adapters/gradle_adapter.py @@ -329,7 +329,11 @@ def run(self, target: str | None = None) -> Verdict: verdict.target_outcome = PASSED elif target in failed: verdict.target_outcome = FAILED - verdict.target_failure = failures.get(target, "test failed") + failure_text = failures.get(target, "test failed") + verdict.target_failure = failure_text + verdict.target_evidence = next( + (ln for ln in failure_text.splitlines() if ln.strip()), "" + ) # else NOT_FOUND (default): the suite ran but never produced this id return verdict diff --git a/src/tddcli/adapters/pytest_adapter.py b/src/tddcli/adapters/pytest_adapter.py index b7897f4..eb00975 100644 --- a/src/tddcli/adapters/pytest_adapter.py +++ b/src/tddcli/adapters/pytest_adapter.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import re import shlex import tempfile from pathlib import Path @@ -148,7 +149,9 @@ def run(self, target: str | None = None) -> Verdict: if hit is not None: verdict.target_outcome = PASSED if hit["outcome"] == "passed" else FAILED call = hit.get("call") or hit.get("setup") or {} - verdict.target_failure = clip_failure(str(call.get("longrepr", ""))) + longrepr = str(call.get("longrepr", "")) + verdict.target_failure = clip_failure(longrepr) + verdict.target_evidence = self._evidence_line(longrepr) else: target_file = native.split("::", 1)[0] if any(c == target_file or c.startswith(target_file) for c in uncollectable): @@ -160,6 +163,14 @@ def run(self, target: str | None = None) -> Verdict: verdict.target_outcome = NOT_FOUND return verdict + @staticmethod + def _evidence_line(longrepr: str) -> str: + for line in longrepr.splitlines(): + m = re.match(r"^E\s+(.*)", line) + if m: + return m.group(1) + return "" + @staticmethod def _collector_error(collectors: list[dict], target_file: str) -> str: for collector in collectors: diff --git a/src/tddcli/adapters/vitest_adapter.py b/src/tddcli/adapters/vitest_adapter.py index a0cc315..821c11e 100644 --- a/src/tddcli/adapters/vitest_adapter.py +++ b/src/tddcli/adapters/vitest_adapter.py @@ -152,8 +152,13 @@ def run(self, target: str | None = None) -> Verdict: self.normalise_id(self._id_for(suite.get("name", ""), t["fullName"])) == ntarget ): + messages = t.get("failureMessages", []) verdict.target_failure = "\n".join( - clip_failure(m, 600) for m in t.get("failureMessages", [])[:3] + clip_failure(m, 600) for m in messages[:3] + ) + first_msg = messages[0] if messages else "" + verdict.target_evidence = next( + (ln for ln in first_msg.splitlines() if ln.strip()), "" ) return verdict diff --git a/src/tddcli/adapters/xctest_adapter.py b/src/tddcli/adapters/xctest_adapter.py index ac12cfc..2d591e6 100644 --- a/src/tddcli/adapters/xctest_adapter.py +++ b/src/tddcli/adapters/xctest_adapter.py @@ -272,7 +272,9 @@ def run(self, target: str | None = None) -> Verdict: verdict.target_outcome = PASSED elif target in verdict.failed: verdict.target_outcome = FAILED - verdict.target_failure = self._failure_for(combined, self.strip(target)) + window = self._failure_for(combined, self.strip(target)) + verdict.target_failure = window + verdict.target_evidence = self._evidence_line(window) # else NOT_FOUND (default) return verdict @@ -289,6 +291,13 @@ def _build_errors(self, combined: str) -> str: if ": error:" in line or "** BUILD FAILED **" in line ) + @staticmethod + def _evidence_line(window: str) -> str: + for line in window.splitlines(): + if ": error:" in line: + return line + return "" + def _failure_for(self, combined: str, native_id: str) -> str: """Capture the assertion lines between 'started' and 'failed' for one test. diff --git a/src/tddcli/cli.py b/src/tddcli/cli.py index 725c087..ce1ee15 100644 --- a/src/tddcli/cli.py +++ b/src/tddcli/cli.py @@ -1134,16 +1134,18 @@ def cmd_sensitivity(args) -> Envelope: projects = json.loads(cycle["projects"]) if args.step == "check": - outcomes, _, _, failure_text = engine.run_projects( + outcomes, _, verdicts, failure_text = engine.run_projects( projects, targets, cycle, "SENSITIVITY", False ) # A mutation that breaks collection also proves the test depends on the code. bites = bool(outcomes) and all(o in (FAILED, NOT_COLLECTED) for o in outcomes.values()) + evidence = next((v.target_evidence for v in verdicts if v.target_evidence), "") ledger.update( "sensitivity_check", open_check["id"], mutation_diff=gitutil.diff_text(worktree)[:20000], observed_failure=failure_text[:4000], + evidence_line=evidence, ) if not bites: return Envelope( diff --git a/src/tddcli/ledger.py b/src/tddcli/ledger.py index bb4de6c..c3608e5 100644 --- a/src/tddcli/ledger.py +++ b/src/tddcli/ledger.py @@ -13,7 +13,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path -SCHEMA_VERSION = 6 +SCHEMA_VERSION = 7 class LedgerVersionError(RuntimeError): @@ -37,6 +37,8 @@ class LedgerVersionError(RuntimeError): 4: "ALTER TABLE baseline ADD COLUMN source TEXT NOT NULL DEFAULT 'probed';", # v5 -> v6 added ancillary_files column to plan_contract; ALTER TABLE covers old ledgers. 5: "ALTER TABLE plan_contract ADD COLUMN ancillary_files TEXT NOT NULL DEFAULT '[]';", + # v6 -> v7 added evidence_line column to sensitivity_check; ALTER TABLE covers old ledgers. + 6: "ALTER TABLE sensitivity_check ADD COLUMN evidence_line TEXT;", } SCHEMA = """ @@ -195,6 +197,7 @@ class LedgerVersionError(RuntimeError): reference_untracked TEXT NOT NULL, mutation_diff TEXT, observed_failure TEXT, + evidence_line TEXT, restored_ok INTEGER, opened_at TEXT NOT NULL, closed_at TEXT diff --git a/src/tddcli/render.py b/src/tddcli/render.py index c302cc2..48fea4b 100644 --- a/src/tddcli/render.py +++ b/src/tddcli/render.py @@ -93,8 +93,15 @@ def friction_log(ledger: Ledger, run) -> str: if sens: a("- **Sensitivity check:** verified, restore byte-identical") if sens["observed_failure"]: - snippet = sens["observed_failure"].strip().splitlines() - a(f" - observed: `{snippet[0][:160] if snippet else ''}`") + evidence = sens["evidence_line"] + if evidence: + capped = ("…" + evidence[-160:]) if len(evidence) > 160 else evidence + a(f" - observed: `{capped}`") + elif evidence == "": + a(" - observed: ") + else: + snippet = sens["observed_failure"].strip().splitlines() + a(f" - observed: `{snippet[0][:160] if snippet else ''}`") commits = ledger.all( "SELECT * FROM commit_record WHERE cycle_id = ? ORDER BY id", (cycle["id"],) diff --git a/tasks/friction-logs/issue-68-sensitivity-evidence-friction.md b/tasks/friction-logs/issue-68-sensitivity-evidence-friction.md new file mode 100644 index 0000000..bef39a1 --- /dev/null +++ b/tasks/friction-logs/issue-68-sensitivity-evidence-friction.md @@ -0,0 +1,121 @@ +# Implementation Friction Log: tasks/issue-68-sensitivity-evidence.md + +- Run: 12 +- Executor: claude-sonnet-4-6 (source: transcript) +- Plan blob: `34ede538eab3a01b75b9d9cdfde5c86000318fe0` (declared) +- Started: 2026-08-28T16:04:38.652188+00:00 Ended: 2026-08-28T17:21:10.569424+00:00 Outcome: complete +- Baseline failures at start: tddcli=0 + +## Plan fidelity + +- Declared cycles: 11 +- Delivered: 11 Skipped: 0 +- Never reached: none +- Human interventions: 0 + +### Cycle 11: a long observed line is capped keeping the tail, not the head _(standard)_ +- **Target:** `tddcli::tests/test_sensitivity_evidence.py::test_long_evidence_is_capped_keeping_the_tail` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** failed (as expected) +- **Commits:** + - `e6d5235ad` [red] test: an over-long observed line keeps its tail (1 files) + - `146e15314` [green] feat: tail-keeping cap on the observed evidence line (1 files) + +### Cycle 10: legacy rows with NULL evidence keep the first-line fallback _(standard)_ +- **Target:** `tddcli::tests/test_sensitivity_evidence.py::test_null_evidence_falls_back_to_first_observed_line` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'SENSITIVITY': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** passed (**passed**) +- **Sensitivity check:** verified, restore byte-identical + - observed: `repo = PosixPath('/private/var/folders/zl/3010c_557g5_2rm9tyqsc03h0000gp/T/pytest-of-headless-coding/pytest-239/test_null_evidence_falls_back_0/workspace')` +- **Commits:** + - `a7004cc9f` [refactor] refactor: legacy rows with NULL evidence keep the first-line fallback (1 files) +- **Event — red_first_violation:** ["tddcli::tests/test_sensitivity_evidence.py::test_null_evidence_falls_back_to_first_observed_line"] + +### Cycle 9: empty evidence renders an explicit no-assertion-line sentinel _(standard)_ +- **Target:** `tddcli::tests/test_sensitivity_evidence.py::test_empty_evidence_renders_the_sentinel` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** failed (as expected) +- **Commits:** + - `8e68e09f0` [red] test: empty evidence renders (1 files) + - `8e0308820` [green] feat: render the no-assertion-line sentinel instead of wire noise (1 files) + +### Cycle 8: friction log observed line renders the stored evidence line _(standard)_ +- **Target:** `tddcli::tests/test_sensitivity_evidence.py::test_friction_log_observed_line_is_the_evidence_line` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** failed (as expected) +- **Commits:** + - `b2dc16f52` [red] test: observed line shows evidence_line, not the raw first line (1 files) + - `ddc60246f` [green] feat: friction log prefers evidence_line for the observed snippet (1 files) + +### Cycle 7: sensitivity check stores the adapter's evidence line in the ledger _(standard)_ +- **Target:** `tddcli::tests/test_sensitivity_evidence.py::test_sensitivity_check_records_the_evidence_line` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `53ad9734d` [red] test: sensitivity check persists evidence_line on its ledger row (1 files) + - `619c0bd3d` [green] feat: schema v7 — sensitivity_check.evidence_line stored at check time (2 files) + - `739ed8a0f` [refactor] refactor: sensitivity check stores the adapter's evidence line in the ledger (1 files) + +### Cycle 6: exec evidence falls back to the last non-empty output line _(standard)_ +- **Target:** `tddcli::tests/test_evidence_extraction.py::test_exec_evidence_is_the_last_nonempty_output_line` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `1df55d55d` [red] test: exec evidence is the last non-empty combined-output line (1 files) + - `78771ff01` [green] feat: exec evidence falls back to the last non-empty line (1 files) + - `9012b32fe` [refactor] refactor: exec evidence falls back to the last non-empty output line (1 files) + +### Cycle 5: gradle evidence is the first line of the junit failure message _(standard)_ +- **Target:** `tddcli::tests/test_evidence_extraction.py::test_gradle_evidence_is_the_first_failure_message_line` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** failed (as expected) +- **Commits:** + - `e0a5301d1` [red] test: gradle evidence is the junit failure message line (1 files) + - `bebee75f9` [green] feat: gradle evidence extracted from the failure element's message (1 files) + +### Cycle 4: vitest evidence is the first line of the first failure message _(standard)_ +- **Target:** `tddcli::tests/test_evidence_extraction.py::test_vitest_evidence_is_the_first_failure_message_line` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** failed (as expected) +- **Commits:** + - `3c0c01982` [red] test: vitest evidence is the first failureMessage line (1 files) + - `248e55040` [green] feat: vitest evidence extracted from failureMessages[0] (1 files) + +### Cycle 3: xctest evidence is the error line, not interleaved console noise _(standard)_ +- **Target:** `tddcli::tests/test_evidence_extraction.py::test_xctest_evidence_is_the_error_line_not_console_noise` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 1, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `e04e0098a` [red] test: xctest evidence line ignores console noise in the test window (1 files) + - `5817c6ab2` [green] feat: xctest evidence is the first ': error:' line of the test's window (1 files) + - `1f1b40b52` [refactor] refactor: xctest evidence is the error line, not interleaved console noise (1 files) + +### Cycle 2: pytest evidence is empty when longrepr has no assertion line _(pin)_ +- **Target:** `tddcli::tests/test_evidence_extraction.py::test_pytest_evidence_is_empty_when_no_assertion_line_exists` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_PIN': 1, 'SENSITIVITY': 1, 'CLOSE_SWEEP': 1} +- **First run outcome:** passed (as expected) +- **Sensitivity check:** verified, restore byte-identical + - observed: `tmp_path = PosixPath('/private/var/folders/zl/3010c_557g5_2rm9tyqsc03h0000gp/T/pytest-of-headless-coding/pytest-162/test_pytest_evidence_is_empty_0')` +- **Commits:** + - `dad64c656` [pin] test: pin empty pytest evidence when no E-line exists (1 files) + +### Cycle 1: pytest evidence is the assertion line, not the xdist worker header _(standard)_ +- **Target:** `tddcli::tests/test_evidence_extraction.py::test_pytest_evidence_is_the_assertion_line_not_the_xdist_header` +- **Projects:** `tddcli` +- **Suite runs by phase:** {'AWAITING_TEST': 2, 'AWAITING_IMPL': 1, 'CLOSE_SWEEP': 2} +- **First run outcome:** failed (as expected) +- **Commits:** + - `a4be0c744` [red] test: pytest evidence line skips the xdist worker header (1 files) + - `a2b7ecb47` [green] feat: Verdict.target_evidence — pytest extracts the first E-line of longrepr (2 files) + - `b464cf742` [refactor] refactor: pytest evidence is the assertion line, not the xdist worker header (1 files) + diff --git a/tests/test_evidence_extraction.py b/tests/test_evidence_extraction.py new file mode 100644 index 0000000..91fdf2f --- /dev/null +++ b/tests/test_evidence_extraction.py @@ -0,0 +1,226 @@ +"""Per-adapter evidence line extraction (issue #68). + +Each adapter extracts a single plausible assertion/failure line from its +runner output, stored as Verdict.target_evidence, so that the sensitivity +check's observed: line is auditable even under xdist headers or console noise. +""" + +from __future__ import annotations + +import json +import stat +from pathlib import Path +from unittest.mock import patch + +import tddcli.adapters.base as adapters_base +from tddcli import adapters +from tddcli import config as config_mod +from tddcli.adapters.exec_adapter import ExecAdapter +from tddcli.adapters.gradle_adapter import GradleAdapter +from tddcli.adapters.vitest_adapter import VitestAdapter +from tddcli.adapters.xctest_adapter import XCTestAdapter + +_XCTEST_TOML = ( + "[project.native-ios]\n" + 'root = "native-ios"\n' + 'adapter = "xctest"\n' + 'test_paths = ["AppTests/"]\n' + 'test_command = "xcodebuild test -project App.xcodeproj -scheme AppTests"\n' +) + + +_GRADLE_TOML = ( + "[project.android-app]\n" + 'root = "android-app"\n' + 'adapter = "gradle"\n' + 'test_paths = ["src/test/"]\n' + 'test_command = "./gradlew testDebugUnitTest"\n' +) + + +def _exec_adapter(tmp_path): + (tmp_path / "tdd.toml").write_text( + "[project.gates]\n" + 'root = "gates"\n' + 'adapter = "exec"\n' + 'test_paths = ["scripts/check-*.sh"]\n' + ) + (tmp_path / "gates" / "scripts").mkdir(parents=True) + return ExecAdapter(config_mod.load(tmp_path).project("gates"), tmp_path) + + +def _write_script(path: Path, body: str) -> None: + path.write_text(f"#!/bin/sh\n{body}\n") + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + +def _gradle_adapter(tmp_path): + (tmp_path / "tdd.toml").write_text(_GRADLE_TOML) + (tmp_path / "android-app" / "src" / "test").mkdir(parents=True) + return GradleAdapter(config_mod.load(tmp_path).project("android-app"), tmp_path) + + +def _gradle_write_results(adapter, xml, task="testDebugUnitTest"): + out = adapter.root / "build" / "test-results" / task / "TEST-com.example.CalcTest.xml" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(xml) + + +def _vitest_adapter(tmp_path): + (tmp_path / "tdd.toml").write_text( + "[project.frontend]\n" + 'root = "frontend"\n' + 'adapter = "vitest"\n' + 'test_paths = ["**/*.test.ts"]\n' + ) + (tmp_path / "frontend").mkdir() + return VitestAdapter(config_mod.load(tmp_path).project("frontend"), tmp_path) + + +def _xctest_adapter(tmp_path): + (tmp_path / "tdd.toml").write_text(_XCTEST_TOML) + (tmp_path / "native-ios" / "AppTests").mkdir(parents=True) + return XCTestAdapter(config_mod.load(tmp_path).project("native-ios"), tmp_path) + + +def _pytest_adapter(tmp_path): + (tmp_path / "tdd.toml").write_text( + "[project.backend]\n" + 'root = "backend"\n' + 'adapter = "pytest"\n' + 'test_paths = ["tests/"]\n' + 'test_command = "pytest tests"\n' + ) + return adapters.build(config_mod.load(tmp_path).project("backend"), tmp_path) + + +def test_pytest_evidence_is_empty_when_no_assertion_line_exists(tmp_path, monkeypatch): + adapter = _pytest_adapter(tmp_path) + longrepr = ( + "tests/test_calc.py:10: RecursionError\n" + "RecursionError: maximum recursion depth exceeded\n" + ) + + def fake(command, cwd, timeout=1800, extra_env=None, label=None): + marker = "--json-report-file=" + path = command.split(marker, 1)[1].split(" --", 1)[0] + Path(path.strip("'\"")).write_text(json.dumps({ + "tests": [{ + "nodeid": "tests/test_calc.py::test_recurse", + "outcome": "failed", + "call": {"longrepr": longrepr}, + }], + })) + return 1, "", "" + + monkeypatch.setattr(adapters_base, "run_command", fake) + verdict = adapter.run("backend::tests/test_calc.py::test_recurse") + assert verdict.target_evidence == "" + + +def test_pytest_evidence_is_the_assertion_line_not_the_xdist_header(tmp_path, monkeypatch): + adapter = _pytest_adapter(tmp_path) + longrepr = ( + "[gw0] darwin -- Python 3.12.8 /tmp/x/bin/python\n" + "\n" + "tests/test_calc.py:10: in test_add\n" + " assert result == expected\n" + "E AssertionError: reversed mismatch\n" + "E assert [1, 2] == [2, 1]\n" + "+ Where:\n" + " expected = [2, 1]\n" + ) + + def fake(command, cwd, timeout=1800, extra_env=None, label=None): + marker = "--json-report-file=" + path = command.split(marker, 1)[1].split(" --", 1)[0] + Path(path.strip("'\"")).write_text(json.dumps({ + "tests": [{ + "nodeid": "tests/test_calc.py::test_add", + "outcome": "failed", + "call": {"longrepr": longrepr}, + }], + })) + return 1, "", "" + + monkeypatch.setattr(adapters_base, "run_command", fake) + verdict = adapter.run("backend::tests/test_calc.py::test_add") + assert verdict.target_evidence == "AssertionError: reversed mismatch" + + +def test_xctest_evidence_is_the_error_line_not_console_noise(tmp_path): + adapter = _xctest_adapter(tmp_path) + target = "native-ios::AppTests/RecTests/testStopsRecording" + canned = ( + "Test Suite 'All tests' started at 2026-08-27 10:00:00.000.\n" + "Test Case '-[AppTests.RecTests testStopsRecording]' started.\n" + "2026-08-27 10:00:00.001 AppTests[1234:5678] Socket SO_ERROR [61: Connection refused]\n" + "/Users/x/RecTests.swift:42: error: -[AppTests.RecTests testStopsRecording] :" + " XCTAssertEqual failed: (\"recording\") is not equal to (\"stopped\")\n" + "Test Case '-[AppTests.RecTests testStopsRecording]' failed (0.002 seconds).\n" + "Test Suite 'All tests' failed at 2026-08-27 10:00:00.003.\n" + "** TEST FAILED **\n" + ) + with patch.object(type(adapter), "_run_suite", return_value=(1, canned, "")): + verdict = adapter.run(target) + assert "XCTAssertEqual failed" in verdict.target_evidence + assert "Socket SO_ERROR" not in verdict.target_evidence + + +def test_vitest_evidence_is_the_first_failure_message_line(tmp_path): + adapter = _vitest_adapter(tmp_path) + suite_path = str(tmp_path / "frontend" / "calc.test.ts") + target = "frontend::calc.test.ts > calc add returns the sum" + report = { + "testResults": [{ + "name": suite_path, + "status": "failed", + "assertionResults": [{ + "fullName": "calc add returns the sum", + "status": "failed", + "failureMessages": [ + "AssertionError: expected 2 to be 3 // Object.is equality\n at Object. (calc.test.ts:5:14)\n" + ], + }], + }], + } + with patch.object(type(adapter), "_run_suite", return_value=(1, json.dumps(report), "")): + verdict = adapter.run(target) + assert verdict.target_evidence == "AssertionError: expected 2 to be 3 // Object.is equality" + + +def test_gradle_evidence_is_the_first_failure_message_line(tmp_path): + adapter = _gradle_adapter(tmp_path) + target = "android-app::com.example.CalcTest/add" + junit_xml = ( + '\n' + '\n' + ' \n' + ' ' + "org.opentest4j.AssertionFailedError: expected:<1000> but was:<500>\n" + "\tat com.example.CalcTest.add(CalcTest.kt:10)\n" + "\n" + " \n" + "\n" + ) + + def side_effect(cmd, env=None): + _gradle_write_results(adapter, junit_xml) + return (1, "", "") + + with patch.object(type(adapter), "_run_suite", side_effect=side_effect): + verdict = adapter.run(target) + assert verdict.target_evidence == "expected:<1000> but was:<500>" + + +def test_exec_evidence_is_the_last_nonempty_output_line(tmp_path): + adapter = _exec_adapter(tmp_path) + script = tmp_path / "gates" / "scripts" / "check-deploy.sh" + _write_script( + script, + 'echo "INFO: starting deployment"\necho "INFO: checking service"\necho "FAIL: expected exit 0, got 1"; exit 1', + ) + target = "gates::scripts/check-deploy.sh" + verdict = adapter.run(target) + assert verdict.target_evidence == "FAIL: expected exit 0, got 1" diff --git a/tests/test_sensitivity_evidence.py b/tests/test_sensitivity_evidence.py new file mode 100644 index 0000000..c7f4ec4 --- /dev/null +++ b/tests/test_sensitivity_evidence.py @@ -0,0 +1,157 @@ +"""Sensitivity evidence line: ledger storage and friction-log rendering (issue #68).""" + +from __future__ import annotations + +from conftest import git, run_cli, write_plan +from tddcli import gitutil +from tddcli.ledger import Ledger + +PLAN = """--- +cycles: + - n: 1 + project: backend + title: "adding two numbers" + test: "tests/test_add.py::test_add_two_numbers" + commit_red: "test: add" + commit_green: "feat: add()" +--- + +# Plan +""" + +TEST_ADD = """from app.calc import add + + +def test_add_two_numbers(): + assert add(2, 3) == 5 +""" + +CALC_WORKING = "def add(a, b):\n return a + b\n" +CALC_MUTATED = "def add(a, b):\n return 0\n" + + +def _start(repo): + (repo / "backend" / "app" / "calc.py").write_text(CALC_WORKING) + (repo / "backend" / "tests" / "test_add.py").write_text(TEST_ADD) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "add calc.py and test") + plan = write_plan(repo, PLAN) + reg = run_cli(repo, "plan", "register", plan) + assert reg["ok"], reg + started = run_cli(repo, "run", "start", "--plan", plan) + assert started["ok"], started + return started + + +def _drive_sensitivity(repo): + """Advance → SENSITIVITY_REQUIRED, then run sensitivity begin/check/end.""" + out = run_cli(repo, "advance") + assert out["run"]["phase"] == "SENSITIVITY_REQUIRED", out + run_cli(repo, "sensitivity", "begin") + (repo / "backend" / "app" / "calc.py").write_text(CALC_MUTATED) + checked = run_cli(repo, "sensitivity", "check") + assert checked["ok"], checked + ended = run_cli(repo, "sensitivity", "end") + assert ended["result"]["restored_ok"] is True + return checked + + +def _close_cycle_and_render(repo, out_path): + """Advance through AWAITING_REFACTOR and CLOSE_SWEEP, then render the log.""" + run_cli(repo, "advance") # -> AWAITING_REFACTOR + run_cli(repo, "advance") # -> close sweep / next cycle + run_cli(repo, "log", "render", "--out", str(out_path)) + + +def test_sensitivity_check_records_the_evidence_line(repo): + _start(repo) + _drive_sensitivity(repo) + led = Ledger(gitutil.repo_identity(repo)) + row = led.one("SELECT evidence_line FROM sensitivity_check ORDER BY id DESC LIMIT 1") + assert row is not None + assert row["evidence_line"].startswith("assert") + + +def test_long_evidence_is_capped_keeping_the_tail(repo, tmp_path): + _start(repo) + _drive_sensitivity(repo) + led = Ledger(gitutil.repo_identity(repo)) + check_row = led.one("SELECT id FROM sensitivity_check ORDER BY id DESC LIMIT 1") + tail = 'is not equal to ("stopped") -[AppTests.RecTests testStopsRecording]' + long_line = "A" * 250 + tail + led.update( + "sensitivity_check", + check_row["id"], + observed_failure="something", + evidence_line=long_line, + ) + out = tmp_path / "log.md" + run_cli(repo, "advance") # -> AWAITING_REFACTOR + run_cli(repo, "advance") # -> close sweep + run_cli(repo, "log", "render", "--out", str(out)) + rendered = out.read_text() + assert tail in rendered + assert "…" in rendered + assert "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" not in rendered + + +def test_null_evidence_falls_back_to_first_observed_line(repo, tmp_path): + _start(repo) + _drive_sensitivity(repo) + led = Ledger(gitutil.repo_identity(repo)) + check_row = led.one("SELECT id FROM sensitivity_check ORDER BY id DESC LIMIT 1") + led.update( + "sensitivity_check", + check_row["id"], + observed_failure="legacy first line\nsecond line", + evidence_line=None, + ) + out = tmp_path / "log.md" + run_cli(repo, "advance") # -> AWAITING_REFACTOR + run_cli(repo, "advance") # -> close sweep + run_cli(repo, "log", "render", "--out", str(out)) + rendered = out.read_text() + assert "observed: `legacy first line`" in rendered + + +def test_empty_evidence_renders_the_sentinel(repo, tmp_path): + _start(repo) + _drive_sensitivity(repo) + led = Ledger(gitutil.repo_identity(repo)) + check_row = led.one("SELECT id FROM sensitivity_check ORDER BY id DESC LIMIT 1") + led.update( + "sensitivity_check", + check_row["id"], + observed_failure="[gw0] darwin -- Python 3.12.8 /tmp/x\nnoise line", + evidence_line="", + ) + out = tmp_path / "log.md" + run_cli(repo, "advance") # -> AWAITING_REFACTOR + run_cli(repo, "advance") # -> close sweep + run_cli(repo, "log", "render", "--out", str(out)) + rendered = out.read_text() + assert "" in rendered + assert "[gw0]" not in rendered + + +def test_friction_log_observed_line_is_the_evidence_line(repo, tmp_path): + _start(repo) + _drive_sensitivity(repo) + led = Ledger(gitutil.repo_identity(repo)) + check_row = led.one("SELECT id FROM sensitivity_check ORDER BY id DESC LIMIT 1") + led.update( + "sensitivity_check", + check_row["id"], + observed_failure=( + "[gw0] darwin -- Python 3.12.8 /tmp/x\n\n def test_add_two_numbers():\n" + "E AssertionError: reversed mismatch" + ), + evidence_line="AssertionError: reversed mismatch", + ) + out = tmp_path / "log.md" + run_cli(repo, "advance") # -> AWAITING_REFACTOR + run_cli(repo, "advance") # -> close sweep + run_cli(repo, "log", "render", "--out", str(out)) + rendered = out.read_text() + assert "observed: `AssertionError: reversed mismatch`" in rendered + assert "[gw0]" not in rendered