diff --git a/.codd/canon.lock b/.codd/canon.lock index 947ab33c..73382b3c 100644 --- a/.codd/canon.lock +++ b/.codd/canon.lock @@ -38,4 +38,4 @@ docs/requirements/extractor-requirements.md sha256:fa02e6eddbb42e96e693aae72ee82 docs/requirements/require-command-requirements.md sha256:5ef134be5650dd10e77de818a019f02beabec763a1794358639c706baa635b37 v3.39.0-canon-tripwire -docs/requirements/system-requirements.md sha256:9c34a9e4e20ec5d2abc99d582aeeaab2b93347c101e3edd5ab9636b4ed228aae v3.39.0-canon-tripwire +docs/requirements/system-requirements.md sha256:9f7f5168b587ff0467c936cca54427ec2605faf2d898e1479a165c82253e6aa1 issue-43 diff --git a/.codd/dag.json b/.codd/dag.json index 47bbe4f0..85f9b6ab 100644 --- a/.codd/dag.json +++ b/.codd/dag.json @@ -1,6 +1,6 @@ { "version": "1", - "built_at": "2026-09-11T11:20:56.349213+00:00", + "built_at": "2026-09-14T13:21:51.898973+00:00", "project_root": ".", "nodes": [ { @@ -49,9 +49,11 @@ "__future__", "re", "dataclasses", + "datetime", "pathlib", "typing", "codd.requirement_reconciliation", + "codd.runtime_record", "codd.dag.builder", "codd.parsing.filesystem_routes" ], @@ -156,6 +158,19 @@ "runtime_evidence": [] } }, + { + "id": "codd/ansi.py", + "kind": "impl_file", + "path": "codd/ansi.py", + "attributes": { + "language": "python", + "imports": [ + "__future__", + "re" + ], + "runtime_evidence": [] + } + }, { "id": "codd/artifact_contract.py", "kind": "impl_file", @@ -1961,6 +1976,7 @@ "pathlib", "typing", "yaml", + "codd.ansi", "codd.deployment.providers" ], "runtime_evidence": [] @@ -2941,6 +2957,7 @@ "shlex", "typing", "yaml", + "codd.path_safety", "codd.stack.lock", "codd.stack.project", "codd.stack.command_authenticity", @@ -3761,6 +3778,7 @@ "subprocess", "dataclasses", "pathlib", + "codd.ansi", ".builtin_adapters", ".registry", ".verify_plan", @@ -4521,6 +4539,7 @@ "typing", "yaml", "codd.generator", + "codd.path_safety", "codd.project_types", "codd.validator", "codd.config", @@ -5010,10 +5029,11 @@ "language": "python", "imports": [ "__future__", + "re", "dataclasses", "pathlib", - "re", - "typing" + "typing", + "codd.ansi" ], "runtime_evidence": [] } @@ -5043,6 +5063,7 @@ "codd.languages.verify_executor", "codd.languages.verify_plan", "yaml", + "codd.ansi", "codd.config", "codd.dag", "codd.dag.builder", @@ -5292,6 +5313,28 @@ "runtime_evidence": [] } }, + { + "id": "codd/runtime_record.py", + "kind": "impl_file", + "path": "codd/runtime_record.py", + "attributes": { + "language": "python", + "imports": [ + "__future__", + "hashlib", + "json", + "os", + "re", + "dataclasses", + "datetime", + "pathlib", + "typing", + "yaml", + "codd.config" + ], + "runtime_evidence": [] + } + }, { "id": "codd/runtime_smoke/__init__.py", "kind": "impl_file", @@ -5371,7 +5414,8 @@ "pathlib", "codd.runtime_smoke.checks", "codd.runtime_smoke.config", - "codd.runtime_smoke.report" + "codd.runtime_smoke.report", + "codd.runtime_record" ], "runtime_evidence": [] } @@ -8481,6 +8525,7 @@ "__future__", "json", "textwrap", + "datetime", "pathlib", "yaml", "codd.acceptance_evidence", @@ -8489,6 +8534,7 @@ "codd.dag.checks.acceptance_evidence", "codd.dag.runner", "codd.config", + "codd.runtime_record", "codd.acceptance_record", "codd.acceptance_sync", "codd.verifiable_behavior_audit", @@ -9961,6 +10007,7 @@ "dataclasses", "pathlib", "pytest", + "tests.greenfield.conftest", "codd.cli", "codd.greenfield.pipeline", "codd.greenfield.test_rederivation", @@ -9980,6 +10027,7 @@ "dataclasses", "pathlib", "pytest", + "tests.greenfield.conftest", "codd.greenfield.test_rederivation" ] } @@ -12287,6 +12335,21 @@ ] } }, + { + "id": "tests/test_ansi_sanitized_runner_output.py", + "kind": "test_file", + "path": "tests/test_ansi_sanitized_runner_output.py", + "attributes": { + "language": "python", + "imports": [ + "__future__", + "pathlib", + "codd.ansi", + "codd.deployment.providers.verification.pytest_http", + "codd.repair.test_failure_attribution" + ] + } + }, { "id": "tests/test_api_contract_proto_enum_render.py", "kind": "test_file", @@ -16176,7 +16239,8 @@ "codd.runtime_smoke.checks", "codd.runtime_smoke.config", "codd.runtime_smoke.report", - "codd.runtime_smoke.runner" + "codd.runtime_smoke.runner", + "codd.runtime_record" ] } }, @@ -17460,6 +17524,11 @@ "to_id": "codd/requirement_reconciliation.py", "kind": "imports" }, + { + "from_id": "codd/acceptance_evidence.py", + "to_id": "codd/runtime_record.py", + "kind": "imports" + }, { "from_id": "codd/acceptance_evidence.py", "to_id": "tests/dag/test_acceptance_evidence.py", @@ -17560,6 +17629,11 @@ "to_id": "tests/test_implement_syntax_gate.py", "kind": "tested_by" }, + { + "from_id": "codd/ansi.py", + "to_id": "tests/test_ansi_sanitized_runner_output.py", + "kind": "tested_by" + }, { "from_id": "codd/artifact_contract.py", "to_id": "codd/artifact_ids.py", @@ -21720,11 +21794,21 @@ "to_id": "tests/test_deployment_verification_templates.py", "kind": "tested_by" }, + { + "from_id": "codd/deployment/providers/verification/pytest_http.py", + "to_id": "codd/ansi.py", + "kind": "imports" + }, { "from_id": "codd/deployment/providers/verification/pytest_http.py", "to_id": "codd/deployment/providers/__init__.py", "kind": "imports" }, + { + "from_id": "codd/deployment/providers/verification/pytest_http.py", + "to_id": "tests/test_ansi_sanitized_runner_output.py", + "kind": "tested_by" + }, { "from_id": "codd/deployment/providers/verification/pytest_http.py", "to_id": "tests/test_e2e_harness.py", @@ -23640,6 +23724,11 @@ "to_id": "codd/operational_e2e_audit.py", "kind": "imports" }, + { + "from_id": "codd/greenfield/pipeline.py", + "to_id": "codd/path_safety.py", + "kind": "imports" + }, { "from_id": "codd/greenfield/pipeline.py", "to_id": "codd/planner.py", @@ -25445,6 +25534,11 @@ "to_id": "tests/test_stack_profile.py", "kind": "tested_by" }, + { + "from_id": "codd/languages/verify_executor.py", + "to_id": "codd/ansi.py", + "kind": "imports" + }, { "from_id": "codd/languages/verify_executor.py", "to_id": "codd/languages/adapters/runner_report.py", @@ -26540,6 +26634,11 @@ "to_id": "codd/lexicon.py", "kind": "imports" }, + { + "from_id": "codd/planner.py", + "to_id": "codd/path_safety.py", + "kind": "imports" + }, { "from_id": "codd/planner.py", "to_id": "codd/project_types.py", @@ -27680,6 +27779,11 @@ "to_id": "tests/test_verify_integrity.py", "kind": "tested_by" }, + { + "from_id": "codd/repair/test_failure_attribution.py", + "to_id": "codd/ansi.py", + "kind": "imports" + }, { "from_id": "codd/repair/test_failure_attribution.py", "to_id": "tests/repair/test_env_failure_defense.py", @@ -27690,6 +27794,16 @@ "to_id": "tests/repair/test_test_failure_attribution.py", "kind": "tested_by" }, + { + "from_id": "codd/repair/test_failure_attribution.py", + "to_id": "tests/test_ansi_sanitized_runner_output.py", + "kind": "tested_by" + }, + { + "from_id": "codd/repair/verify_runner.py", + "to_id": "codd/ansi.py", + "kind": "imports" + }, { "from_id": "codd/repair/verify_runner.py", "to_id": "codd/config.py", @@ -28240,6 +28354,21 @@ "to_id": "tests/test_extract_routes.py", "kind": "tested_by" }, + { + "from_id": "codd/runtime_record.py", + "to_id": "codd/config.py", + "kind": "imports" + }, + { + "from_id": "codd/runtime_record.py", + "to_id": "tests/dag/test_acceptance_evidence.py", + "kind": "tested_by" + }, + { + "from_id": "codd/runtime_record.py", + "to_id": "tests/test_runtime_smoke.py", + "kind": "tested_by" + }, { "from_id": "codd/runtime_smoke/__init__.py", "to_id": "codd/runtime_smoke/config.py", @@ -28290,6 +28419,11 @@ "to_id": "tests/test_runtime_smoke.py", "kind": "tested_by" }, + { + "from_id": "codd/runtime_smoke/runner.py", + "to_id": "codd/runtime_record.py", + "kind": "imports" + }, { "from_id": "codd/runtime_smoke/runner.py", "to_id": "codd/runtime_smoke/checks.py", diff --git a/CHANGELOG.md b/CHANGELOG.md index 463c35fa..e72bffc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,26 @@ Install or upgrade with: pip install -U codd-dev ``` +## [3.42.0] - 2026-09-14 — Fix: a switch is a declaration, not evidence — the runtime stage has to have RUN + +3.41.0 closed the *demotion* hole: an acceptance criterion whose only evidence is a runtime behaviour, in a project whose `runtime_smoke` stage is switched off, is a criterion proved by nothing. It left the mirror image open. Turning the stage **on** costs one line of YAML, and the gate read that line as evidence. + +Measured on a real project: adding `runtime_smoke.enabled: true` — no dev server, no database, `codd verify --runtime` never invoked once — moved `runtime_evidence_not_executable` **50 → 0**, `unbound_acceptance` **61 → 20**, and the total finding count **180 → 89**. Half the findings cleared because a boolean was typed. The cheapest way through the acceptance gate was to declare rather than to verify, which is the incentive exactly backwards, and the check that exists to catch missing evidence was the one accepting a claim in its place. + +**The rule now: only an EXECUTED runtime obligation binds.** + +- **`codd/runtime_record.py` (NEW) — the runtime execution ledger.** The runtime smoke runner writes `/runtime_ledger.json` when the stage is enabled **and at least one check actually ran**; an all-skipped run and a disabled stage write nothing, because a record of nothing is not a record. It holds `recorded_at` (UTC), `passed`, each check's `name` / `category` / `passed` / `skipped`, the `target_url` the run actually went against (`--runtime-base-url` may have pointed it elsewhere), and `config_digest` — a hash of the project's **own** `runtime_smoke` (minus `report.log_to_file` / `report.file_path`, which decide where output goes; `report.fail_fast` stays in, because it decides whether the checks after the first failure run at all) and `runtime` sections, read from `codd.yaml` rather than the defaults-merged view so that upgrading CoDD does not expire every recorded run at once. Missing, malformed, unversioned, or listing no executed check reads as **no evidence**, never as evidence — the same rule `acceptance_ledger.json` already follows. Written through a temp file and renamed, so two runs racing leave one whole record rather than half of two. Like that ledger it is a project artifact meant to be committed and read in diffs; unlike it, it is written automatically, and that difference is the point — a manual record is a human verdict that only a human may refresh, this one records that a machine ran, which only the machine that ran can honestly write. +- **`runtime_evidence_not_executed` (NEW finding).** The stage is on but no run discharged the obligation: no ledger (`no_record`), a ledger written against a runtime configuration the project has since changed (`config_changed`) or against another deployment (`target_changed`), a ledger past the project's freshness window — or dated in the future, which no window can judge (`stale`), a run that failed (`run_failed`), or — for an explicitly declared `verified_by: runtime:` — a ledger in which no check answering to that name passed (`target_not_executed`). `runtime_evidence_not_executable` is unchanged and still means the other thing: the stage is off. Two holes, two remedies, two findings. +- **The criterion goes back to `unbound_acceptance` as well** — and, while the stage is enabled, at `runtime_execution_severity` rather than `unbound_severity`. An enabled-but-never-run obligation binds exactly as much as a switched-off one: nothing. But its unbound-ness is *caused by* the not-executed-ness — one defect, one remedy ("run it") — and pricing it as an ordinary unbound would strand a strict project: red before Step 8 runs, and Step 8 is what writes the record that clears it. With the stage OFF the ordinary `unbound_severity` still applies; such a project has no runtime path to run at all. +- **Explicit and inferred targets are treated differently, as they already were.** An explicit `runtime:` names a case, so a run in which no non-skipped check answers to that name has not discharged it (`print_sheet` / `Print Sheet` / `print-sheet` are one name, not three). An **inferred** obligation — an `operation_flow.` reference in a project that never adopted the `verified_by` column — declares no such correspondence, so any recorded run satisfies it. CoDD does not invent a mapping the project never wrote down. A run that FAILED is executed but proves nothing (`run_failed`): within the run Step 8 reports it in red, and the ledger is read by every LATER verification, which is where "it ran, and it was broken" would otherwise pass for proof. A failure in one check does not sink another — an explicitly named case is discharged when the check answering to that name passed, whatever else failed beside it. +- **`acceptance_evidence.runtime_execution_severity` (NEW, default amber even under `mode: strict`)** and **`acceptance_evidence.runtime_max_age_hours` (NEW, unset by default)**. A project that never persisted the ledger is **unproven**, not proven wrong, and upgrading CoDD must not turn an existing build red on its own; a project that wants the hard failure sets the key. With no age declared, the clock alone never expires a run — a configuration change still does. + +A ledger the runner could not write (read-only checkout, full disk) is a lost artifact, not a false green — the next verification honestly reports `no_record` — so Step 8 says so loudly on stderr instead of failing the run. The one branch that IS a false green, an EARLIER passing record surviving a run that could not overwrite it, is not left standing: the stale record is removed, and if even that is refused Step 8 FAILS and names the file. + +Not closed, and stated rather than left to be discovered: runtime execution evidence is **not yet bound to implementation content** the way a manual record is. A rewritten implementation under an unchanged runtime configuration does not invalidate a recorded run. Closing that needs a content digest both the runner and the gate can compute, and the runner has no dependency graph; until then the ledger proves the stage RAN, not that it ran against today's code. + +Tests: 34 new (`tests/dag/test_acceptance_evidence.py`, `tests/test_runtime_smoke.py`), all `tmp_path` synthetic miniatures — no framework, URL or project literal. Dogfood on the real project that produced the measurement, same tree, same requirement documents: `enabled: true` with no recorded run now reports **180** findings (`runtime_evidence_not_executed` 50, `unbound_acceptance` 61) instead of 89 — identical to the stage being off; seeding an execution record drops it to 89; changing the runtime configuration after that run puts it straight back to 180. + ## [3.41.0] - 2026-09-11 — Feature: the acceptance criterion the customer signed, wired to something a machine runs A requirement shipped with a passing test suite and the feature was still wrong. Reconstructed afterwards: the requirement said "15 per sheet", the acceptance document said 20, the code said 20; one test existed and it exercised a command-line script, while the thing users pressed was a button on an admin page that no test touched; the runtime verification stage was disabled, so it was skipped **without printing anything**; and a manual "looks right" check from two weeks earlier still read as evidence for an implementation that had since been rewritten. diff --git a/README.md b/README.md index 5b62f534..3a6d96e2 100644 --- a/README.md +++ b/README.md @@ -201,6 +201,8 @@ CoDD ships **39 ready-made "lexicons"** — checklists drawn from real industry **What becomes visible.** Every acceptance criterion in your requirement tables is reconciled against evidence a machine runs: `codd verify` now reports criteria bound to no test, evidence that asserts nothing or is skipped, runtime obligations in a project whose runtime stage is off, values retyped instead of read by name, and evidence that never runs through the entry point users actually reach. Findings are **advisory (amber) by default** — upgrading CoDD never turns your build red on its own — and they are never silent: each one names the criterion and what to do about it. +**A switch is a declaration, not evidence.** A criterion whose evidence is "the running system does X" is discharged by a runtime stage that RAN, not by one that is configured. Each `codd verify --runtime` writes `/runtime_ledger.json` — what ran, whether it passed, and a digest of the runtime configuration it ran against — and the check reads that record: no record, a record written against a configuration you have since changed, a record older than `acceptance_evidence.runtime_max_age_hours` (unset by default), or a `verified_by: runtime:` no check answered to, and the criterion is reported as `runtime_evidence_not_executed` and still bound to nothing. Commit the ledger like any other evidence artifact. This class stays amber even under `mode: strict` unless you set `runtime_execution_severity: red`: no record means unproven, not proven wrong. + **Turning it into a gate.** Put `acceptance_evidence: {mode: strict}` in `codd.yaml` (`codd init` writes it for new projects) and the same findings become red. Per-class overrides let you hard-fail one class while the rest stay advisory: `acceptance_evidence: {unbound_severity: red}`. Opt out entirely with `acceptance_evidence: {enabled: false}` — the findings disappear with it, which is the point of choosing. diff --git a/codd/__init__.py b/codd/__init__.py index 53fdd9e9..01beb64a 100644 --- a/codd/__init__.py +++ b/codd/__init__.py @@ -1,3 +1,3 @@ """CoDD — Coherence-Driven Development.""" -__version__ = "3.41.0" +__version__ = "3.42.0" diff --git a/codd/acceptance_evidence.py b/codd/acceptance_evidence.py index eb4f3fc6..f4e3b701 100644 --- a/codd/acceptance_evidence.py +++ b/codd/acceptance_evidence.py @@ -47,6 +47,7 @@ import re from dataclasses import dataclass, field +from datetime import datetime from pathlib import Path from typing import Any, Iterable, Mapping, Sequence @@ -189,6 +190,8 @@ class AcceptanceSettings: sections: tuple[str, ...] = () acceptance_columns: tuple[str, ...] = () runtime_severity: str = "amber" + runtime_execution_severity: str = "amber" + runtime_max_age_hours: float | None = None unbound_severity: str = "amber" param_severity: str = "amber" undeclared_numeric_severity: str = "amber" @@ -239,6 +242,14 @@ def severity(key: str, strict_default: str) -> str: except (TypeError, ValueError): max_findings = 30 + raw_max_age = section.get("runtime_max_age_hours") + try: + max_age_hours = float(raw_max_age) if raw_max_age is not None else None + except (TypeError, ValueError): + max_age_hours = None + if max_age_hours is not None and max_age_hours <= 0: + max_age_hours = None + return AcceptanceSettings( enabled=bool(section.get("enabled", True)), mode=mode, @@ -246,6 +257,11 @@ def severity(key: str, strict_default: str) -> str: sections=sections, acceptance_columns=_string_tuple(section.get("acceptance_columns")), runtime_severity=severity("runtime_severity", "red"), + # Amber even under `mode: strict` unless set explicitly: a project that + # never persisted an execution record is UNPROVEN, not proven wrong, and + # upgrading CoDD must not turn an existing build red on its own. + runtime_execution_severity=severity("runtime_execution_severity", "amber"), + runtime_max_age_hours=max_age_hours, unbound_severity=severity("unbound_severity", "red"), param_severity=severity("param_severity", "red"), undeclared_numeric_severity=severity("undeclared_numeric_severity", "amber"), @@ -402,6 +418,146 @@ def runtime_smoke_enabled(config: Mapping[str, Any] | None) -> bool: return bool(section.get("enabled", False)) +# Why an enabled runtime stage is not yet evidence, in one word each. +EXECUTION_NO_RECORD = "no_record" +EXECUTION_CONFIG_CHANGED = "config_changed" +EXECUTION_TARGET_CHANGED = "target_changed" +EXECUTION_STALE = "stale" +EXECUTION_RUN_FAILED = "run_failed" +EXECUTION_TARGET_NOT_EXECUTED = "target_not_executed" + +EXECUTION_REASON_TEXT = { + EXECUTION_NO_RECORD: ( + "no runtime execution has been recorded (`{ledger}` is absent, unreadable, " + "or records no executed check)" + ), + EXECUTION_CONFIG_CHANGED: ( + "the recorded run targeted a DIFFERENT runtime configuration — " + "`runtime_smoke`/`runtime` changed in codd.yaml since `{ledger}` was written" + ), + EXECUTION_TARGET_CHANGED: ( + "the recorded run went against `{recorded}`, not the configured `{configured}` " + "— evidence about one deployment is not evidence about another" + ), + EXECUTION_STALE: "the recorded run is {age:.0f}h old, past `runtime_max_age_hours: {limit:.0f}`", + "future": ( + "the recorded run is dated in the FUTURE, so its age cannot be trusted " + "against `runtime_max_age_hours`" + ), + EXECUTION_RUN_FAILED: ( + "the recorded run FAILED. It executed, so this is not a missing run — but a " + "failing run is not evidence that the behaviour works" + ), +} + + +def _configured_target(config: Mapping[str, Any] | None) -> str: + section = config.get("runtime_smoke") if isinstance(config, Mapping) else None + dev_server = section.get("dev_server") if isinstance(section, Mapping) else None + url = dev_server.get("url") if isinstance(dev_server, Mapping) else None + return str(url or "").strip() + + +def runtime_execution_evidence( + project_root: Path | str, + config: Mapping[str, Any] | None, + max_age_hours: float | None = None, + now: datetime | None = None, +) -> tuple[Any | None, str, str]: + """The recorded runtime execution, or WHY there is none to lean on. + + ``runtime_smoke.enabled: true`` declares the stage; this asks whether it ran. + The ways a declaration fails to become evidence, each with its own remedy: + nothing was ever recorded, the recorded run targeted a configuration the + project has since changed, it went against a different deployment than the + one configured, the record is older than the project's freshness window (or + dated in the future, which no window can judge), or the run failed. + + Returns ``(record, reason, detail)`` — ``reason`` is empty exactly when the + record itself may be leaned on. Two per-criterion questions remain after + that, because the answer differs by criterion rather than by record: whether + an explicitly named case passed (:func:`unexecuted_runtime_targets`), and + whether a run that failed SOMEWHERE still proves an inferred obligation + (:func:`inferred_obligation_unproven`) — it does not, while a named case that + passed is unaffected by a different check failing beside it. + """ + + from codd.runtime_record import ( + ledger_path, + load_runtime_ledger, + runtime_config_digest, + ) + + root = Path(project_root) + display = ledger_path(root).name + record = load_runtime_ledger(root) + if record is None: + return None, EXECUTION_NO_RECORD, EXECUTION_REASON_TEXT[EXECUTION_NO_RECORD].format(ledger=display) + if record.config_digest != runtime_config_digest(root): + return ( + record, + EXECUTION_CONFIG_CHANGED, + EXECUTION_REASON_TEXT[EXECUTION_CONFIG_CHANGED].format(ledger=display), + ) + configured = _configured_target(config) + # A record that names no target while the project configures one is not a + # match either: every record this CoDD writes carries the target it went + # against, so a blank one is a record from somewhere else. + if configured and record.target_url != configured: + return ( + record, + EXECUTION_TARGET_CHANGED, + EXECUTION_REASON_TEXT[EXECUTION_TARGET_CHANGED].format( + recorded=record.target_url or "(unrecorded)", configured=configured + ), + ) + if max_age_hours is not None: + age = record.age_hours(now) + if age < 0: + return record, EXECUTION_STALE, EXECUTION_REASON_TEXT["future"] + if age > max_age_hours: + return ( + record, + EXECUTION_STALE, + EXECUTION_REASON_TEXT[EXECUTION_STALE].format(age=age, limit=max_age_hours), + ) + return record, "", "" + + +def inferred_obligation_unproven(criterion: AcceptanceCriterion, record: Any) -> str: + """Why a usable record still does not prove an INFERRED runtime obligation. + + An inferred obligation names no case, so the whole run is its evidence — and + a run that failed is not evidence that the behaviour works. (An EXPLICIT case + is judged on its own check instead: a database check failing beside it does + not un-prove the case that passed.) + """ + + if record is None or criterion.evidence_of("runtime"): + return "" + return "" if record.passed else EXECUTION_RUN_FAILED + + +def unexecuted_runtime_targets( + criterion: AcceptanceCriterion, + targets: Iterable[str], + record: Any, +) -> tuple[str, ...]: + """Declared runtime targets that no passing check of the recorded run covers. + + The same explicit/inferred asymmetry :func:`runtime_obligations` is built on. + An EXPLICIT ``verified_by: runtime:`` names a case, so a run in which no + check answering to that name passed has not discharged it. An INFERRED + obligation (an ``operation_flow.`` reference in a project that never + adopted the column) declares no such correspondence, and CoDD does not invent + one — a passing run of the project's own stage satisfies it. + """ + + if record is None or not criterion.evidence_of("runtime"): + return () + return tuple(target for target in targets if not record.covers(target)) + + def runtime_obligations( criteria: Iterable[AcceptanceCriterion], declared_operation_ids: frozenset[str], diff --git a/codd/cli.py b/codd/cli.py index 3aa2e8e9..aec0d8f4 100644 --- a/codd/cli.py +++ b/codd/cli.py @@ -6185,11 +6185,45 @@ def _run_runtime_smoke_gate(path: str, runtime_base_url: str | None, runtime_ski click.echo(smoke_result.markdown_section.rstrip()) if smoke_result.report_path is not None: click.echo(f"[codd verify] Runtime smoke report: {_display_path(smoke_result.report_path, Path(path).resolve())}") + if smoke_result.ledger_path is not None: + click.echo( + "[codd verify] Runtime execution record: " + f"{_display_path(smoke_result.ledger_path, Path(path).resolve())} (commit it)" + ) + elif smoke_result.ledger_status == "stale_record_left": + # The one branch that IS a false green: an older, passing record survived + # a run that could not overwrite it, so the next verification would read + # yesterday's verdict as today's. Fail rather than leave that standing. + click.echo( + "[FAIL] Step 8 could not write the runtime execution record, and the " + "EARLIER record could not be removed either. The next `codd verify` would " + "read that stale record as current evidence. Remove " + f"`{_display_path(_runtime_ledger_path(Path(path).resolve()), Path(path).resolve())}` " + "(or fix the permissions) before relying on this run.", + err=True, + ) + raise SystemExit(1) + elif smoke_result.ledger_status == "write_failed": + # Not a false green — no record now stands, so the next verification + # honestly reports that no run is on record — but the evidence this run + # produced is gone, and a silent loss is how the gate goes quiet again. + click.echo( + "[WARN] Runtime execution record could NOT be written. This run's evidence " + "is lost: the next `codd verify` will report `runtime_evidence_not_executed` " + "for every criterion that declares runtime evidence.", + err=True, + ) if not smoke_result.overall_passed: click.echo("[FAIL] Step 8 runtime smoke failed", err=True) raise SystemExit(1) +def _runtime_ledger_path(project_root: Path) -> Path: + from codd.runtime_record import ledger_path + + return ledger_path(project_root) + + def _runtime_smoke_skip(runtime_skip: tuple[str, ...]) -> tuple[str, ...]: return tuple(item for item in runtime_skip if item != "verification-test") diff --git a/codd/dag/checks/acceptance_evidence.py b/codd/dag/checks/acceptance_evidence.py index 83bfac8b..9cb8e5c0 100644 --- a/codd/dag/checks/acceptance_evidence.py +++ b/codd/dag/checks/acceptance_evidence.py @@ -11,6 +11,8 @@ * (a) ``runtime_evidence_not_executable`` — a runtime obligation in a project whose runtime stage is switched off (a silent skip that used to read green); + ``runtime_evidence_not_executed`` — the stage is switched ON but nothing ever + recorded a run of it, so the switch is a declaration standing in for evidence; ``unbound_acceptance`` / ``unresolved_evidence`` / ``manual_evidence_missing`` — a criterion that reaches no machine-checked evidence at all. * (b) ``off_shipped_path`` / ``multiple_implementers`` / ``reachability_unknown`` @@ -23,7 +25,8 @@ See ``docs/design/acceptance-evidence-invariant.md``. Generality: the check reads the project's own declarations (requirement tables, -``operation_flow``, ``runtime_smoke``) and carries no project, framework or +``operation_flow``, ``runtime_smoke``) and the project's own execution record +(``/runtime_ledger.json``), and carries no project, framework or language literal. It is dormant for a project that declares no acceptance criteria — there is nothing to certify — and every severity is configurable, so a project may downgrade a class to amber without losing the finding. @@ -36,6 +39,8 @@ from typing import Any, Iterable, Mapping from codd.acceptance_evidence import ( + EXECUTION_REASON_TEXT, + EXECUTION_TARGET_NOT_EXECUTED, SETTINGS_KEY, AcceptanceCriterion, acceptance_settings, @@ -48,8 +53,11 @@ read_test_text, requirement_anchor_pattern, resolve_test_targets, + inferred_obligation_unproven, + runtime_execution_evidence, runtime_obligations, runtime_smoke_enabled, + unexecuted_runtime_targets, scan_requirement_anchor_hits, substance_at, substantive_tests, @@ -139,21 +147,19 @@ def run( anchors = {req_id: set(by_path) for req_id, by_path in anchor_hits.items()} violations: list[dict[str, Any]] = [] - violations.extend( - _runtime_violations(criteria, declared_ids, config, resolved.runtime_severity) - ) - # Only an EXECUTABLE runtime obligation counts as a binding. When the - # runtime stage is off, the criterion really is proved by nothing, and - # the two findings ask for two different remedies (turn the stage on / - # write a test) — that is two defects, not one reported twice. - runtime_bound_ids = ( - frozenset( - criterion.req_id - for criterion, _targets in runtime_obligations(criteria, declared_ids) - ) - if runtime_smoke_enabled(config) - else frozenset() + # Only an EXECUTED runtime obligation counts as a binding. When the + # runtime stage is off — or on but never run — the criterion really is + # proved by nothing, and the findings ask for different remedies (turn + # the stage on / run it / write a test) — that is two defects, not one + # reported twice. + runtime_findings, runtime_bound_ids, runtime_pending_ids = _runtime_violations( + criteria, + declared_ids, + config, + resolved, + project_root=root, ) + violations.extend(runtime_findings) binding_violations, bound_by_req = _binding_violations( root, criteria, @@ -165,6 +171,7 @@ def run( anchors=anchors, anchor_offsets=anchor_hits, runtime_bound_ids=runtime_bound_ids, + runtime_pending_ids=runtime_pending_ids, ) violations.extend(binding_violations) violations.extend(_parameter_violations(root, criteria, bound_by_req, resolved)) @@ -195,24 +202,44 @@ def _runtime_violations( criteria: Iterable[AcceptanceCriterion], declared_ids: frozenset[str], config: Mapping[str, Any], - severity: str, -) -> list[dict[str, Any]]: - """Runtime-evidence obligations in a project whose runtime stage cannot run. - - This is the *demotion* hole: ``codd verify`` runs Step 8 only when the - project's ``runtime_smoke`` section is enabled, and an absent section made - the step vanish without a word. A criterion whose evidence is "the running - system does X" then had NO evidence while the run still reported green. + settings: Any, + *, + project_root: Path, +) -> tuple[list[dict[str, Any]], frozenset[str], frozenset[str]]: + """Runtime-evidence obligations that no run has actually discharged. + + Two holes of the same shape, one behind the other: + + * the *demotion* hole — ``codd verify`` runs Step 8 only when the project's + ``runtime_smoke`` section is enabled, and an absent section made the step + vanish without a word. A criterion whose evidence is "the running system + does X" then had NO evidence while the run still reported green. + * the *declaration* hole — turning the section on costs one line of YAML and + used to be read as evidence by itself. Measured on a real project: adding + ``enabled: true``, with no dev server and no ``codd verify --runtime``, + moved this finding 56 -> 0 and ``unbound_acceptance`` 57 -> 19. The + cheapest way to clear the gate was to declare rather than to verify. + + So the question is not "is the stage switched on" but "did it RUN": a + recorded execution (``/runtime_ledger.json``) whose configuration + digest still matches, is inside the project's freshness window, and — for an + explicitly named ``runtime:`` — actually exercised that case. + + Returns the findings, the criteria whose runtime obligation IS discharged + (the only ones that count as bound downstream), and the criteria still + WAITING on a run — whose `unbound_acceptance` is priced at the execution + severity, because "run the stage" is the one remedy for both. """ - if runtime_smoke_enabled(config): - return [] - violations: list[dict[str, Any]] = [] - for criterion, targets in runtime_obligations(criteria, declared_ids): - violations.append( + obligations = runtime_obligations(criteria, declared_ids) + if not obligations: + return [], frozenset(), frozenset() + + if not runtime_smoke_enabled(config): + return [ { "type": "runtime_evidence_not_executable", - "severity": severity, + "severity": settings.runtime_severity, "req_id": criterion.req_id, "source": criterion.source, "targets": list(targets), @@ -227,8 +254,57 @@ def _runtime_violations( "or downgrade with `acceptance_evidence.runtime_severity: amber`." ), } + for criterion, targets in obligations + ], frozenset(), frozenset() + + record, reason, detail = runtime_execution_evidence( + project_root, + config, + max_age_hours=settings.runtime_max_age_hours, + ) + + violations: list[dict[str, Any]] = [] + bound: set[str] = set() + pending: set[str] = set() + for criterion, targets in obligations: + criterion_reason, why = reason, detail + if not criterion_reason: + # Two questions the record cannot answer for every criterion at once. + missing = unexecuted_runtime_targets(criterion, targets, record) + if missing: + criterion_reason = EXECUTION_TARGET_NOT_EXECUTED + why = "no check of the recorded run answering to " + ", ".join( + f"`{target}`" for target in missing + ) + " passed" + else: + criterion_reason = inferred_obligation_unproven(criterion, record) + why = EXECUTION_REASON_TEXT.get(criterion_reason, "") + if not criterion_reason: + bound.add(criterion.req_id) + continue + pending.add(criterion.req_id) + violations.append( + { + "type": "runtime_evidence_not_executed", + "severity": settings.runtime_execution_severity, + "req_id": criterion.req_id, + "source": criterion.source, + "targets": list(targets), + "reason": criterion_reason, + "message": ( + f"[acceptance_evidence] Acceptance criterion `{criterion.req_id}` " + f"({criterion.source}) declares runtime evidence " + f"({', '.join(targets)}) and `runtime_smoke` is enabled, but " + f"{why}. `enabled: true` declares the stage; it does not execute " + "it, so the criterion still rests on nothing. Run " + "`codd verify --runtime` and commit the execution record it " + "writes, declare different evidence in the criterion's " + "`verified_by` column, or downgrade with " + "`acceptance_evidence.runtime_execution_severity: amber`." + ), + } ) - return violations + return violations, frozenset(bound), frozenset(pending) # --------------------------------------------------------------------------- @@ -248,6 +324,7 @@ def _binding_violations( anchors: Mapping[str, set[str]], anchor_offsets: Mapping[str, Mapping[str, tuple[int, ...]]], runtime_bound_ids: frozenset[str], + runtime_pending_ids: frozenset[str] = frozenset(), ) -> tuple[list[dict[str, Any]], dict[str, list[str]]]: """Every acceptance criterion must reach evidence a machine can re-run. @@ -407,7 +484,16 @@ def _binding_violations( violations.append( { "type": "unbound_acceptance", - "severity": settings.unbound_severity, + # A criterion waiting on a run it declared is unbound BECAUSE the + # run has not happened: one defect, one remedy, one severity. At + # `unbound_severity` a strict project would be red before it can + # run the stage that clears it — and the stage is gated on the + # verification it just failed. + "severity": ( + settings.runtime_execution_severity + if criterion.req_id in runtime_pending_ids + else settings.unbound_severity + ), "req_id": criterion.req_id, "source": criterion.source, "message": ( diff --git a/codd/defaults.yaml b/codd/defaults.yaml index 740a4691..2976c1cf 100644 --- a/codd/defaults.yaml +++ b/codd/defaults.yaml @@ -448,12 +448,30 @@ acceptance_evidence: # the one class it has already cleaned up, or run strict with one class still # amber while it migrates. Commented out = follow the mode. # runtime_severity runtime obligation, runtime stage disabled + # runtime_execution_severity runtime stage ENABLED but never recorded a run # unbound_severity AC bound to no machine-checked evidence # vacuous_severity evidence that runs nothing / asserts nothing # param_severity declared parameter absent from its evidence # undeclared_numeric_severity literal number in an AC with no params # reachability_severity evidence off the shipped path / unknown # freshness_severity manual evidence stale against its implementation + # `runtime_execution_severity` is the one class that stays amber even under + # `mode: strict` unless set explicitly. `enabled: true` is a DECLARATION; the + # evidence is the execution record the runtime smoke runner writes to + # /runtime_ledger.json (a PASSING run, against a matching + # `config_digest` and target, inside the freshness window +, for an explicit + # `verified_by: runtime:`, a passing check of that name). Without a + # record the criterion is unproven, not proven wrong — and upgrading CoDD must + # not turn an existing build red on its own. The `unbound_acceptance` raised + # alongside it takes this same severity while the stage is enabled, so a strict + # project is not red before it can run the stage that clears it. + # Commit the ledger like any other evidence artifact; `codd verify --runtime` + # refreshes it. + # + # Age at which a recorded runtime run stops counting as evidence. Unset = + # never expires by clock alone (a configuration change still expires it via + # `config_digest`). Set a number of hours to require a recent run. + # runtime_max_age_hours: # A requirement legitimately spans layers, so several files claiming the same # id is normal. The finding fires only when a requirement is implemented on # BOTH sides of the shipped path and the evidence proves the off-path side. diff --git a/codd/runtime_record.py b/codd/runtime_record.py new file mode 100644 index 00000000..88f04ce8 --- /dev/null +++ b/codd/runtime_record.py @@ -0,0 +1,382 @@ +"""Runtime execution ledger — invariant (a), the runtime stage actually RAN. + +``runtime_smoke.enabled: true`` is a **declaration**. It says the stage is +configured; it says nothing about whether the stage was ever executed. Reading +that boolean as evidence made the cheapest way to clear the acceptance gate +"write one line of YAML" instead of "run the system" — measured on a real +project, flipping it moved ``runtime_evidence_not_executable`` 56 -> 0 and +``unbound_acceptance`` 57 -> 19 without a single runtime check being executed. + +This ledger is the evidence the declaration is not. The runtime smoke runner +writes it when the stage is enabled AND at least one check actually ran; the +acceptance-evidence gate reads it and asks three questions a boolean cannot +answer: did a run happen, did it target what the project targets NOW +(``config_digest``), and is it recent enough to speak for the current state +(``recorded_at``). + +Storage mirrors the ``acceptance_ledger.json`` precedent: a JSON document in the +project's codd directory, pretty-printed because it is reviewed in diffs. Unlike +the manual ledger it is written automatically, and that difference is the point +rather than an inconsistency — a manual record is a HUMAN VERDICT, which nothing +but a human may refresh, while this one records THE FACT THAT A MACHINE RAN, +which only the machine that ran can honestly write. + +A missing or malformed ledger reads as no evidence, never as evidence. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping + +LEDGER_FILENAME = "runtime_ledger.json" +LEDGER_VERSION = 1 + +# The configuration sections that decide WHAT the runtime stage exercises. A +# change to either means an earlier run no longer covers what is declared today. +DIGESTED_SECTIONS = ("runtime_smoke", "runtime") + +# ...minus the keys that decide only where OUTPUT is written. Moving a report +# file does not change what was exercised, and expiring a run over it would train +# owners to ignore the finding. `report.fail_fast` is NOT among them: it decides +# whether the checks after the first failure run at all, which is the plan. +UNDIGESTED_REPORT_KEYS = ("log_to_file", "file_path") + +# Fold only the separators one NAME is spelled with. `.` and `/` are left alone: +# collapsing them would make `a.b`, `a/b` and `ab` one name. +_NORMALISE_RE = re.compile(r"[\s_\-]+") + + +def _normalise(value: str) -> str: + """Fold the spellings one name is written in across a config and a document. + + ``print_sheet`` / ``Print Sheet`` / ``print-sheet`` name the same check. This + is orthography, not domain knowledge: no project, framework or language + vocabulary is involved. + """ + + return _NORMALISE_RE.sub("", str(value)).strip().lower() + + +@dataclass(frozen=True) +class RuntimeCheckRecord: + """One check as the run left it.""" + + name: str + category: str = "" + passed: bool = False + skipped: bool = False + + @property + def executed(self) -> bool: + return not self.skipped + + @property + def discharged(self) -> bool: + """Ran AND passed — the only state that proves anything about a target.""" + + return self.executed and self.passed + + def answers_to(self, target: str) -> bool: + wanted = _normalise(target) + if not wanted: + return False + return wanted in {_normalise(self.name), _normalise(self.category)} + + +@dataclass(frozen=True) +class RuntimeExecutionRecord: + """A recorded execution of the runtime stage.""" + + recorded_at: datetime + passed: bool + config_digest: str + checks: tuple[RuntimeCheckRecord, ...] = () + # The effective target the run went against (``--runtime-base-url`` may have + # pointed it somewhere other than the configured dev server). Empty when the + # run recorded none. + target_url: str = "" + + @property + def executed_checks(self) -> tuple[RuntimeCheckRecord, ...]: + return tuple(check for check in self.checks if check.executed) + + def covers(self, target: str) -> bool: + """Whether a check of this run answering to *target* actually PASSED. + + A failed check is executed but proves nothing, and the ledger is read by + every LATER verification — where "it ran, and it was broken" would + otherwise pass for proof. A failure elsewhere in the same run does not + sink this target: what is asked is whether the named check passed. + """ + + return any(check.discharged and check.answers_to(target) for check in self.checks) + + def age_hours(self, now: datetime | None = None) -> float: + """Hours since the run. NEGATIVE for a record dated in the future. + + Not clamped: a future timestamp (a skewed clock, a hand-edited file) that + read as "zero hours old" would be permanently fresh. + """ + + moment = now or datetime.now(timezone.utc) + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return (moment - self.recorded_at).total_seconds() / 3600.0 + + +def _raw_project_config(project_root: Path | str) -> Mapping[str, Any]: + """The project's OWN codd.yaml, unmerged. + + Deliberately not the defaults-merged view: merging makes the hash depend on + the CoDD version, so upgrading the tool would expire every recorded run + across every project at once. + """ + + import yaml + + from codd.config import find_codd_dir + + root = Path(project_root).resolve() + try: + codd_dir = find_codd_dir(root) + except Exception: # pragma: no cover - config-less project + codd_dir = None + path = (Path(codd_dir) if codd_dir else root / "codd") / "codd.yaml" + try: + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError): + return {} + return loaded if isinstance(loaded, Mapping) else {} + + +def runtime_config_digest(project_root: Path | str) -> str: + """Hash of the configuration that decides what the runtime stage exercises. + + Canonical JSON (sorted keys) of the project's own ``runtime_smoke`` (minus + ``report``) and ``runtime`` sections, so the same configuration hashes the + same on the producing and the consuming side, and any change to what the + stage targets expires the earlier run. + """ + + config = _raw_project_config(project_root) + payload: dict[str, Any] = {} + for section in DIGESTED_SECTIONS: + value = config.get(section) + if section == "runtime_smoke" and isinstance(value, Mapping): + report = value.get("report") + if isinstance(report, Mapping): + kept = { + key: item + for key, item in report.items() + if key not in UNDIGESTED_REPORT_KEYS + } + value = dict(value) + # An empty remainder is dropped, not stored: `report: {file_path: + # ...}` must hash the same as no `report:` at all, or declaring + # where output goes would expire the run after all. + if kept: + value["report"] = kept + else: + value.pop("report", None) + payload[section] = value + canonical = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def ledger_path(project_root: Path | str, codd_dir: Path | None = None) -> Path: + """Location of the ledger: ``/runtime_ledger.json``.""" + + root = Path(project_root).resolve() + if codd_dir is not None: + return Path(codd_dir) / LEDGER_FILENAME + from codd.config import find_codd_dir + + try: + resolved = find_codd_dir(root) + except Exception: # pragma: no cover - config-less project + resolved = None + return (Path(resolved) if resolved else root / "codd") / LEDGER_FILENAME + + +def _strict_bool(raw: Any) -> bool: + """Only a real JSON ``true`` is true. + + ``bool("false")`` is ``True``, so coercing a hand-edited string would read the + word "false" as a pass. + """ + + return raw is True + + +def _parse_recorded_at(raw: Any) -> datetime | None: + text = str(raw or "").strip() + if not text: + return None + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def load_runtime_ledger( + project_root: Path | str, + codd_dir: Path | None = None, +) -> RuntimeExecutionRecord | None: + """Read the ledger. Anything that is not a well-formed record of an actual + run reads as NO EVIDENCE: missing, malformed, undated, written by a version + of the format this CoDD does not know, or listing no executed check at all. + + That last one matters because the file is hand-editable: ``{"checks": []}`` + with today's date would otherwise certify every inferred obligation in the + project. This is not tamper-proofing — a JSON file never is — it is refusing + to read "nothing ran" as "something ran". + """ + + path = ledger_path(project_root, codd_dir) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError): + return None + if not isinstance(payload, Mapping): + return None + if payload.get("version") != LEDGER_VERSION: + return None + recorded_at = _parse_recorded_at(payload.get("recorded_at")) + if recorded_at is None: + return None + + raw_checks = payload.get("checks") + checks: list[RuntimeCheckRecord] = [] + for raw in raw_checks if isinstance(raw_checks, list) else []: + if not isinstance(raw, Mapping): + continue + checks.append( + RuntimeCheckRecord( + name=str(raw.get("name", "")), + category=str(raw.get("category", "")), + passed=_strict_bool(raw.get("passed")), + skipped=_strict_bool(raw.get("skipped")), + ) + ) + if not any(check.executed for check in checks): + return None + return RuntimeExecutionRecord( + recorded_at=recorded_at, + passed=_strict_bool(payload.get("passed")), + config_digest=str(payload.get("config_digest", "")), + checks=tuple(checks), + target_url=str(payload.get("target_url", "")), + ) + + +def write_runtime_ledger( + project_root: Path | str, + record: RuntimeExecutionRecord, + codd_dir: Path | None = None, +) -> Path | None: + """Persist the ledger. A filesystem that refuses the write never fails the run.""" + + path = ledger_path(project_root, codd_dir) + payload = { + "version": LEDGER_VERSION, + "recorded_at": record.recorded_at.astimezone(timezone.utc).isoformat(), + "passed": record.passed, + "config_digest": record.config_digest, + "target_url": record.target_url, + "checks": [ + { + "name": check.name, + "category": check.category, + "passed": check.passed, + "skipped": check.skipped, + } + for check in record.checks + ], + } + body = json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + # Written through a sibling temp file and renamed: two runs racing produce + # one whole record or the other, never a half-read file that would then be + # discarded as malformed. + temp = path.with_name(path.name + f".{os.getpid()}.tmp") + try: + path.parent.mkdir(parents=True, exist_ok=True) + temp.write_text(body, encoding="utf-8") + os.replace(temp, path) + except OSError: + try: + temp.unlink() + except OSError: + pass + return None + return path + + +NOTHING_EXECUTED = "nothing_executed" +WRITE_FAILED = "write_failed" +STALE_RECORD_LEFT = "stale_record_left" + + +def record_runtime_execution( + project_root: Path | str, + checks: Iterable[Any], + target_url: str = "", + codd_dir: Path | None = None, + now: datetime | None = None, +) -> tuple[Path | None, str]: + """Write the ledger for a run that EXECUTED something; otherwise write nothing. + + *checks* are the runner's own result objects (anything carrying ``name`` / + ``category`` / ``passed`` / ``skipped``). A run in which every check was + skipped records nothing: a record of nothing is not a record, and writing one + would re-create the hole this ledger closes. + + Returns ``(path, status)`` — ``status`` is empty on success, and otherwise + says which of the two silences happened, so the caller can stay quiet about + a run that had nothing to record and say so loudly about evidence it lost. + """ + + collected = tuple( + RuntimeCheckRecord( + name=str(getattr(check, "name", "")), + category=str(getattr(check, "category", "") or ""), + passed=bool(getattr(check, "passed", False)), + skipped=bool(getattr(check, "skipped", False)), + ) + for check in checks + ) + if not any(check.executed for check in collected): + return None, NOTHING_EXECUTED + record = RuntimeExecutionRecord( + recorded_at=(now or datetime.now(timezone.utc)).astimezone(timezone.utc), + passed=all(check.passed or check.skipped for check in collected), + config_digest=runtime_config_digest(project_root), + checks=collected, + # Stripped on both sides: the gate compares against the configured URL, + # and YAML keeps whatever whitespace the author typed. + target_url=(target_url or "").strip(), + ) + written = write_runtime_ledger(project_root, record, codd_dir) + if written is not None: + return written, "" + # The write failed. An OLDER record must not survive this run: yesterday's + # green ledger plus today's unrecordable run reads as evidence for a system + # that has since been proved otherwise. Remove it, and if even that is + # refused, say the stale record is still standing. + stale = ledger_path(project_root, codd_dir) + try: + stale.unlink() + except FileNotFoundError: + return None, WRITE_FAILED + except OSError: + return None, STALE_RECORD_LEFT + return None, WRITE_FAILED diff --git a/codd/runtime_smoke/runner.py b/codd/runtime_smoke/runner.py index ecb8b51d..1181efd4 100644 --- a/codd/runtime_smoke/runner.py +++ b/codd/runtime_smoke/runner.py @@ -26,6 +26,10 @@ class SmokeResult: overall_passed: bool markdown_section: str report_path: Path | None = None + ledger_path: Path | None = None + # "" when the record was written or there was nothing to record; + # "write_failed" when the run executed and the evidence could not be stored. + ledger_status: str = "" @property def passed(self) -> bool: @@ -147,9 +151,42 @@ def _finish(runtime_config: RuntimeSmokeConfig, checks: list[CheckResult], *, wr result = SmokeResult(checks=checks, overall_passed=overall_passed, markdown_section=markdown) if write_report and runtime_config.report.log_to_file: result.report_path = write_markdown_report(result, _report_path(runtime_config)) + if runtime_config.enabled: + result.ledger_path, result.ledger_status = _record_execution(runtime_config, checks) return result +def _record_execution( + runtime_config: RuntimeSmokeConfig, checks: list[CheckResult] +) -> tuple[Path | None, str]: + """Leave a machine-readable record that this stage RAN. + + ``runtime_smoke.enabled: true`` only declares the stage; the acceptance + gate needs proof it executed, and a Markdown report under a timestamped + (and switch-offable) path is not something a check can read. So each run + that executed at least one check writes ``/runtime_ledger.json`` + — what ran, whether it passed, the target it went against, and the digest + of the configuration it ran against. A run that executed nothing writes + nothing. + + Never fails the run: a filesystem that refuses the write loses the + evidence, not the verdict. It is not a false green either — the next + verification honestly reports that no run is on record — but it IS a loss, + so the status comes back for the caller to say so out loud. + """ + + from codd.runtime_record import WRITE_FAILED, record_runtime_execution + + try: + return record_runtime_execution( + runtime_config.project_root, + checks, + target_url=runtime_config.dev_server.url or "", + ) + except OSError: # pragma: no cover - defensive: evidence is never worth the run + return None, WRITE_FAILED + + def _should_stop(runtime_config: RuntimeSmokeConfig, checks: list[CheckResult]) -> bool: return bool(runtime_config.report.fail_fast and any(not result.passed and not result.skipped for result in checks)) diff --git a/docs/design/acceptance-evidence-invariant.md b/docs/design/acceptance-evidence-invariant.md index 8aef695d..a77e39a8 100644 --- a/docs/design/acceptance-evidence-invariant.md +++ b/docs/design/acceptance-evidence-invariant.md @@ -40,6 +40,8 @@ declares no verifiable behaviors" as a pass. |---|---| | Parser + resolver (acceptance criteria, evidence refs, params) | `codd/acceptance_evidence.py` | | Gate | `codd/dag/checks/acceptance_evidence.py` (registered check, runs in `codd verify`) | +| Manual-verdict ledger (invariant (d)) | `/acceptance_ledger.json`, written by `codd acceptance record` | +| Runtime **execution** ledger (invariant (a)) | `/runtime_ledger.json`, written by the runtime smoke runner | | Configuration | `acceptance_evidence:` in `codd/defaults.yaml` | ## How a criterion is found (and why no project has to restructure) @@ -73,7 +75,8 @@ markers), and adopting a column makes its declaration authoritative for that row | Stage | Invariant | Finding | Severity | |---|---|---|---| -| 1 | (a) demotion | `runtime_evidence_not_executable` — the criterion declares a runtime obligation and `runtime_smoke` is not enabled, so the stage is skipped silently | red | +| 1a | (a) demotion | `runtime_evidence_not_executable` — the criterion declares a runtime obligation and `runtime_smoke` is not enabled, so the stage is skipped silently | red | +| 1b | (a) execution | `runtime_evidence_not_executed` — the stage is enabled but no run discharged the obligation: no ledger (`no_record`), a ledger written against a different runtime configuration (`config_changed`) or against a different target (`target_changed`), a ledger older than the configured window (`stale`), a run that failed (`run_failed`), or — for an explicitly declared `runtime:` — a ledger in which no check of that name passed (`target_not_executed`) | amber | | 2 | (a) wiring, (d) freshness | `vb_registry_missing`, `unbound_acceptance`, `unresolved_evidence`, `manual_evidence_missing`, `stale_manual_evidence` | red | | 3 | (c) parameters | `param_not_referenced` (red), `undeclared_numeric` (amber) | red / amber | | 4 | (b) shipped path | `off_shipped_path`, `multiple_implementers`, `reachability_unknown` | amber | @@ -96,6 +99,66 @@ the operation's `route:` resolved through the project's declared `filesystem_routes`, or an explicit `entry_file:` on the operation, which always wins over inference. +## A switch is a declaration, not evidence + +Stage 1a asked whether the runtime stage *could* run. That left the mirror-image +hole: a project turns `runtime_smoke.enabled: true` on, never runs it, and the +criterion reads as bound to evidence that has never executed once. Measured on a +real project: flipping that one boolean, with no dev server and no +`codd verify --runtime`, moved `runtime_evidence_not_executable` 56 → 0 and +`unbound_acceptance` 57 → 19. The cheapest way to clear the gate was to declare, +not to verify — which is the incentive exactly backwards. + +So **only an EXECUTED runtime obligation binds**, and the execution has to leave +a record a machine can read: + +- the runtime smoke runner writes `/runtime_ledger.json` when the + stage is enabled **and at least one check actually ran** (an all-skipped run, + or a disabled stage, writes nothing — a record of nothing is not a record); +- the ledger holds `recorded_at` (UTC), `passed`, the per-check + `name` / `category` / `passed` / `skipped`, and `config_digest`, a hash of the + project's own `runtime_smoke` + `runtime` configuration as it stood at run + time. Change what the stage targets and the old run stops certifying it; +- a missing, malformed or unversioned ledger reads as **no evidence**, never as + evidence — the same rule the manual ledger already follows, and so does a + ledger recording no executed check at all, which a hand-written file could + otherwise use to certify everything; +- the ledger also records the **target** the run went against (the effective + dev-server URL, which `--runtime-base-url` may have overridden). A run aimed + at a different deployment is not evidence about this one; +- a run that FAILED does not discharge anything. It is still *executed* — the + finding says `run_failed`, not "never ran" — but a failing run recorded on + Monday must not read as green evidence on Tuesday. Within the run itself Step + 8 reports the failure in red; the ledger is read by every LATER verification, + and that is where "it ran, and it was broken" would otherwise pass for proof. + A failure in one check does not sink another: an explicitly named + `runtime:` is discharged when the check answering to that name PASSED, + whatever else in the same run did not. + +Target matching follows the asymmetry the obligation itself is built on: an +**explicit** `verified_by: runtime:` names a case, so a ledger with no +passing check of that `name` or `category` has not discharged it; an +**inferred** obligation (an `operation_flow.` reference in a project that +never adopted the `verified_by` column) declares no such mapping, so a passing +run of the project's own stage satisfies it. CoDD does not invent a +correspondence the project never wrote down. + +The digest is taken over the project's **own** `runtime_smoke` and `runtime` +sections, read from `codd.yaml` itself rather than from the defaults-merged view +— so upgrading CoDD does not expire every recorded run in every project at once. +Only `report.log_to_file` and `report.file_path` are left out of it: they decide +where output is written, not what is exercised. `report.fail_fast` stays IN, +because it decides whether the checks after the first failure run at all, and +that is the plan rather than the paperwork. + +Like the manual ledger, this file is a project artifact meant to be committed +and read in diffs: it is the standing answer to "when did the declared runtime +evidence last actually run, and against what configuration?". Unlike the manual +ledger it is written automatically — and that is consistent, not a contradiction: +a manual ledger records a *human verdict*, which nothing but a human may refresh, +while this one records *the fact that a machine ran*, which only the machine that +ran can honestly write. + ## What stays outside - `manual` evidence still needs a person. What is general is the machinery @@ -104,3 +167,44 @@ wins over inference. criterion ride on a manual record alone. - (c) only bites on criteria that declare parameters. An undeclared literal is amber (`undeclared_numeric`) — a nudge to declare, not an accusation. +- Runtime execution evidence is **not yet bound to implementation content**, the + way a manual record is. A recorded run expires when the runtime configuration + changes (`config_digest`) or when `runtime_max_age_hours` says so, but a + rewritten implementation under an unchanged configuration does not invalidate + it. Closing that would need a content digest both the runner and the gate can + compute, and the runner has no dependency graph; until then the honest + statement is the narrow one: the ledger proves the stage RAN, not that it ran + against today's code. +- `runtime_evidence_not_executed` is amber even under `mode: strict` + (`runtime_execution_severity`). A project that never persisted the ledger is + not thereby proven wrong — it is unproven — and upgrading CoDD must not turn + an existing green build red on its own. The `unbound_acceptance` that + accompanies it takes the SAME severity, for one criterion whose only declared + binding is that unexecuted runtime obligation: the unbound-ness is *caused by* + the not-executed-ness, one defect with one remedy ("run it"), and pricing it + at `unbound_severity` would strand a strict project — red before Step 8 runs, + and Step 8 is what writes the record that would clear it. When the stage is + OFF the ordinary `unbound_severity` applies, because such a project has no + runtime path to run at all. +- A ledger the runner could not write (a read-only checkout, a full disk) is a + lost artifact, not a false green: the run's own record simply does not exist, + and the next verification honestly reports `no_record`. Step 8 says so loudly + on stderr rather than failing the run. The one branch that IS a false green — + an EARLIER, passing record surviving a run that could not overwrite it, so + yesterday's verdict reads as today's — is not left standing: the stale record + is removed, and if even that is refused, Step 8 FAILS and names the file. +- A `--runtime-skip` narrows what a run proves, and the ledger records exactly + which checks were skipped, but the check does not yet reason about the gap: a + run in which one category passed and the rest were skipped by the operator + still reads as a passing run for an inferred obligation. What is auditable is + in the file; what is judged is only pass/fail. +- The ledger is a JSON file in the repository, so it is **hand-editable**, and + nothing here pretends otherwise. What the loader refuses is the accident and + the shortcut, not the forgery: an unknown format version, a record listing no + executed check, a `"passed": "false"` that Python's `bool()` would read as + true. Proof against a deliberate edit would need signing and provenance, which + is a different mechanism from this one. +- Runtime evidence is **not bound to implementation content** (see above). This + is the largest remaining gap and it is deliberate for now: closing it needs a + digest both the runner and the gate can compute over the same file set, and + the runner has no dependency graph. diff --git a/docs/requirements/system-requirements.md b/docs/requirements/system-requirements.md index f7a59e1b..1ce5d3f2 100644 --- a/docs/requirements/system-requirements.md +++ b/docs/requirements/system-requirements.md @@ -253,3 +253,6 @@ column the invariant asks every project to write. | ID | Requirement | Acceptance criteria | verified_by | params | | --- | --- | --- | --- | --- | | AC-ACC-1 | Adopting the acceptance-evidence check must never turn an existing project red by itself | On a brownfield project with acceptance criteria, no `verified_by` column and no verifiable-behavior registry, the check reports its findings and still passes: severity `amber`, status `warn`, and the missing registry is named rather than silently tolerated | test:test_acceptance_evidence | | +| AC-ACC-2 | A configuration switch must never stand in for evidence: declaring the runtime stage is not running it | On a project whose acceptance criterion declares runtime evidence and whose `runtime_smoke` section is enabled, with no execution recorded, the check reports `runtime_evidence_not_executed` and still reports the criterion as bound to nothing — the same two findings it reports when the stage is switched off | test:test_acceptance_evidence | | +| AC-ACC-3 | A recorded runtime execution counts as evidence only while it still speaks for the project as it stands now | With an execution record present, the runtime obligation is discharged; the same record stops discharging it once the runtime configuration it ran against has changed, once it is older than the project's declared freshness window, or, for an explicitly named runtime case, when no check answering to that name actually ran | test:test_acceptance_evidence | | +| AC-ACC-4 | Upgrading CoDD must not turn an existing green build red on its own | The `runtime_evidence_not_executed` finding is amber even under `mode: strict` unless the project explicitly sets `runtime_execution_severity`, and the freshness window is unset by default | test:test_acceptance_evidence | | diff --git a/pyproject.toml b/pyproject.toml index 2dd6d9a3..e6746b76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "codd-dev" -version = "3.41.0" +version = "3.42.0" description = "CoDD: Coherence-Driven Development — cross-artifact change impact analysis" readme = "README.md" license = "MIT" diff --git a/tests/dag/test_acceptance_evidence.py b/tests/dag/test_acceptance_evidence.py index 2704db5c..c2239a40 100644 --- a/tests/dag/test_acceptance_evidence.py +++ b/tests/dag/test_acceptance_evidence.py @@ -12,6 +12,7 @@ import json import textwrap +from datetime import datetime, timedelta, timezone from pathlib import Path import yaml @@ -295,6 +296,334 @@ def test_runtime_severity_is_configurable_to_amber(tmp_path): assert found and found[0]["severity"] == "amber" +# --------------------------------------------------------------------------- +# stage 1b — (a) the stage is ON, but was it ever RUN? +# --------------------------------------------------------------------------- + +RUNTIME_ROW = "| R-1 | 印刷用の一覧を出す | 15人が並ぶ `operation_flow.sheet_print` |\n" +RUNTIME_OPERATION = [{"id": "sheet_print", "actor": "operator", "verb": "print", "target": "sheet"}] + + +def _runtime_project(tmp_path: Path, **kwargs) -> Path: + return _write_project( + tmp_path, + requirements=_requirements_doc(RUNTIME_ROW), + operations=RUNTIME_OPERATION, + runtime_smoke={"enabled": True, "dev_server": {"url": "http://localhost:3000"}}, + **kwargs, + ) + + +def test_enabled_runtime_stage_with_no_recorded_run_is_not_evidence(tmp_path): + """`enabled: true` is a DECLARATION. Declaring a stage does not execute it.""" + + root = _runtime_project(tmp_path) + result = _run_check(root) + found = _violations(result, "runtime_evidence_not_executed") + assert [item["req_id"] for item in found] == ["R-1"] + assert found[0]["reason"] == "no_record" + # ...and the criterion is bound to nothing, which is a SECOND defect with a + # different remedy — exactly as when the stage is switched off. + assert [item["req_id"] for item in _violations(result, "unbound_acceptance")] == ["R-1"] + # The switched-OFF finding is not raised: the stage is on, it just never ran. + assert _violations(result, "runtime_evidence_not_executable") == [] + + +def test_a_recorded_run_discharges_the_runtime_obligation(tmp_path): + root = _runtime_project(tmp_path) + _record_runtime_run(root) + result = _run_check(root) + assert _violations(result, "runtime_evidence_not_executed") == [] + assert _violations(result, "runtime_evidence_not_executable") == [] + assert _violations(result, "unbound_acceptance") == [] + + +def test_a_run_recorded_against_a_different_configuration_is_not_evidence(tmp_path): + """Change what the stage targets and the earlier run stops certifying it.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root, config_digest="0" * 64) + result = _run_check(root) + found = _violations(result, "runtime_evidence_not_executed") + assert [item["req_id"] for item in found] == ["R-1"] + assert found[0]["reason"] == "config_changed" + assert [item["req_id"] for item in _violations(result, "unbound_acceptance")] == ["R-1"] + + +def test_a_run_older_than_the_projects_freshness_window_is_not_evidence(tmp_path): + root = _runtime_project( + tmp_path, + extra_config={"acceptance_evidence": {"runtime_max_age_hours": 6}}, + ) + _record_runtime_run(root, hours_ago=48) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert [item["req_id"] for item in found] == ["R-1"] + assert found[0]["reason"] == "stale" + + +def test_a_recent_run_inside_the_freshness_window_still_counts(tmp_path): + root = _runtime_project( + tmp_path, + extra_config={"acceptance_evidence": {"runtime_max_age_hours": 6}}, + ) + _record_runtime_run(root, hours_ago=1) + assert _violations(_run_check(root), "runtime_evidence_not_executed") == [] + + +def test_no_freshness_window_means_an_old_run_still_counts(tmp_path): + """Opt-in: a project that never declared a window is not expired by the clock.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root, hours_ago=24 * 90) + assert _violations(_run_check(root), "runtime_evidence_not_executed") == [] + + +def _explicit_runtime_project(tmp_path: Path) -> Path: + return _write_project( + tmp_path, + requirements=_requirements_doc( + "| R-1 | 出す | 15人が並ぶ `operation_flow.sheet_print` | runtime:print_sheet |\n", + header="| ID | 要件 | 検収条件 | verified_by |\n| --- | --- | --- | --- |\n", + ), + operations=RUNTIME_OPERATION, + runtime_smoke={"enabled": True, "dev_server": {"url": "http://localhost:3000"}}, + ) + + +def test_an_explicitly_named_runtime_case_needs_a_check_that_actually_ran(tmp_path): + root = _explicit_runtime_project(tmp_path) + _record_runtime_run(root, checks=[("something else", "connectivity", True, True)]) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert [item["req_id"] for item in found] == ["R-1"] + assert found[0]["reason"] == "target_not_executed" + + +def test_a_skipped_check_does_not_discharge_the_case_it_is_named_after(tmp_path): + root = _explicit_runtime_project(tmp_path) + _record_runtime_run(root, checks=[("print_sheet", "e2e", False, False), ("other", "db", True, True)]) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "target_not_executed" + + +def test_the_named_case_matches_its_check_across_spellings(tmp_path): + """`print_sheet` / `Print Sheet` / `print-sheet` are one name, not three.""" + + root = _explicit_runtime_project(tmp_path) + _record_runtime_run(root, checks=[("Print Sheet", "e2e", True, True)]) + assert _violations(_run_check(root), "runtime_evidence_not_executed") == [] + + +def test_an_inferred_obligation_needs_a_run_but_not_a_named_check(tmp_path): + """The project never declared which check covers the operation, so CoDD does + not invent the correspondence — any recorded run discharges it.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root, checks=[("whatever the project named it", "db", True, True)]) + assert _violations(_run_check(root), "runtime_evidence_not_executed") == [] + + +def test_not_executed_stays_amber_even_under_strict(tmp_path): + """Upgrading CoDD must not turn an existing build red on its own: an absent + record leaves the criterion UNPROVEN, which is not the same as proven wrong.""" + + root = _runtime_project(tmp_path, strict=True) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["severity"] == "amber" + + +def test_not_executed_severity_is_configurable_to_red(tmp_path): + root = _runtime_project( + tmp_path, + extra_config={"acceptance_evidence": {"runtime_execution_severity": "red"}}, + ) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["severity"] == "red" + + +def test_a_disabled_stage_reports_the_switch_not_the_missing_record(tmp_path): + """Two holes, two remedies: turn the stage on vs. run the stage.""" + + root = _write_project( + tmp_path, + requirements=_requirements_doc(RUNTIME_ROW), + operations=RUNTIME_OPERATION, + runtime_smoke={"enabled": False}, + ) + result = _run_check(root) + assert _violations(result, "runtime_evidence_not_executable") + assert _violations(result, "runtime_evidence_not_executed") == [] + + +def test_a_strict_project_waiting_on_a_run_is_amber_not_red(tmp_path): + """The migration path has to be walkable. + + A criterion waiting on a run it declared is unbound BECAUSE the run has not + happened. Pricing that at `unbound_severity` would leave a strict project red + before it can run the stage — and `codd verify --runtime` runs the stage only + after the verification it just failed. One defect, one remedy, one severity. + """ + + root = _runtime_project(tmp_path, strict=True) + result = _run_check(root) + assert result.passed is True + assert result.status == "warn" + unbound = _violations(result, "unbound_acceptance") + assert [item["req_id"] for item in unbound] == ["R-1"] + assert unbound[0]["severity"] == "amber" + + +def test_a_strict_project_with_the_stage_off_keeps_the_ordinary_unbound_severity(tmp_path): + """No runtime path at all is a different defect: there is nothing to run.""" + + root = _write_project( + tmp_path, + requirements=_requirements_doc(RUNTIME_ROW), + operations=RUNTIME_OPERATION, + runtime_smoke={"enabled": False}, + strict=True, + ) + result = _run_check(root) + assert result.passed is False + unbound = _violations(result, "unbound_acceptance") + assert unbound and unbound[0]["severity"] == "red" + + +def test_a_failed_run_does_not_prove_an_inferred_obligation(tmp_path): + """It executed — so this is not `no_record` — but a failing run is not proof. + + Within the run Step 8 reports the failure in red. The ledger is read by every + LATER verification, and that is where "it ran, and it was broken" would + otherwise pass for evidence. + """ + + root = _runtime_project(tmp_path) + _record_runtime_run(root, checks=[("Dev server up", "dev-server", True, False)]) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert [item["req_id"] for item in found] == ["R-1"] + assert found[0]["reason"] == "run_failed" + + +def test_a_named_case_that_passed_survives_a_failure_beside_it(tmp_path): + """A database check failing does not un-prove the case that passed.""" + + root = _explicit_runtime_project(tmp_path) + _record_runtime_run( + root, + checks=[("print_sheet", "e2e", True, True), ("DB up", "db", True, False)], + ) + assert _violations(_run_check(root), "runtime_evidence_not_executed") == [] + + +def test_a_run_against_a_different_target_is_not_evidence_about_this_one(tmp_path): + """`--runtime-base-url` can point a run at another deployment.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root, target_url="http://staging.internal:3000") + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert [item["req_id"] for item in found] == ["R-1"] + assert found[0]["reason"] == "target_changed" + + +def test_moving_the_report_file_does_not_expire_a_run(tmp_path): + """`report:` decides where OUTPUT goes, not what is exercised.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root) + config_path = root / "codd" / "codd.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["runtime_smoke"]["report"] = {"file_path": "somewhere/else.md"} + config_path.write_text(yaml.safe_dump(config, allow_unicode=True), encoding="utf-8") + assert _violations(_run_check(root), "runtime_evidence_not_executed") == [] + + +def test_a_record_dated_in_the_future_is_not_fresh(tmp_path): + """A clock skew or a hand-edited date must not buy permanent freshness.""" + + root = _runtime_project( + tmp_path, + extra_config={"acceptance_evidence": {"runtime_max_age_hours": 6}}, + ) + _record_runtime_run(root, hours_ago=-240) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "stale" + + +def test_a_record_of_no_executed_check_is_not_a_record(tmp_path): + """The file is hand-editable; an empty check list must not certify anything.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root) + ledger = root / "codd" / "runtime_ledger.json" + payload = json.loads(ledger.read_text(encoding="utf-8")) + payload["checks"] = [] + ledger.write_text(json.dumps(payload), encoding="utf-8") + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "no_record" + + +def test_a_record_written_by_an_unknown_format_version_is_not_read(tmp_path): + root = _runtime_project(tmp_path) + _record_runtime_run(root) + ledger = root / "codd" / "runtime_ledger.json" + payload = json.loads(ledger.read_text(encoding="utf-8")) + payload["version"] = 99 + ledger.write_text(json.dumps(payload), encoding="utf-8") + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "no_record" + + +def test_name_folding_does_not_merge_names_that_differ_by_a_separator(tmp_path): + """`print.sheet` and `printsheet` are two names; only spacing folds.""" + + root = _explicit_runtime_project(tmp_path) + _record_runtime_run(root, checks=[("print.sheet", "e2e", True, True)]) + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "target_not_executed" + + +def test_changing_fail_fast_does_expire_a_run(tmp_path): + """`fail_fast` decides whether the checks after the first failure run at all, + which is the execution PLAN, not the output location.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root) + config_path = root / "codd" / "codd.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["runtime_smoke"]["report"] = {"fail_fast": True} + config_path.write_text(yaml.safe_dump(config, allow_unicode=True), encoding="utf-8") + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "config_changed" + + +def test_a_record_that_names_no_target_does_not_certify_a_configured_one(tmp_path): + """Every record CoDD writes carries the target; a blank one came from elsewhere.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root, target_url="") + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "target_changed" + + +def test_the_word_false_is_not_a_pass(tmp_path): + """The ledger is hand-editable and `bool("false")` is True.""" + + root = _runtime_project(tmp_path) + _record_runtime_run(root) + ledger = root / "codd" / "runtime_ledger.json" + payload = json.loads(ledger.read_text(encoding="utf-8")) + payload["passed"] = "false" + ledger.write_text(json.dumps(payload), encoding="utf-8") + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "run_failed" + + +def test_a_malformed_execution_record_reads_as_no_evidence(tmp_path): + root = _runtime_project(tmp_path) + (root / "codd" / "runtime_ledger.json").write_text("{not json", encoding="utf-8") + found = _violations(_run_check(root), "runtime_evidence_not_executed") + assert found and found[0]["reason"] == "no_record" + + def test_project_without_acceptance_criteria_is_dormant_not_green(tmp_path): """No criteria == nothing to certify: skip (checked_count 0), never a 'verified' pass that the materiality overlay would read as a clean run.""" @@ -365,6 +694,49 @@ def test_dag_build_is_not_required_for_the_operation_universe(tmp_path): RUNTIME_ON = {"enabled": True, "dev_server": {"url": "http://localhost:3000"}} +def _record_runtime_run( + root: Path, + *, + checks: list[tuple[str, str, bool, bool]] | None = None, + hours_ago: float = 0.0, + config_digest: str | None = None, + target_url: str | None = None, +) -> Path: + """Seed the execution record a real runtime run would have left behind. + + ``checks`` are ``(name, category, executed, passed)`` quadruples — the + miniature of what the runner writes. Nothing here names a framework, a URL or + a project: the record is a list of check names the project itself chose. + """ + + from codd.runtime_record import ( + RuntimeCheckRecord, + RuntimeExecutionRecord, + runtime_config_digest, + write_runtime_ledger, + ) + + rows = checks if checks is not None else [("Smoke connectivity", "connectivity", True, True)] + if target_url is None: + # By default the run went where the project points: a real run records + # the effective dev-server URL, and a blank one means "somewhere else". + config = yaml.safe_load((root / "codd" / "codd.yaml").read_text(encoding="utf-8")) or {} + target_url = str(((config.get("runtime_smoke") or {}).get("dev_server") or {}).get("url") or "") + record = RuntimeExecutionRecord( + recorded_at=datetime.now(timezone.utc) - timedelta(hours=hours_ago), + passed=all(passed or not executed for _name, _category, executed, passed in rows), + config_digest=(config_digest if config_digest is not None else runtime_config_digest(root)), + checks=tuple( + RuntimeCheckRecord(name=name, category=category, passed=passed, skipped=not executed) + for name, category, executed, passed in rows + ), + target_url=target_url, + ) + written = write_runtime_ledger(root, record) + assert written is not None + return written + + def test_no_vb_registry_is_red_when_the_project_has_criteria_to_certify(tmp_path): """The "empty registry passes with a notice" rule made empty the safest state.""" @@ -400,7 +772,9 @@ def test_criterion_bound_to_nothing_is_red(tmp_path): runtime_smoke=RUNTIME_ON, files={"tests/unit/other.test.ts": "test('other', () => {});\n"}, ) - # runtime_smoke is ON, so the operation anchor is executable evidence. + # runtime_smoke is ON *and a run is on record*, so the operation anchor is + # executed evidence. (Enabled alone is not — see the stage-1b tests.) + _record_runtime_run(root) assert _violations(_run_check(root), "unbound_acceptance") == [] root2 = _write_project( @@ -410,6 +784,7 @@ def test_criterion_bound_to_nothing_is_red(tmp_path): operations=SHEET_OPERATION, runtime_smoke=RUNTIME_ON, ) + _record_runtime_run(root2) found = _violations(_run_check(root2), "unbound_acceptance") assert [item["req_id"] for item in found] == ["R-1"] diff --git a/tests/test_runtime_smoke.py b/tests/test_runtime_smoke.py index 71804019..d39dec65 100644 --- a/tests/test_runtime_smoke.py +++ b/tests/test_runtime_smoke.py @@ -1454,3 +1454,230 @@ def test_t42_doctor_accepts_business_screen_with_ancestor_navigation(tmp_path): assert result.exit_code == 0 assert "escape route/navigation evidence" not in result.output + + +# --------------------------------------------------------------------------- +# execution ledger — the record that turns a declaration into evidence +# --------------------------------------------------------------------------- + + +def _ledger(project): + from codd.runtime_record import load_runtime_ledger + + return load_runtime_ledger(project) + + +def test_t43_an_executed_run_records_what_ran_and_against_which_config(tmp_path, monkeypatch): + project = _project( + tmp_path, + """ +runtime_smoke: + enabled: true + db_check: + command: "check-db" + dev_server: + url: "http://127.0.0.1:3000" + report: + log_to_file: false +""", + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="ok", stderr=""), + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.httpx.get", + lambda *args, **kwargs: SimpleNamespace(status_code=200), + ) + + result = run_runtime_smoke(project) + + assert result.ledger_path is not None and result.ledger_path.exists() + record = _ledger(project) + assert record is not None + # A run that FAILED is still a run: the verdict is recorded as it was, and + # Step 8 already reported the failure in red. What the ledger certifies is + # execution, not success. + assert record.passed == result.overall_passed + assert record.executed_checks # at least one check actually ran + + # The digest is the one the acceptance gate recomputes from the same config. + from codd.runtime_record import runtime_config_digest + + assert record.config_digest == runtime_config_digest(project) + # ...and the record names the target the run actually went against. + assert record.target_url == "http://127.0.0.1:3000" + + +def test_t44_a_disabled_stage_records_nothing(tmp_path): + project = _project(tmp_path, "runtime_smoke:\n enabled: false\n") + + result = run_runtime_smoke(project) + + assert result.ledger_path is None + assert _ledger(project) is None + + +def test_t45_a_run_that_executed_nothing_records_nothing(tmp_path): + """A record of nothing is not a record — it would re-open the hole.""" + + project = _project( + tmp_path, + """ +runtime_smoke: + enabled: true + db_check: + command: "check-db" + dev_server: + url: "http://127.0.0.1:3000" + e2e: + command: "npx playwright test" + report: + log_to_file: false +""", + ) + + result = run_runtime_smoke( + project, skip_checks=["db", "dev-server", "connectivity", "e2e", "crud-flow", "action-outcome", "global-action"] + ) + + assert all(check.skipped for check in result.checks) + assert result.ledger_path is None + assert _ledger(project) is None + + +def test_t46_a_lost_execution_record_is_reported_not_swallowed(tmp_path, monkeypatch): + """Losing the evidence is not a false green, but it must not be silent.""" + + project = _project( + tmp_path, + """ +runtime_smoke: + enabled: true + db_check: + command: "check-db" + dev_server: + url: "http://127.0.0.1:3000" + report: + log_to_file: false +""", + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="ok", stderr=""), + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.httpx.get", + lambda *args, **kwargs: SimpleNamespace(status_code=200), + ) + monkeypatch.setattr("codd.runtime_record.write_runtime_ledger", lambda *a, **k: None) + + result = run_runtime_smoke(project) + + assert result.ledger_path is None + assert result.ledger_status == "write_failed" + assert _ledger(project) is None + + +def test_t47_a_run_pointed_elsewhere_records_where_it_actually_went(tmp_path, monkeypatch): + project = _project( + tmp_path, + """ +runtime_smoke: + enabled: true + db_check: + command: "check-db" + dev_server: + url: "http://127.0.0.1:3000" + report: + log_to_file: false +""", + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="ok", stderr=""), + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.httpx.get", + lambda *args, **kwargs: SimpleNamespace(status_code=200), + ) + + run_runtime_smoke(project, base_url_override="http://staging.internal:8080") + + record = _ledger(project) + assert record is not None + assert record.target_url == "http://staging.internal:8080" + + +def test_t48_a_failed_write_does_not_leave_an_older_record_standing(tmp_path, monkeypatch): + """Yesterday's green ledger must not survive a run that could not overwrite it.""" + + project = _project( + tmp_path, + """ +runtime_smoke: + enabled: true + db_check: + command: "check-db" + dev_server: + url: "http://127.0.0.1:3000" + report: + log_to_file: false +""", + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="ok", stderr=""), + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.httpx.get", + lambda *args, **kwargs: SimpleNamespace(status_code=200), + ) + + run_runtime_smoke(project) # leaves a record behind + assert _ledger(project) is not None + + monkeypatch.setattr("codd.runtime_record.write_runtime_ledger", lambda *a, **k: None) + result = run_runtime_smoke(project) + + assert result.ledger_status == "write_failed" + assert _ledger(project) is None # the older record is gone, not standing + + +def test_t49_an_unremovable_older_record_is_reported_as_such(tmp_path, monkeypatch): + project = _project( + tmp_path, + """ +runtime_smoke: + enabled: true + db_check: + command: "check-db" + dev_server: + url: "http://127.0.0.1:3000" + report: + log_to_file: false +""", + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.subprocess.run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="ok", stderr=""), + ) + monkeypatch.setattr( + "codd.runtime_smoke.checks.httpx.get", + lambda *args, **kwargs: SimpleNamespace(status_code=200), + ) + run_runtime_smoke(project) + + monkeypatch.setattr("codd.runtime_record.write_runtime_ledger", lambda *a, **k: None) + + real_unlink = Path.unlink + + def refuse(self, *args, **kwargs): + if self.name == "runtime_ledger.json": + raise PermissionError("read-only") + return real_unlink(self, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", refuse) + result = run_runtime_smoke(project) + + assert result.ledger_status == "stale_record_left"