diff --git a/benchmarks/real_world/README.md b/benchmarks/real_world/README.md index 1132558..3a6096c 100644 --- a/benchmarks/real_world/README.md +++ b/benchmarks/real_world/README.md @@ -248,18 +248,29 @@ scoring, so a candidate cannot improve its score by omitting hard PRs. Missing, `unknown`, and `not_evaluable` labels are never treated as negatives. Versioned scope membership and source hashes live under `scopes/`. -Schema-v3 runner output calls its current measurement -`cold_no_cache_analyzer_wall`. It is the wall time of a one-shot analyzer process -run with `--no-cache`; it is **not incremental latency** and is ineligible for the -incremental p95 gate. Historical `incremental_seconds` values are accepted only -through the legacy adapter and reported as unattested. Real warm no-change and -one-file update measurements require backend cache-reuse/invalidation telemetry. -Checked-in legacy prediction JSONL remains scoreable through the strict legacy -adapter, but historical schema-v1/v2 manifests cannot be supplied as a -schema-v3 `--prediction-manifest`; those evaluations explicitly remain -unattested. A schema-v3 manifest records and validates secure execution flags; -dry-run or `--allow-upstream-execution` output is explicitly ineligible for the -official secure score. +Prediction rows remain schema v3 and call their current measurement +`cold_no_cache_analyzer_wall`. It is the wall time of the one-shot analyzer +subprocess run with `--no-cache`; it is **not incremental latency**. Runner +manifest schema v4 records exactly four phase states. `baseline_target_preparation` +starts immediately before detached target/baseline worktree materialization and +ends after the local diff is written. `cold_build` is the analyzer subprocess wall +time. `warm_no_change` and `one_file_incremental_update` are explicitly +`not_measured` until a backend attests cache reuse and invalidation. Peak process- +tree RSS and backend cache size are likewise `not_measured`; they are never +inferred from unrelated process counters or filesystem size. Preparation must be +measured before cold analysis can be measured. A PR run with status `unresolved` +attests no measured phases: failures clear preparation/cold telemetry and their +corresponding timing fields while retaining truthful total and internal runner +diagnostics. Only measured, finite samples from completed or partial predictions +are aggregated, and `incremental_valid` remains false. + +Historical `incremental_seconds` values are accepted only through the legacy +adapter and reported as unattested. Checked-in legacy prediction JSONL remains +scoreable, and schema-v3 manifests remain readable, but historical schema-v1/v2 +manifests cannot be supplied as `--prediction-manifest`. Manifest v4 continues +to record and validate secure execution flags; dry-run or +`--allow-upstream-execution` output is explicitly ineligible for the official +secure score. ### Current candidate runner diff --git a/benchmarks/real_world/evaluate.py b/benchmarks/real_world/evaluate.py index 481a497..92b0cf8 100644 --- a/benchmarks/real_world/evaluate.py +++ b/benchmarks/real_world/evaluate.py @@ -461,6 +461,23 @@ def percentile(fraction: float) -> float: "command", "performance_protocol", } +_PERFORMANCE_PHASES = ( + "baseline_target_preparation", + "cold_build", + "warm_no_change", + "one_file_incremental_update", +) +_MEASURED_PHASE_TIMING_FIELDS = { + "baseline_target_preparation": "baseline_target_preparation", + "cold_build": "analyzer", +} +_UNMEASURED_PERFORMANCE_PHASES = {"warm_no_change", "one_file_incremental_update"} +_RESOURCE_METRICS = ("peak_rss_bytes", "cache_size_bytes") +_V4_PREDICTION_STATUS_BY_MANIFEST_STATUS = { + "completed": "completed", + "completed_with_unresolved": "partial", + "unresolved": "unresolved", +} def _nonempty_string(value: object, field: str) -> str: @@ -573,9 +590,29 @@ def _validate_manifest_configuration(manifest: dict[str, Any]) -> None: raise BenchmarkSchemaError("prediction manifest PR filters are invalid") -def _validate_manifest_prs( # noqa: PLR0912 - fail-closed PR schema checks are explicit - value: object, prediction_count: int -) -> None: +def _validate_measurement_state(value: object, field: str) -> float | None: + if not isinstance(value, dict): + raise BenchmarkSchemaError(f"prediction manifest {field} must be an object") + status = value.get("status") + if status == "measured": + if set(value) != {"status", "seconds"}: + raise BenchmarkSchemaError( + f"prediction manifest {field} measured state requires only seconds" + ) + return finite_nonnegative(value["seconds"], f"prediction manifest {field}.seconds") + if status == "not_measured": + if set(value) != {"status", "reason"}: + raise BenchmarkSchemaError( + f"prediction manifest {field} not_measured state forbids samples" + ) + _nonempty_string(value["reason"], f"{field}.reason") + return None + raise BenchmarkSchemaError(f"prediction manifest {field} status is invalid") + + +def _validate_manifest_prs( # noqa: PLR0912, PLR0915 - schema checks stay explicit + value: object, prediction_count: int, schema_version: int +) -> dict[str, list[float]]: required = { "repository", "pr", @@ -587,6 +624,8 @@ def _validate_manifest_prs( # noqa: PLR0912 - fail-closed PR schema checks are "status", "timing_seconds", } + if schema_version == 4: + required.add("phase_telemetry") optional = { "reason", "candidate_endpoint_count", @@ -595,6 +634,7 @@ def _validate_manifest_prs( # noqa: PLR0912 - fail-closed PR schema checks are } if not isinstance(value, list) or len(value) != prediction_count: raise BenchmarkSchemaError("prediction manifest PR records are invalid") + phase_samples: dict[str, list[float]] = {phase: [] for phase in _PERFORMANCE_PHASES} for item in value: if not isinstance(item, dict) or not required <= set(item) <= required | optional: raise BenchmarkSchemaError("prediction manifest PR record fields are invalid") @@ -619,6 +659,48 @@ def _validate_manifest_prs( # noqa: PLR0912 - fail-closed PR schema checks are raise BenchmarkSchemaError("prediction manifest PR timing is invalid") for name, timing_value in timing.items(): finite_nonnegative(timing_value, f"prediction manifest prs.timing_seconds.{name}") + if schema_version == 4: + telemetry = item["phase_telemetry"] + if not isinstance(telemetry, dict) or set(telemetry) != set(_PERFORMANCE_PHASES): + raise BenchmarkSchemaError("prediction manifest PR phase telemetry is invalid") + measured_samples: dict[str, float | None] = {} + for phase in _PERFORMANCE_PHASES: + sample = _validate_measurement_state( + telemetry[phase], f"prs.phase_telemetry.{phase}" + ) + measured_samples[phase] = sample + if phase in _UNMEASURED_PERFORMANCE_PHASES and sample is not None: + raise BenchmarkSchemaError( + f"prediction manifest prs.phase_telemetry.{phase} must be not_measured" + ) + if sample is not None: + phase_samples[phase].append(sample) + preparation_sample = measured_samples["baseline_target_preparation"] + cold_sample = measured_samples["cold_build"] + if cold_sample is not None and preparation_sample is None: + raise BenchmarkSchemaError( + "prediction manifest PR cold phase requires measured preparation" + ) + if item["status"] == "unresolved" and any( + measured_samples[phase] is not None for phase in _MEASURED_PHASE_TIMING_FIELDS + ): + raise BenchmarkSchemaError( + "unresolved prediction manifest PR must not attest measured phases" + ) + if item["status"] != "unresolved" and any( + measured_samples[phase] is None for phase in _MEASURED_PHASE_TIMING_FIELDS + ): + raise BenchmarkSchemaError( + "completed prediction manifest PR must measure preparation and cold phases" + ) + for phase, timing_field in _MEASURED_PHASE_TIMING_FIELDS.items(): + sample = measured_samples[phase] + if (sample is None and timing_field in timing) or ( + sample is not None and timing.get(timing_field) != sample + ): + raise BenchmarkSchemaError( + f"prediction manifest PR {phase} telemetry does not match timing" + ) if item["status"] == "unresolved": _nonempty_string(item.get("reason"), "prs.reason") for field in ("candidate_endpoint_count", "effect_evidence_count"): @@ -634,13 +716,48 @@ def _validate_manifest_prs( # noqa: PLR0912 - fail-closed PR schema checks are raise BenchmarkSchemaError( "prediction manifest PR candidate confidence counts are invalid" ) + return phase_samples + + +def _validate_aggregate_phase_state( + value: object, expected_samples: list[float], field: str +) -> None: + if not isinstance(value, dict): + raise BenchmarkSchemaError(f"prediction manifest {field} must be an object") + if value.get("status") == "measured": + if set(value) != {"status", "samples"} or not isinstance(value["samples"], list): + raise BenchmarkSchemaError( + f"prediction manifest {field} measured state requires samples" + ) + samples = [ + finite_nonnegative(sample, f"prediction manifest {field}.samples") + for sample in value["samples"] + ] + if not samples or samples != expected_samples: + raise BenchmarkSchemaError( + f"prediction manifest {field} samples do not match PR telemetry" + ) + return + if value.get("status") == "not_measured": + if set(value) != {"status", "reason"}: + raise BenchmarkSchemaError( + f"prediction manifest {field} not_measured state forbids samples" + ) + _nonempty_string(value["reason"], f"{field}.reason") + if expected_samples: + raise BenchmarkSchemaError(f"prediction manifest {field} drops measured PR telemetry") + return + raise BenchmarkSchemaError(f"prediction manifest {field} status is invalid") -def _validate_manifest_shape(manifest: object, prediction_count: int) -> dict[str, Any]: +def _validate_manifest_shape( # noqa: PLR0912 - schema versions are fail-closed + manifest: object, prediction_count: int +) -> dict[str, Any]: if not isinstance(manifest, dict) or set(manifest) != _RUNNER_MANIFEST_FIELDS: raise BenchmarkSchemaError("prediction manifest has unknown or missing fields") - if manifest["schema_version"] != 3 or type(manifest["schema_version"]) is not int: - raise BenchmarkSchemaError("prediction manifest must use schema_version 3") + schema_version = manifest["schema_version"] + if type(schema_version) is not int or schema_version not in {3, 4}: + raise BenchmarkSchemaError("prediction manifest must use schema_version 3 or 4") if ( manifest["prediction_schema_version"] != 3 or type(manifest["prediction_schema_version"]) is not int @@ -659,35 +776,71 @@ def _validate_manifest_shape(manifest: object, prediction_count: int) -> dict[st if not isinstance(manifest[field], dict): raise BenchmarkSchemaError(f"prediction manifest {field} must be an object") _validate_manifest_configuration(manifest) - _validate_manifest_prs(manifest["prs"], prediction_count) + phase_samples = _validate_manifest_prs(manifest["prs"], prediction_count, schema_version) timing = manifest["timing"] - if not isinstance(timing, dict) or set(timing) != { + legacy_fields = { "started_at", "finished_at", "total_seconds", "protocol", "incremental_valid", "not_measured", - }: + } + telemetry_fields = { + "started_at", + "finished_at", + "total_seconds", + "protocol", + "incremental_valid", + "phases", + "resources", + } + expected_fields = legacy_fields if schema_version == 3 else telemetry_fields + if not isinstance(timing, dict) or set(timing) != expected_fields: raise BenchmarkSchemaError("prediction manifest timing fields are invalid") _validate_timestamp(timing["started_at"], "timing.started_at") _validate_timestamp(timing["finished_at"], "timing.finished_at") finite_nonnegative(timing["total_seconds"], "prediction manifest timing.total_seconds") - if ( - timing["protocol"] != "cold-no-cache-analyzer-wall-v1" - or timing["incremental_valid"] is not False - or not isinstance(timing["not_measured"], list) - or any(not isinstance(item, str) or not item for item in timing["not_measured"]) - ): + if timing["incremental_valid"] is not False: + raise BenchmarkSchemaError("prediction manifest incremental validity is unsupported") + if schema_version == 3: + if ( + timing["protocol"] != "cold-no-cache-analyzer-wall-v1" + or not isinstance(timing["not_measured"], list) + or any(not isinstance(item, str) or not item for item in timing["not_measured"]) + ): + raise BenchmarkSchemaError("prediction manifest timing protocol is invalid") + return manifest + if timing["protocol"] != "phase-telemetry-v1": raise BenchmarkSchemaError("prediction manifest timing protocol is invalid") + phases = timing["phases"] + if not isinstance(phases, dict) or set(phases) != set(_PERFORMANCE_PHASES): + raise BenchmarkSchemaError("prediction manifest timing phases are invalid") + for phase in _PERFORMANCE_PHASES: + _validate_aggregate_phase_state( + phases[phase], phase_samples[phase], f"timing.phases.{phase}" + ) + resources = timing["resources"] + if not isinstance(resources, dict) or set(resources) != set(_RESOURCE_METRICS): + raise BenchmarkSchemaError("prediction manifest timing resources are invalid") + for resource_name in _RESOURCE_METRICS: + if ( + _validate_measurement_state( + resources[resource_name], f"timing.resources.{resource_name}" + ) + is not None + ): + raise BenchmarkSchemaError( + f"prediction manifest timing.resources.{resource_name} must be not_measured" + ) return manifest -def read_prediction_manifest( # noqa: PLR0912 - cross-artifact bindings stay explicit +def read_prediction_manifest( # noqa: PLR0912, PLR0915 - bindings stay explicit path: Path, predictions: PrimaryArtifact, ) -> dict[str, Any]: - """Validate a schema-v3 runner manifest against exact prediction bytes and rows.""" + """Validate a schema-v3/v4 runner manifest against prediction bytes and rows.""" raw = path.read_bytes() manifest = _validate_manifest_shape( strict_json_loads(raw.decode("utf-8"), str(path)), len(predictions.records) @@ -730,13 +883,41 @@ def read_prediction_manifest( # noqa: PLR0912 - cross-artifact bindings stay ex prediction_keys = {key(item) for item in predictions.records} if manifest_keys != prediction_keys: raise BenchmarkSchemaError("prediction manifest selected keys do not match predictions") - pr_keys = { - (item.get("repository"), item.get("pr")) - for item in manifest["prs"] - if isinstance(item, dict) - } - if pr_keys != prediction_keys: + manifest_prs = {key(item): item for item in manifest["prs"] if isinstance(item, dict)} + if set(manifest_prs) != prediction_keys: raise BenchmarkSchemaError("prediction manifest PR records do not match predictions") + if manifest["schema_version"] == 4: + predictions_by_key = {key(item): item for item in predictions.records} + for record_key, manifest_pr in manifest_prs.items(): + prediction = predictions_by_key[record_key] + expected_prediction_status = _V4_PREDICTION_STATUS_BY_MANIFEST_STATUS[ + manifest_pr["status"] + ] + if prediction["status"] != expected_prediction_status: + raise BenchmarkSchemaError( + f"prediction manifest PR status does not match prediction for {record_key}" + ) + cold_state = manifest_pr["phase_telemetry"]["cold_build"] + prediction_timing = prediction["timing_seconds"] + cold_timing_name = "cold_no_cache_analyzer_wall" + expected_timing_names = ( + set() if prediction["status"] == "unresolved" else {cold_timing_name} + ) + if set(prediction_timing) != expected_timing_names: + raise BenchmarkSchemaError( + "prediction manifest PR prediction timing keys do not match " + f"status for {record_key}" + ) + if cold_state["status"] == "measured": + if prediction_timing.get(cold_timing_name) != cold_state["seconds"]: + raise BenchmarkSchemaError( + "prediction manifest PR cold timing does not match " + f"prediction for {record_key}" + ) + elif cold_timing_name in prediction_timing: + raise BenchmarkSchemaError( + f"prediction manifest PR cold timing does not match prediction for {record_key}" + ) candidate = manifest["candidate"] if manifest["git"] != {"candidate_sha": candidate["git_sha"]}: raise BenchmarkSchemaError("prediction manifest Git binding is invalid") @@ -755,6 +936,26 @@ def read_prediction_manifest( # noqa: PLR0912 - cross-artifact bindings stay ex ineligibility_reasons.append("dry_run") if configuration["allow_upstream_execution"]: ineligibility_reasons.append("upstream_execution") + performance_telemetry: dict[str, Any] | None = None + if manifest["schema_version"] == 4: + timing = manifest["timing"] + summarized_phases: dict[str, Any] = {} + for phase in _PERFORMANCE_PHASES: + state = timing["phases"][phase] + summarized_phases[phase] = ( + { + "status": "measured", + **_timing_summary([float(item) for item in state["samples"]]), + } + if state["status"] == "measured" + else dict(state) + ) + performance_telemetry = { + "protocol": timing["protocol"], + "incremental_valid": timing["incremental_valid"], + "phases": summarized_phases, + "resources": timing["resources"], + } return { "path": str(path), "sha256": hashlib.sha256(raw).hexdigest(), @@ -764,6 +965,7 @@ def read_prediction_manifest( # noqa: PLR0912 - cross-artifact bindings stay ex "runner_provenance_validated": True, "secure_execution_eligible": not ineligibility_reasons, "execution_ineligibility_reasons": ineligibility_reasons, + "performance_telemetry": performance_telemetry, } @@ -777,7 +979,7 @@ def main() -> None: # noqa: PLR0912, PLR0915 - raw and normalized metrics share parser.add_argument( "--prediction-manifest", type=Path, - help="schema-v3 runner manifest that authenticates prediction bytes and selection", + help="schema-v3/v4 runner manifest that authenticates predictions and telemetry", ) args = parser.parse_args() @@ -1280,6 +1482,11 @@ def confidence_diagnostics( "performance": { "percentile_method": "nearest-rank", "protocols": timing_results, + "manifest_phase_telemetry": ( + prediction_integrity["performance_telemetry"] + if prediction_integrity is not None + else None + ), "incremental_gate_eligible": False, "reason": "no backend currently attests incremental index reuse", }, diff --git a/benchmarks/real_world/run_current.py b/benchmarks/real_world/run_current.py index 5094eaf..192e3e2 100644 --- a/benchmarks/real_world/run_current.py +++ b/benchmarks/real_world/run_current.py @@ -12,6 +12,7 @@ import hashlib import importlib.metadata import json +import math import os import platform import re @@ -34,6 +35,19 @@ PROJECT_ROOT = HERE.parents[1] SHA_RE = re.compile(r"[0-9a-fA-F]{40}") REPOSITORY_RE = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+") +PERFORMANCE_PHASES = ( + "baseline_target_preparation", + "cold_build", + "warm_no_change", + "one_file_incremental_update", +) +FROZEN_OUTPUT_FILES = ( + HERE / "corpus.json", + HERE / "adjudicated.jsonl", + HERE / "review-a.jsonl", + HERE / "review-b.jsonl", +) +FROZEN_OUTPUT_ROOTS = (PROJECT_ROOT / "benchmarks" / "results",) class RunnerError(RuntimeError): @@ -63,6 +77,17 @@ def utc_now() -> str: return datetime.now(timezone.utc).isoformat() +def validate_output_destination(path: Path, field: str) -> None: + """Reject runner output aliases into frozen benchmark artifacts.""" + resolved = path.resolve(strict=False) + if resolved in {item.resolve(strict=False) for item in FROZEN_OUTPUT_FILES} or any( + resolved == root.resolve(strict=False) + or resolved.is_relative_to(root.resolve(strict=False)) + for root in FROZEN_OUTPUT_ROOTS + ): + raise RunnerError(f"{field} cannot target a frozen benchmark artifact: {path}") + + def atomic_write(path: Path, content: str) -> None: """Replace *path* atomically with UTF-8 text.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -519,6 +544,79 @@ def prediction_identity(entry: dict[str, Any]) -> tuple[str, int]: return repository, number +def _not_measured(reason: str) -> dict[str, str]: + return {"status": "not_measured", "reason": reason} + + +def _measured(seconds: float) -> dict[str, str | float]: + return {"status": "measured", "seconds": seconds} + + +def new_phase_telemetry() -> dict[str, dict[str, Any]]: + """Return the complete truthful phase contract before any work is measured.""" + return { + "baseline_target_preparation": _not_measured("source_preparation_not_completed"), + "cold_build": _not_measured("cold_analyzer_not_completed"), + "warm_no_change": _not_measured("backend_cache_reuse_not_implemented"), + "one_file_incremental_update": _not_measured( + "backend_invalidation_telemetry_not_implemented" + ), + } + + +def normalize_unresolved_phase_telemetry( + timings: dict[str, float], phase_telemetry: dict[str, dict[str, Any]] +) -> None: + """Clear phase attestations when the enclosing PR run does not complete.""" + timings.pop("baseline_target_preparation", None) + timings.pop("analyzer", None) + phase_telemetry["baseline_target_preparation"] = _not_measured("run_unresolved") + phase_telemetry["cold_build"] = _not_measured("run_unresolved") + + +def aggregate_phase_telemetry( + records: Sequence[dict[str, Any]], +) -> dict[str, dict[str, Any]]: + """Validate and aggregate measured PR samples without inventing values.""" + samples_by_phase: dict[str, list[float]] = {phase: [] for phase in PERFORMANCE_PHASES} + for record in records: + telemetry = record.get("phase_telemetry") + if not isinstance(telemetry, dict) or set(telemetry) != set(PERFORMANCE_PHASES): + raise RunnerError("manifest PR phase telemetry is incomplete") + for phase in PERFORMANCE_PHASES: + state = telemetry[phase] + if not isinstance(state, dict): + raise RunnerError(f"manifest PR phase {phase} must be an object") + if state.get("status") == "measured": + seconds = state.get("seconds") + if ( + set(state) != {"status", "seconds"} + or isinstance(seconds, bool) + or not isinstance(seconds, (int, float)) + or not math.isfinite(seconds) + or seconds < 0 + ): + raise RunnerError(f"manifest PR measured phase {phase} is invalid") + samples_by_phase[phase].append(float(seconds)) + elif state.get("status") == "not_measured": + if ( + set(state) != {"status", "reason"} + or not isinstance(state.get("reason"), str) + or not state["reason"].strip() + ): + raise RunnerError(f"manifest PR not_measured phase {phase} is invalid") + else: + raise RunnerError(f"manifest PR phase {phase} status is invalid") + return { + phase: ( + {"status": "measured", "samples": samples} + if (samples := samples_by_phase[phase]) + else _not_measured("no_measured_pr_samples") + ) + for phase in PERFORMANCE_PHASES + } + + def unresolved_prediction( repository: str, pr: int, candidate_id: str, reason: str ) -> dict[str, Any]: @@ -544,6 +642,7 @@ def process_entry( # noqa: PLR0915 started = time.monotonic() phase_started = started timings: dict[str, float] = {} + phase_telemetry = new_phase_telemetry() merge_sha: str | None = None base_sha: str | None = None configured_root = config.app_roots.get(repository, config.default_app_root) @@ -559,6 +658,7 @@ def process_entry( # noqa: PLR0915 "base_sha": None, "status": "unresolved", "timing_seconds": timings, + "phase_telemetry": phase_telemetry, } merge_data = entry.get("mergeCommit") if isinstance(merge_data, dict): @@ -597,6 +697,7 @@ def process_entry( # noqa: PLR0915 timings["parent_resolution"] = time.monotonic() - phase_started phase_started = time.monotonic() + preparation_started = time.monotonic() with tempfile.TemporaryDirectory(prefix="current-analyzer-") as temporary_name: temporary = Path(temporary_name) worktree = temporary / "target" @@ -620,7 +721,9 @@ def process_entry( # noqa: PLR0915 write_local_diff(repository_cache, base_sha, merge_sha, patch_path) timings["diff"] = time.monotonic() - phase_started - phase_started = time.monotonic() + preparation_seconds = time.monotonic() - preparation_started + timings["baseline_target_preparation"] = preparation_seconds + phase_telemetry["baseline_target_preparation"] = _measured(preparation_seconds) endpoints, candidates, unresolved, analyzer_seconds = invoke_analyzer( config.candidate_root, @@ -634,6 +737,7 @@ def process_entry( # noqa: PLR0915 baseline_app_root=baseline_app_root, ) timings["analyzer"] = analyzer_seconds + phase_telemetry["cold_build"] = _measured(analyzer_seconds) finally: if baseline_worktree_added: remove_worktree(repository_cache, baseline_worktree) @@ -667,6 +771,7 @@ def process_entry( # noqa: PLR0915 except (RunnerError, OSError) as error: elapsed = time.monotonic() - started timings["total"] = elapsed + normalize_unresolved_phase_telemetry(timings, phase_telemetry) reason = str(error) manifest_record["merge_sha"] = merge_sha manifest_record["base_sha"] = base_sha @@ -892,6 +997,8 @@ def main(argv: Sequence[str] | None = None) -> int: } if len(artifact_paths) != 3: raise RunnerError("--corpus, --output, and --manifest must be distinct paths") + validate_output_destination(args.output, "--output") + validate_output_destination(args.manifest, "--manifest") app_roots = parse_app_roots(args.app_root) app_entries = parse_app_entries(args.app_entry) bootstrap_entries = parse_app_entries(args.bootstrap_entry, "--bootstrap-entry") @@ -945,7 +1052,7 @@ def main(argv: Sequence[str] | None = None) -> int: jsonl = "".join(json.dumps(item, sort_keys=True) + "\n" for item in predictions) prediction_sha256 = hashlib.sha256(jsonl.encode()).hexdigest() manifest = { - "schema_version": 3, + "schema_version": 4, "prediction_schema_version": 3, "created_at": utc_now(), "candidate": candidate, @@ -990,14 +1097,13 @@ def main(argv: Sequence[str] | None = None) -> int: "started_at": started_wall, "finished_at": utc_now(), "total_seconds": time.monotonic() - started, - "protocol": "cold-no-cache-analyzer-wall-v1", + "protocol": "phase-telemetry-v1", "incremental_valid": False, - "not_measured": [ - "warm_no_change", - "one_file_incremental_update", - "peak_rss", - "cache_size", - ], + "phases": aggregate_phase_telemetry(pr_manifest), + "resources": { + "peak_rss_bytes": _not_measured("process_tree_rss_not_sampled"), + "cache_size_bytes": _not_measured("backend_cache_size_not_measured"), + }, }, } atomic_write(config.output, jsonl) diff --git a/tests/benchmarks/test_evaluate.py b/tests/benchmarks/test_evaluate.py index 0996e54..443ba82 100644 --- a/tests/benchmarks/test_evaluate.py +++ b/tests/benchmarks/test_evaluate.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from collections.abc import Callable from pathlib import Path + from typing import Any def test_ranked_kind_preserves_declared_non_http_kind() -> None: @@ -918,6 +919,365 @@ def test_schema_v3_rejects_unknown_fields_float_versions_and_malformed_adapter_c read_primary_jsonl(path, "prediction") +@pytest.mark.parametrize( + ("state", "message"), + [ + ({"status": "measured"}, "requires only seconds"), + ( + {"status": "not_measured", "reason": "missing", "seconds": 1.0}, + "forbids samples", + ), + ({"status": "measured", "seconds": float("nan")}, "finite non-negative"), + ({"status": "unknown"}, "status is invalid"), + ], +) +def test_schema_v4_phase_states_reject_malformed_measured_statuses( + state: dict[str, object], message: str +) -> None: + with pytest.raises(ValueError, match=message): + evaluate._validate_measurement_state(state, "prs.phase_telemetry.cold_build") + + +def test_schema_v4_aggregate_rejects_mixed_or_dropped_measured_samples() -> None: + with pytest.raises(ValueError, match="drops measured PR telemetry"): + evaluate._validate_aggregate_phase_state( + {"status": "not_measured", "reason": "missing"}, + [1.0], + "timing.phases.cold_build", + ) + with pytest.raises(ValueError, match="do not match PR telemetry"): + evaluate._validate_aggregate_phase_state( + {"status": "measured", "samples": [2.0]}, + [1.0], + "timing.phases.cold_build", + ) + with pytest.raises(ValueError, match="forbids samples"): + evaluate._validate_aggregate_phase_state( + {"status": "not_measured", "reason": "missing", "samples": []}, + [], + "timing.phases.warm_no_change", + ) + + +def _schema_v4_manifest_fixture( + tmp_path: Path, +) -> tuple[Path, Path, dict[str, Any], dict[str, Any]]: + predictions = tmp_path / "predictions.jsonl" + manifest_path = tmp_path / "manifest.json" + prediction = { + "schema_version": 3, + "repository": "owner/repo", + "pr": 1, + "candidate": "candidate/v1", + "adapter": "fastapi-adapter-v1", + "status": "completed", + "affected_entrypoints": [], + "candidate_entrypoints": [], + "unresolved": [], + "timing_seconds": {"cold_no_cache_analyzer_wall": 2.0}, + } + manifest = { + "schema_version": 4, + "prediction_schema_version": 3, + "created_at": "2026-01-01T00:00:00+00:00", + "candidate": { + "id": "candidate/v1", + "name": "fastapi-endpoint-detector", + "version": "test", + "adapter": "fastapi-adapter-v1", + "git_sha": "a" * 40, + "config_hash": "d" * 12, + "dirty": False, + "dirty_sha256": None, + "uv_lock_sha256": "b" * 64, + "uv_version": "uv test", + "command": "uv run --frozen fastapi-endpoint-detector analyze --no-cache", + "performance_protocol": { + "id": "cold-no-cache-analyzer-wall-v1", + "cache_enabled": False, + "incremental_valid": False, + }, + }, + "git": {"candidate_sha": "a" * 40}, + "prediction_output": {"path": str(predictions), "sha256": "", "records": 1}, + "selected_keys": [{"repository": "owner/repo", "pr": 1}], + "python": "Python test", + "platform": "test-platform", + "corpus": {"path": "corpus.json", "sha256": "c" * 64}, + "root_config": {"default": ".", "repositories": {}}, + "app_entry_config": {}, + "bootstrap_entry_config": {}, + "configuration": { + "cache": "/tmp/cache", + "output": str(predictions), + "manifest": str(manifest_path), + "timeout_seconds": 60.0, + "dry_run": False, + "allow_upstream_execution": False, + "use_scip": False, + "filters": {"limit": None, "repositories": [], "prs": []}, + }, + "selection_count": 1, + "prs": [ + { + "repository": "owner/repo", + "pr": 1, + "configured_app_root": ".", + "configured_app_entry": None, + "configured_bootstrap_entry": None, + "merge_sha": "a" * 40, + "base_sha": "e" * 40, + "status": "completed", + "timing_seconds": { + "baseline_target_preparation": 1.0, + "analyzer": 2.0, + "total": 3.0, + }, + "phase_telemetry": { + "baseline_target_preparation": {"status": "measured", "seconds": 1.0}, + "cold_build": {"status": "measured", "seconds": 2.0}, + "warm_no_change": { + "status": "not_measured", + "reason": "backend_cache_reuse_not_implemented", + }, + "one_file_incremental_update": { + "status": "not_measured", + "reason": "backend_invalidation_telemetry_not_implemented", + }, + }, + } + ], + "timing": { + "started_at": "2026-01-01T00:00:00+00:00", + "finished_at": "2026-01-01T00:00:03+00:00", + "total_seconds": 3.0, + "protocol": "phase-telemetry-v1", + "incremental_valid": False, + "phases": { + "baseline_target_preparation": {"status": "measured", "samples": [1.0]}, + "cold_build": {"status": "measured", "samples": [2.0]}, + "warm_no_change": {"status": "not_measured", "reason": "no_samples"}, + "one_file_incremental_update": { + "status": "not_measured", + "reason": "no_samples", + }, + }, + "resources": { + "peak_rss_bytes": {"status": "not_measured", "reason": "not_sampled"}, + "cache_size_bytes": {"status": "not_measured", "reason": "not_sampled"}, + }, + }, + } + return predictions, manifest_path, prediction, manifest + + +def _write_schema_v4_manifest_fixture( + predictions: Path, + manifest_path: Path, + prediction: dict[str, Any], + manifest: dict[str, Any], +) -> None: + prediction_content = json.dumps(prediction) + "\n" + predictions.write_text(prediction_content) + manifest["prediction_output"]["sha256"] = hashlib.sha256( + prediction_content.encode() + ).hexdigest() + manifest_path.write_text(json.dumps(manifest)) + + +def test_schema_v4_manifest_rejects_fabricated_warm_telemetry(tmp_path: Path) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + manifest["prs"][0]["phase_telemetry"]["warm_no_change"] = { + "status": "measured", + "seconds": 123.0, + } + manifest["timing"]["phases"]["warm_no_change"] = { + "status": "measured", + "samples": [123.0], + } + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match="warm_no_change must be not_measured"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + +def test_schema_v4_manifest_rejects_completed_with_all_phases_not_measured( + tmp_path: Path, +) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + for phase in evaluate._PERFORMANCE_PHASES: + manifest["prs"][0]["phase_telemetry"][phase] = { + "status": "not_measured", + "reason": "fabricated_missing_sample", + } + manifest["timing"]["phases"][phase] = { + "status": "not_measured", + "reason": "fabricated_missing_sample", + } + manifest["prs"][0]["timing_seconds"] = {"total": 3.0} + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match="must measure preparation and cold phases"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + +def test_schema_v4_manifest_rejects_prediction_status_disagreement(tmp_path: Path) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + manifest["prs"][0]["status"] = "completed_with_unresolved" + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match="PR status does not match prediction"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + +def test_schema_v4_manifest_rejects_prediction_cold_timing_disagreement(tmp_path: Path) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + prediction["timing_seconds"]["cold_no_cache_analyzer_wall"] = 9.0 + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match="PR cold timing does not match prediction"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + +def test_schema_v4_manifest_rejects_completed_prediction_extra_timing(tmp_path: Path) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + prediction["timing_seconds"]["warm_no_change"] = 123.0 + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match="prediction timing keys do not match status"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + +def test_schema_v4_manifest_rejects_unresolved_prediction_extra_timing(tmp_path: Path) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + prediction["status"] = "unresolved" + prediction["unresolved"] = ["fabricated failure"] + prediction["timing_seconds"] = {"one_file_incremental_update": 456.0} + manifest_pr = manifest["prs"][0] + manifest_pr["status"] = "unresolved" + manifest_pr["reason"] = "fabricated failure" + for phase, timing_name in evaluate._MEASURED_PHASE_TIMING_FIELDS.items(): + manifest_pr["timing_seconds"].pop(timing_name) + manifest_pr["phase_telemetry"][phase] = { + "status": "not_measured", + "reason": "run_unresolved", + } + manifest["timing"]["phases"][phase] = { + "status": "not_measured", + "reason": "no_measured_pr_samples", + } + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match="prediction timing keys do not match status"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + +def test_schema_v4_evaluator_outputs_only_attested_prediction_timing( + tmp_path: Path, monkeypatch, capsys +) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + truth = tmp_path / "truth.jsonl" + truth.write_text( + json.dumps( + { + "repository": "owner/repo", + "pr": 1, + "status": "adjudicated", + "affected_entrypoints": [], + } + ) + + "\n" + ) + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + monkeypatch.setattr( + sys, + "argv", + [ + "evaluate.py", + "--ground-truth", + str(truth), + "--predictions", + str(predictions), + "--prediction-manifest", + str(manifest_path), + ], + ) + + evaluate.main() + + result = json.loads(capsys.readouterr().out) + performance = result["performance"] + assert set(performance["protocols"]) == {"cold_no_cache_analyzer_wall"} + assert performance["protocols"]["cold_no_cache_analyzer_wall"]["samples"] == 1 + phases = performance["manifest_phase_telemetry"]["phases"] + for phase in ("warm_no_change", "one_file_incremental_update"): + assert phases[phase]["status"] == "not_measured" + assert phase not in performance["protocols"] + + +def test_schema_v4_manifest_rejects_unresolved_cold_without_preparation( + tmp_path: Path, +) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + prediction["status"] = "unresolved" + prediction["unresolved"] = ["fabricated failure"] + manifest_pr = manifest["prs"][0] + manifest_pr["status"] = "unresolved" + manifest_pr["reason"] = "fabricated failure" + manifest_pr["timing_seconds"].pop("baseline_target_preparation") + manifest_pr["phase_telemetry"]["baseline_target_preparation"] = { + "status": "not_measured", + "reason": "fabricated_missing_preparation", + } + manifest["timing"]["phases"]["baseline_target_preparation"] = { + "status": "not_measured", + "reason": "fabricated_missing_preparation", + } + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match="cold phase requires measured preparation"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + +def test_schema_v4_manifest_rejects_unresolved_measured_preparation(tmp_path: Path) -> None: + predictions, manifest_path, prediction, manifest = _schema_v4_manifest_fixture(tmp_path) + prediction["status"] = "unresolved" + prediction["unresolved"] = ["fabricated failure"] + prediction["timing_seconds"] = {} + manifest_pr = manifest["prs"][0] + manifest_pr["status"] = "unresolved" + manifest_pr["reason"] = "fabricated failure" + manifest_pr["timing_seconds"].pop("analyzer") + manifest_pr["phase_telemetry"]["cold_build"] = { + "status": "not_measured", + "reason": "fabricated_missing_cold", + } + manifest["timing"]["phases"]["cold_build"] = { + "status": "not_measured", + "reason": "fabricated_missing_cold", + } + _write_schema_v4_manifest_fixture(predictions, manifest_path, prediction, manifest) + + with pytest.raises(ValueError, match=r"unresolved.*must not attest measured phases"): + evaluate.read_prediction_manifest( + manifest_path, read_primary_artifact(predictions, "prediction") + ) + + def test_explicit_empty_candidate_list_does_not_fall_back_to_affected() -> None: record = { "affected_entrypoints": [{"id": "HTTP GET /selected", "kind": "http"}], diff --git a/tests/benchmarks/test_run_current.py b/tests/benchmarks/test_run_current.py index b33bc0e..517b206 100644 --- a/tests/benchmarks/test_run_current.py +++ b/tests/benchmarks/test_run_current.py @@ -163,6 +163,18 @@ def make_worktree(_cache: Path, worktree: Path, _base: str) -> None: self.assertEqual(add_worktree.call_args.args[2], SHA_A) self.assertEqual(prediction["unresolved"], []) self.assertEqual(manifest["status"], "completed") + self.assertEqual( + manifest["phase_telemetry"]["baseline_target_preparation"]["status"], + "measured", + ) + self.assertEqual( + manifest["phase_telemetry"]["cold_build"], + {"status": "measured", "seconds": 0.25}, + ) + self.assertEqual( + manifest["phase_telemetry"]["warm_no_change"]["status"], + "not_measured", + ) def test_scip_materializes_and_cleans_target_and_baseline(self) -> None: with tempfile.TemporaryDirectory() as temporary_name: @@ -400,6 +412,17 @@ def test_report_errors_and_warnings_are_preserved(self) -> None: class OutputCardinalityTests(unittest.TestCase): + def test_phase_aggregation_rejects_malformed_or_partial_states(self) -> None: + telemetry = run_current.new_phase_telemetry() + telemetry["cold_build"] = {"status": "measured", "seconds": float("nan")} + with self.assertRaisesRegex(run_current.RunnerError, "measured phase cold_build"): + run_current.aggregate_phase_telemetry([{"phase_telemetry": telemetry}]) + + telemetry = run_current.new_phase_telemetry() + telemetry.pop("warm_no_change") + with self.assertRaisesRegex(run_current.RunnerError, "telemetry is incomplete"): + run_current.aggregate_phase_telemetry([{"phase_telemetry": telemetry}]) + def test_main_rejects_colliding_artifact_paths(self) -> None: with tempfile.TemporaryDirectory() as temporary_name: corpus = Path(temporary_name) / "corpus.json" @@ -416,6 +439,26 @@ def test_main_rejects_colliding_artifact_paths(self) -> None: ] ) + def test_main_rejects_frozen_result_destinations_before_writing(self) -> None: + frozen_output = run_current.PROJECT_ROOT / "benchmarks/results/runner-must-not-write.jsonl" + self.assertFalse(frozen_output.exists()) + with tempfile.TemporaryDirectory() as temporary_name: + temporary = Path(temporary_name) + corpus = temporary / "corpus.json" + corpus.write_text('{"entries": []}', encoding="utf-8") + with self.assertRaises(SystemExit): + run_current.main( + [ + "--corpus", + str(corpus), + "--output", + str(frozen_output), + "--manifest", + str(temporary / "manifest.json"), + ] + ) + self.assertFalse(frozen_output.exists()) + def test_main_writes_one_unique_row_per_selected_pr(self) -> None: with tempfile.TemporaryDirectory() as temporary_name: temporary = Path(temporary_name) @@ -482,7 +525,7 @@ def test_main_writes_one_unique_row_per_selected_pr(self) -> None: ) self.assertEqual(manifest_data["selection_count"], 2) self.assertEqual(len(manifest_data["prs"]), 2) - self.assertEqual(manifest_data["schema_version"], 3) + self.assertEqual(manifest_data["schema_version"], 4) self.assertEqual(manifest_data["prediction_schema_version"], 3) self.assertEqual(manifest_data["prediction_output"]["records"], 2) self.assertEqual( @@ -497,9 +540,211 @@ def test_main_writes_one_unique_row_per_selected_pr(self) -> None: ], ) self.assertFalse(manifest_data["timing"]["incremental_valid"]) + self.assertEqual( + set(manifest_data["timing"]["phases"]), set(run_current.PERFORMANCE_PHASES) + ) + self.assertTrue( + all( + item["status"] == "not_measured" + for item in manifest_data["timing"]["resources"].values() + ) + ) self.assertEqual(manifest_binding["candidate"], "candidate") self.assertEqual(manifest_binding["prediction_sha256"], prediction_artifact.sha256) + def test_completed_schema_v4_manifest_round_trips_measured_phases(self) -> None: + with tempfile.TemporaryDirectory() as temporary_name: + temporary = Path(temporary_name) + corpus = temporary / "corpus.json" + output = temporary / "predictions.jsonl" + manifest = temporary / "manifest.json" + corpus.write_text(json.dumps({"entries": [entry("owner/repo", 1)]}), encoding="utf-8") + candidate = { + "id": "candidate/v1", + "name": "fastapi-endpoint-detector", + "version": "test", + "adapter": "fastapi-adapter-v1", + "git_sha": SHA_A, + "config_hash": "d" * 12, + "dirty": False, + "dirty_sha256": None, + "uv_lock_sha256": "b" * 64, + "uv_version": "uv test", + "command": "uv run --frozen fastapi-endpoint-detector analyze --no-cache", + "performance_protocol": { + "id": "cold-no-cache-analyzer-wall-v1", + "cache_enabled": False, + "incremental_valid": False, + }, + } + prediction = { + "schema_version": 3, + "repository": "owner/repo", + "pr": 1, + "candidate": "candidate/v1", + "adapter": "fastapi-adapter-v1", + "status": "completed", + "affected_entrypoints": [], + "candidate_entrypoints": [], + "unresolved": [], + "timing_seconds": {"cold_no_cache_analyzer_wall": 2.0}, + } + record = { + "repository": "owner/repo", + "pr": 1, + "configured_app_root": ".", + "configured_app_entry": None, + "configured_bootstrap_entry": None, + "merge_sha": SHA_A, + "base_sha": SHA_B, + "status": "completed", + "timing_seconds": { + "baseline_target_preparation": 1.0, + "analyzer": 2.0, + "total": 3.0, + }, + "phase_telemetry": { + "baseline_target_preparation": {"status": "measured", "seconds": 1.0}, + "cold_build": {"status": "measured", "seconds": 2.0}, + "warm_no_change": { + "status": "not_measured", + "reason": "backend_cache_reuse_not_implemented", + }, + "one_file_incremental_update": { + "status": "not_measured", + "reason": "backend_invalidation_telemetry_not_implemented", + }, + }, + } + with ( + mock.patch.object(run_current, "candidate_metadata", return_value=candidate), + mock.patch.object(run_current, "process_entry", return_value=(prediction, record)), + ): + result = run_current.main( + [ + "--corpus", + str(corpus), + "--output", + str(output), + "--manifest", + str(manifest), + ] + ) + artifact = read_primary_artifact(output, "prediction") + binding = evaluate.read_prediction_manifest(manifest, artifact) + manifest_data = json.loads(manifest.read_text(encoding="utf-8")) + + self.assertEqual(result, 0) + self.assertEqual(manifest_data["schema_version"], 4) + self.assertEqual( + manifest_data["timing"]["phases"]["baseline_target_preparation"], + {"status": "measured", "samples": [1.0]}, + ) + self.assertEqual( + binding["performance_telemetry"]["phases"]["cold_build"], + { + "status": "measured", + "samples": 1, + "mean": 2.0, + "p50": 2.0, + "p95": 2.0, + "max": 2.0, + }, + ) + self.assertFalse(binding["performance_telemetry"]["incremental_valid"]) + self.assertEqual( + binding["performance_telemetry"]["resources"]["peak_rss_bytes"]["status"], + "not_measured", + ) + + def test_cleanup_failure_clears_phase_attestations_and_round_trips(self) -> None: + with tempfile.TemporaryDirectory() as temporary_name: + temporary = Path(temporary_name) + corpus = temporary / "corpus.json" + output = temporary / "predictions.jsonl" + manifest = temporary / "manifest.json" + corpus.write_text(json.dumps({"entries": [entry("owner/repo", 1)]}), encoding="utf-8") + candidate = { + "id": "candidate/v1", + "name": "fastapi-endpoint-detector", + "version": "test", + "adapter": "fastapi-adapter-v1", + "git_sha": SHA_A, + "config_hash": "d" * 12, + "dirty": False, + "dirty_sha256": None, + "uv_lock_sha256": "b" * 64, + "uv_version": "uv test", + "command": "uv run --frozen fastapi-endpoint-detector analyze --no-cache", + "performance_protocol": { + "id": "cold-no-cache-analyzer-wall-v1", + "cache_enabled": False, + "incremental_valid": False, + }, + } + + def make_worktree(_cache: Path, worktree: Path, _sha: str) -> None: + worktree.mkdir() + + with ( + mock.patch.object(run_current, "candidate_metadata", return_value=candidate), + mock.patch.object(run_current, "ensure_cache", return_value=temporary / "bare"), + mock.patch.object(run_current, "merge_parents", return_value=[SHA_B]), + mock.patch.object(run_current, "add_detached_worktree", side_effect=make_worktree), + mock.patch.object(run_current, "write_local_diff"), + mock.patch.object(run_current, "invoke_analyzer", return_value=([], [], [], 0.25)), + mock.patch.object( + run_current, "remove_worktree", side_effect=OSError("cleanup failed") + ), + ): + result = run_current.main( + [ + "--corpus", + str(corpus), + "--output", + str(output), + "--manifest", + str(manifest), + ] + ) + + artifact = read_primary_artifact(output, "prediction") + binding = evaluate.read_prediction_manifest(manifest, artifact) + prediction = artifact.records[0] + manifest_data = json.loads(manifest.read_text(encoding="utf-8")) + manifest_pr = manifest_data["prs"][0] + + self.assertEqual(result, 0) + self.assertEqual(prediction["status"], "unresolved") + self.assertEqual(prediction["timing_seconds"], {}) + self.assertEqual(manifest_pr["status"], "unresolved") + self.assertIn("cleanup failed", manifest_pr["reason"]) + self.assertNotIn("baseline_target_preparation", manifest_pr["timing_seconds"]) + self.assertNotIn("analyzer", manifest_pr["timing_seconds"]) + self.assertLessEqual( + {"cache_fetch", "parent_resolution", "worktree", "diff", "total"}, + set(manifest_pr["timing_seconds"]), + ) + self.assertTrue( + all( + manifest_pr["phase_telemetry"][phase]["status"] == "not_measured" + for phase in run_current.PERFORMANCE_PHASES + ) + ) + self.assertTrue( + all( + phase["status"] == "not_measured" + for phase in manifest_data["timing"]["phases"].values() + ) + ) + self.assertTrue( + all( + resource["status"] == "not_measured" + for resource in manifest_data["timing"]["resources"].values() + ) + ) + self.assertEqual(binding["candidate"], "candidate/v1") + if __name__ == "__main__": unittest.main()