diff --git a/README.md b/README.md index 505c70c3..8d1c6a2a 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ Bring-your-own-key usage is always free and is never gated — Atlas only meters ## How it works -OpenScience runs a local server that hosts the workspace UI, the agent runtime, the complete default skill library, and the tool layer. The agent plans with a research harness, calls tools (shell, editor, LSP, MCP servers, scientific connectors, and skills), and streams its work back to the browser. Models are routed per request, so you can switch between providers or run local models without changing anything else. Sessions, skills, artifacts, and provenance are stored on disk. Atlas adds optional managed models, credential sync, research graphs, library search, and cloud publishing after login. +OpenScience runs a local server that hosts the workspace UI, the agent runtime, the complete default skill library, and the tool layer. The agent plans with a research harness, calls tools (shell, editor, LSP, MCP servers, scientific connectors, and skills), and streams its work back to the browser. Models are routed per request, so you can switch between providers or run local models without changing anything else. Sessions, skills, artifacts, and provenance are stored on disk. Atlas adds optional managed models, credential sync, research graphs, library search, and cloud publishing after login. The verifier-grounded product architecture is documented in [Scientific research harness](docs/HARNESS.md). | Path | Contents | | -------------------- | ------------------------------------------------------------ | diff --git a/backend/cli/skills/physics/simulator-validation/SKILL.md b/backend/cli/skills/physics/simulator-validation/SKILL.md new file mode 100644 index 00000000..18b3400b --- /dev/null +++ b/backend/cli/skills/physics/simulator-validation/SKILL.md @@ -0,0 +1,78 @@ +--- +name: simulator-validation +description: Validate an ODE, PDE, CFD, materials, molecular, or physics simulator with executable convergence, residual, invariant, and reference checks. Use before trusting a numerical result, comparing solvers, or making a benchmark or scientific claim from simulated data. +--- + +# Simulator Validation + +Treat simulation as a numerical experiment with a falsifiable validation contract, not as a picture generator. + +## Select the smallest credible simulator + +Choose from the problem structure and installed capability: + +- analytic, symbolic, or low-dimensional ODE: SymPy and SciPy; +- structured finite differences or finite volumes: NumPy/SciPy or FiPy; +- unstructured finite elements and multiphysics: FEniCSx/DOLFINx; +- spectral PDEs: Dedalus or a documented spectral implementation; +- production CFD: OpenFOAM or SU2 when its model and mesh support are required; +- atomistic/material workflows: ASE or pymatgen as workflow layers plus the declared physical engine; +- molecular dynamics: OpenMM, GROMACS, or LAMMPS according to force field and scale. + +Check the actual executable/import and capture its exact version. Do not silently replace an unavailable solver with a different physical model. + +## Freeze the problem + +Record equations, coefficients, units or nondimensionalization, domain and geometry, material regions, initial and boundary conditions, scheme and formal order, mesh/timestep sequence, linear/nonlinear solvers, tolerances, stopping rules, and random seeds. Hash the effective simulator configuration. + +Choose at least one reference: + +- analytic solution; +- manufactured solution with derived source term; +- trusted benchmark solution; +- independently implemented solver; or +- known limiting/asymptotic result. + +## Run a refinement study + +Use at least three systematically refined levels. Evaluate the same quantity and norm on every level. Capture a validation JSON: + +```json +{ + "simulator":{"name":"solver","version":"1.2.3","command":"solver case.yaml","configSHA256":"64-hex"}, + "expectedOrder":2, + "orderTolerance":0.3, + "maxResidual":1e-8, + "invariantTolerances":{"mass_drift":1e-6}, + "levels":[ + {"label":"coarse","h":0.1,"error":0.01,"residual":1e-9,"invariants":{"mass_drift":2e-7}}, + {"label":"medium","h":0.05,"error":0.0025,"residual":2e-9,"invariants":{"mass_drift":3e-7}}, + {"label":"fine","h":0.025,"error":0.000625,"residual":3e-9,"invariants":{"mass_drift":4e-7}} + ] +} +``` + +Validate it: + +```bash +python scripts/validate_convergence.py validation.json --output validation-report.json +``` + +The script exits nonzero unless resolution decreases, error decreases, median observed order meets tolerance, every residual passes, and every declared invariant deviation stays bounded. + +## Adversarial validation + +Also test applicable properties: + +- timestep and solver-tolerance sensitivity; +- conservation, positivity, symmetry, maximum principle, or boundedness; +- coordinate, sign, and unit conventions; +- stiffness, shocks, singularities, mesh distortion, or chaotic sensitivity; +- domain truncation and boundary reflection; +- independent implementation or clean replay for the headline result. + +Keep failed levels and nonconvergent runs. A small residual alone does not establish discretization accuracy, and visual agreement is not a convergence test. + +## Report + +Publish simulator/version, configuration hash, level table, error norm, observed orders, residuals, invariant deviations, reference identity, artifacts, compute, and the validator report. Do not claim physical fidelity beyond the validated model regime. diff --git a/backend/cli/skills/physics/simulator-validation/agents/openai.yaml b/backend/cli/skills/physics/simulator-validation/agents/openai.yaml new file mode 100644 index 00000000..46199a84 --- /dev/null +++ b/backend/cli/skills/physics/simulator-validation/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Simulator Validation" + short_description: "Validate convergence and physical invariants" + default_prompt: "Use $simulator-validation to validate a simulator with convergence, residual, and invariant checks." diff --git a/backend/cli/skills/physics/simulator-validation/scripts/validate_convergence.py b/backend/cli/skills/physics/simulator-validation/scripts/validate_convergence.py new file mode 100755 index 00000000..1a6d25f5 --- /dev/null +++ b/backend/cli/skills/physics/simulator-validation/scripts/validate_convergence.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Validate a simulator refinement study and emit machine-readable checks.""" + +import argparse +import hashlib +import json +import math +import os +import sys +import tempfile +from pathlib import Path + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def write(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("input", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + source = args.input.read_bytes() + data = json.loads(source) + require(isinstance(data, dict), "validation input must be an object") + simulator = data.get("simulator") + require(isinstance(simulator, dict), "simulator identity is required") + for field in ("name", "version", "command"): + require(isinstance(simulator.get(field), str) and simulator[field], f"simulator.{field} is required") + config_hash = simulator.get("configSHA256") + require(isinstance(config_hash, str) and len(config_hash) == 64, "simulator.configSHA256 must be 64 hex characters") + require(all(character in "0123456789abcdef" for character in config_hash), "simulator.configSHA256 must be lowercase hex") + + expected = data.get("expectedOrder") + tolerance = data.get("orderTolerance") + maximum = data.get("maxResidual") + require(isinstance(expected, (int, float)) and not isinstance(expected, bool) and expected > 0, "expectedOrder must be positive") + require(isinstance(tolerance, (int, float)) and not isinstance(tolerance, bool) and tolerance >= 0, "orderTolerance must be nonnegative") + require(isinstance(maximum, (int, float)) and not isinstance(maximum, bool) and maximum >= 0, "maxResidual must be nonnegative") + invariants = data.get("invariantTolerances", {}) + require(isinstance(invariants, dict), "invariantTolerances must be an object") + require(all(isinstance(key, str) and key for key in invariants), "invariant names must be non-empty strings") + require( + all(isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 for value in invariants.values()), + "invariant tolerances must be finite and nonnegative", + ) + + levels = data.get("levels") + require(isinstance(levels, list) and len(levels) >= 3, "at least three refinement levels are required") + parsed = [] + for index, level in enumerate(levels): + require(isinstance(level, dict), f"level {index} must be an object") + label = level.get("label") + require(isinstance(label, str) and label, f"level {index} needs a label") + values = {key: level.get(key) for key in ("h", "error", "residual")} + require( + all(isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) for value in values.values()), + f"level {label} contains a non-finite numeric value", + ) + require(values["h"] > 0 and values["error"] > 0 and values["residual"] >= 0, f"level {label} has invalid h/error/residual") + observed = level.get("invariants", {}) + require(isinstance(observed, dict), f"level {label} invariants must be an object") + require(set(observed) == set(invariants), f"level {label} must report every declared invariant") + require( + all(isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 for value in observed.values()), + f"level {label} has an invalid invariant deviation", + ) + parsed.append({"label": label, **values, "invariants": observed}) + + resolution = all(left["h"] > right["h"] for left, right in zip(parsed, parsed[1:])) + monotone = all(left["error"] > right["error"] for left, right in zip(parsed, parsed[1:])) + orders = [ + math.log(left["error"] / right["error"]) / math.log(left["h"] / right["h"]) + for left, right in zip(parsed, parsed[1:]) + ] + median = sorted(orders)[len(orders) // 2] if len(orders) % 2 else sum(sorted(orders)[len(orders) // 2 - 1 : len(orders) // 2 + 1]) / 2 + residual = all(level["residual"] <= maximum for level in parsed) + invariant_status = { + name: all(level["invariants"][name] <= limit for level in parsed) for name, limit in invariants.items() + } + checks = { + "resolution_decreases": resolution, + "error_decreases": monotone, + "observed_order": median >= expected - tolerance, + "residual_bound": residual, + **{f"invariant:{name}": status for name, status in invariant_status.items()}, + } + passed = all(checks.values()) + report = { + "schemaVersion": 1, + "passed": passed, + "inputSHA256": hashlib.sha256(source).hexdigest(), + "simulator": simulator, + "observedOrders": orders, + "medianObservedOrder": median, + "requiredOrder": expected - tolerance, + "checks": checks, + "levels": parsed, + } + if args.output: + write(args.output, report) + print(json.dumps(report, sort_keys=True)) + return 0 if passed else 1 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(json.dumps({"error": str(error)}), file=sys.stderr) + raise SystemExit(2) diff --git a/backend/cli/skills/research/active-failure-audit/SKILL.md b/backend/cli/skills/research/active-failure-audit/SKILL.md new file mode 100644 index 00000000..696a4859 --- /dev/null +++ b/backend/cli/skills/research/active-failure-audit/SKILL.md @@ -0,0 +1,66 @@ +--- +name: active-failure-audit +description: Build and run a blinded, commitment-bound active evaluation over a costly hidden probe pool. Use when full evaluation is too expensive, when diverse failure discovery matters, or when a benchmark needs uncertainty-aware sample selection without exposing hidden cases to the agent. +--- + +# Active Failure Audit + +Use an evaluator-owned probe pool to estimate loss and search for diverse failures under a fixed budget. Keep this workflow outside the candidate-producing agent session. + +## Freeze the protocol + +Before evaluating, pin: + +- the exact run or candidate artifact SHA-256; +- `performance`, `failure`, or `hybrid` mode; +- the probe budget and minimum sample count; +- the loss definition and frozen failure threshold; +- the precision tolerance and abstention threshold; +- the feature representation, strata, and evaluation weights; and +- an optional target number of distinct failures. + +Do not tune these fields after seeing outcomes. + +## Build opaque probe commitments + +Prepare evaluator-private JSONL with one object per hidden case: + +```json +{"id":"case-17","hidden":{"prompt":"...","target":"..."},"features":[0.2,-1.1,0.4],"stratum":"long-tail","weight":1,"priorLoss":0.5} +``` + +Run: + +```bash +python scripts/build_probe_manifest.py private-probes.jsonl public-manifest.json +``` + +The script validates one shared finite feature dimension, unique IDs, and unique hidden-case bytes. It emits only opaque IDs, numeric features, strata, weights, prior loss, and SHA-256 commitments. Keep the private JSONL outside the agent workspace. Do not use the generated manifest if its validation fails. + +## Run the audit + +1. Bind the audit configuration in the immutable harness contract. +2. Initialize `/harness/audits` with the evaluator capability, frozen subject artifact, and generated `probes` array. +3. Request one `/selection` at a time. Resolve the returned commitment to the private case inside the evaluator boundary. +4. Evaluate the frozen artifact. Submit loss, threshold-consistent failure label, and observable evidence to `/observations`. +5. Resume through `/status` after interruption. A pending selection is idempotent. +6. Stop only when the persisted state reports a terminal reason. + +Never send hidden text, targets, or expected outputs to OpenScience. Treat numeric features as fixed side information, not agent-generated descriptions of benchmark answers. + +## Interpret the result + +Report posterior mean loss, standard deviation, 95% interval, discovered failures, stratum coverage, sample count, pool fingerprint, artifact hash, and stop reason. Preserve `abstain: true` whenever the minimum sample count or uncertainty requirement is unmet. + +An active-audit estimate is not an official benchmark score. Attach its immutable receipt to a separately authenticated evaluation before using it as evidence. + +## Failure checks + +Reject the audit if: + +- probe bytes, feature vectors, weights, or thresholds changed; +- the evaluator capability or artifact hash does not match; +- selected cases cannot be resolved back to their commitments; +- failure labels contradict the frozen threshold; +- one observation is overwritten; or +- only favorable strata or failure types are reported. diff --git a/backend/cli/skills/research/active-failure-audit/agents/openai.yaml b/backend/cli/skills/research/active-failure-audit/agents/openai.yaml new file mode 100644 index 00000000..200d593d --- /dev/null +++ b/backend/cli/skills/research/active-failure-audit/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Active Failure Audit" + short_description: "Build blinded probes for active evaluation" + default_prompt: "Use $active-failure-audit to prepare an opaque committed probe pool and an uncertainty-aware audit protocol." diff --git a/backend/cli/skills/research/active-failure-audit/scripts/build_probe_manifest.py b/backend/cli/skills/research/active-failure-audit/scripts/build_probe_manifest.py new file mode 100755 index 00000000..d0e522a4 --- /dev/null +++ b/backend/cli/skills/research/active-failure-audit/scripts/build_probe_manifest.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Build a public active-audit manifest without copying hidden probe content.""" + +import argparse +import hashlib +import json +import math +import os +import sys +import tempfile +from pathlib import Path + + +def fail(message: str) -> None: + raise ValueError(message) + + +def canonical(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("input", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + + if args.output.exists() and not args.force: + fail(f"output already exists: {args.output}") + + rows = [] + for number, line in enumerate(args.input.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as error: + fail(f"line {number} is not valid JSON: {error.msg}") + if not isinstance(item, dict): + fail(f"line {number} must be an object") + if "hidden" not in item: + fail(f"line {number} is missing hidden content") + identifier = item.get("id") + features = item.get("features") + stratum = item.get("stratum") + if not isinstance(identifier, str) or not identifier or len(identifier) > 240: + fail(f"line {number} has an invalid id") + if not isinstance(stratum, str) or not stratum or len(stratum) > 120: + fail(f"line {number} has an invalid stratum") + if not isinstance(features, list) or not 1 <= len(features) <= 32: + fail(f"line {number} must contain 1 to 32 features") + if any(isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) for value in features): + fail(f"line {number} contains a non-finite numeric feature") + weight = item.get("weight", 1) + prior = item.get("priorLoss", 0.5) + if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not 0 < weight <= 1000: + fail(f"line {number} has an invalid weight") + if isinstance(prior, bool) or not isinstance(prior, (int, float)) or not 0 <= prior <= 1: + fail(f"line {number} has an invalid priorLoss") + rows.append( + { + "id": identifier, + "commitment": hashlib.sha256(canonical(item["hidden"])).hexdigest(), + "features": features, + "stratum": stratum, + "weight": weight, + "priorLoss": prior, + } + ) + + if len(rows) < 2: + fail("at least two non-empty probes are required") + if len({row["id"] for row in rows}) != len(rows): + fail("probe ids must be unique") + if len({row["commitment"] for row in rows}) != len(rows): + fail("hidden probe commitments must be unique") + if len({len(row["features"]) for row in rows}) != 1: + fail("all probes must share one feature dimension") + + probes = sorted(rows, key=lambda row: row["id"]) + payload = { + "schemaVersion": 1, + "manifestSHA256": hashlib.sha256(canonical(probes)).hexdigest(), + "probes": probes, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=f".{args.output.name}.", dir=args.output.parent) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, ensure_ascii=False) + handle.write("\n") + os.replace(temporary, args.output) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + print(json.dumps({"probes": len(probes), "manifestSHA256": payload["manifestSHA256"]})) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError) as error: + print(json.dumps({"error": str(error)}), file=sys.stderr) + raise SystemExit(2) diff --git a/backend/cli/skills/research/audit-scientific-meaning/SKILL.md b/backend/cli/skills/research/audit-scientific-meaning/SKILL.md new file mode 100644 index 00000000..fe82f3b3 --- /dev/null +++ b/backend/cli/skills/research/audit-scientific-meaning/SKILL.md @@ -0,0 +1,49 @@ +--- +name: audit-scientific-meaning +description: Prepare and preflight an independent OpenScience semantic-audit panel for one run or candidate. Use when a bound semantic-audit-v1 contract requires reviewers to distinguish meaningful scientific resolution from a technically correct but misinterpreted, vacuous, known, rediscovered, or ambiguous result before final evaluation. +--- + +# Audit scientific meaning + +Use this skill only from the independent semantic-review process. It creates evidence-bearing reviewer records for `POST /harness/semantics/receipts`; it never assigns the final receipt status or benchmark score. + +## Workflow + +1. Read the immutable harness contract and locate `semanticAudit`. +2. Verify `semanticAudit.scope.objectiveSHA256` against the exact contract objective. +3. Give each reviewer the frozen objective, criteria, forbidden shortcuts, literature cutoff, corpus commitment, novelty floor, and candidate artifact. Keep reviewer sessions and actors distinct. +4. Each reviewer independently checks: + - factual or mathematical correctness; + - alignment with the intended problem rather than a weaker interpretation; + - whether the result is substantive rather than vacuous; + - every frozen criterion and forbidden shortcut; + - novelty only against literature within the frozen cutoff and corpus scope. +5. Require observable evidence for every criterion and shortcut judgment. A citation or URI is a pointer, not proof that its content supports the judgment. +6. Preserve `inconclusive` and `ambiguous` when evidence cannot decide. Do not lower a novelty floor or reinterpret the objective after seeing the candidate. +7. Build a token-free JSON submission containing `sessionID`, `subject`, and `reviews`. +8. Preflight it: + +```bash +python scripts/validate_submission.py contract.json semantic-submission.json +``` + +9. Inject `reviewerToken` only in memory immediately before the authenticated request. Never write it to the submission or evidence bundle. +10. Store the returned receipt even when it is `technical_only`, `ambiguous`, or `failed`; negative semantic evidence must remain durable. + +## Outcome semantics + +- `meaningful`: every review is correct, aligned, substantive, complete, sufficiently confident, and at or above the novelty floor. +- `technical_only`: the work may be technically valid but misinterprets the problem, is vacuous, uses a forbidden shortcut, fails a frozen criterion, or falls below the novelty floor. +- `ambiguous`: material correctness, alignment, criteria, or reviewer confidence remains unresolved. +- `failed`: at least one reviewer finds the result incorrect. + +The backend recomputes these outcomes and binds the receipt to the exact contract and subject. Reviewers must never submit a desired aggregate status. + +## Guardrails + +- Do not show one reviewer another review before every record is frozen. +- Do not infer novelty from absence in a casual search; cite the frozen corpus search evidence. +- Do not treat independent rediscovery as novel when the contract requires minor, publication-grade, or major novelty. +- Do not accept a mathematically or technically valid loophole that an expert would recognize as outside the intended question. +- Do not replace required physical, statistical, biological, chemical, or empirical validation with semantic review. +- Read [references/protocol.md](references/protocol.md) for the exact decision precedence and novelty scale. diff --git a/backend/cli/skills/research/audit-scientific-meaning/agents/openai.yaml b/backend/cli/skills/research/audit-scientific-meaning/agents/openai.yaml new file mode 100644 index 00000000..7ad4634c --- /dev/null +++ b/backend/cli/skills/research/audit-scientific-meaning/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Audit scientific meaning" + short_description: "Preflight independent intent and novelty reviews" + default_prompt: "Audit this result against its frozen scientific objective, shortcuts, and literature scope, preserving ambiguity and evidence." diff --git a/backend/cli/skills/research/audit-scientific-meaning/references/protocol.md b/backend/cli/skills/research/audit-scientific-meaning/references/protocol.md new file mode 100644 index 00000000..fdb6eada --- /dev/null +++ b/backend/cli/skills/research/audit-scientific-meaning/references/protocol.md @@ -0,0 +1,31 @@ +# Semantic-audit-v1 protocol + +The contract freezes the exact objective hash, required criteria, forbidden shortcuts, literature cutoff, corpus commitment, novelty floor, minimum independent reviewer count, and minimum confidence before candidate execution. + +## Backend precedence + +1. Any `correctness=failed` review derives `failed`. +2. Otherwise, any inconclusive correctness or criterion, ambiguous alignment, or confidence below the contract threshold derives `ambiguous`. +3. Otherwise, any misinterpretation, vacuity, failed criterion, observed shortcut, or novelty below the frozen floor derives `technical_only`. +4. Only a panel with none of those conditions derives `meaningful`. + +The order matters: a wrong answer is not merely a weak novelty result, and missing evidence is not permission to call a loophole meaningful. + +## Novelty scale + +| Level | Meaning | +|---|---| +| `not_required` | The contract makes no novelty claim. | +| `known` | The same substantive result is already in the frozen literature scope. | +| `rediscovery` | The candidate independently reaches an existing substantive result. | +| `minor` | A new result or method with limited research novelty. | +| `publication` | A result plausibly meeting ordinary peer-reviewed research novelty. | +| `major` | A field-level advance requiring exceptional evidence. | + +When the floor is above `not_required`, every reviewer must provide literature-search evidence. A `not_required` review can pass only when the frozen floor is also `not_required`; it is not an alias for `known`. The backend compares ordered levels; review prose cannot override the frozen floor. + +## Why this exists + +[Aletheia](https://arxiv.org/abs/2602.10177) reported that 63 of 200 audited Erdős responses were technically correct, but only 13 addressed the intended problem meaningfully; many valid answers exploited trivial interpretations. The 2026 [Co-Scientist Nature paper](https://www.nature.com/articles/s41586-026-10644-y) instead freezes research goals with attributes and constraints, evaluates novelty relative to publication, uses independent experts, and validates consequential claims experimentally. A 2026 [specification-gaming study](https://arxiv.org/abs/2605.02269) found that prompt-time mitigations reduce but do not eliminate exploitation. + +Semantic review therefore remains separate from ordinary correctness, domain validation, and benchmark scoring. diff --git a/backend/cli/skills/research/audit-scientific-meaning/scripts/validate_submission.py b/backend/cli/skills/research/audit-scientific-meaning/scripts/validate_submission.py new file mode 100755 index 00000000..de7c6f58 --- /dev/null +++ b/backend/cli/skills/research/audit-scientific-meaning/scripts/validate_submission.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +import hashlib +import json +import math +import sys +from pathlib import Path + + +def fail(message: str) -> None: + raise SystemExit(f"invalid semantic audit submission: {message}") + + +if len(sys.argv) != 3: + fail("usage: validate_submission.py ") + +contract = json.loads(Path(sys.argv[1]).read_text()) +submission = json.loads(Path(sys.argv[2]).read_text()) +if not isinstance(contract, dict) or not isinstance(submission, dict): + fail("contract and submission must be objects") +if "reviewerToken" in submission or any("token" in key.lower() for key in submission): + fail("submission must remain token-free on disk") + +protocol = contract.get("semanticAudit") +if not isinstance(protocol, dict) or protocol.get("protocolVersion") != "semantic-audit-v1": + fail("contract does not contain semantic-audit-v1") +scope = protocol.get("scope") +if not isinstance(scope, dict): + fail("semantic scope is required") +objective = contract.get("objective") +if not isinstance(objective, str) or hashlib.sha256(objective.encode()).hexdigest() != scope.get("objectiveSHA256"): + fail("objective commitment does not match the contract objective") +if submission.get("sessionID") != contract.get("sessionID"): + fail("submission session does not match the contract") +subject = submission.get("subject") +if not isinstance(subject, dict) or subject.get("type") not in {"run", "candidate"} or not subject.get("id"): + fail("subject must identify one run or candidate") +if subject.get("type") == "run" and subject.get("id") != contract.get("runID"): + fail("run subject does not match the contract") + +reviews = submission.get("reviews") +minimum = protocol.get("minReviewers") +if not isinstance(reviews, list) or not isinstance(minimum, int) or len(reviews) < minimum or len(reviews) > 5: + fail("review panel does not meet the frozen size") +actors = [item.get("actor") for item in reviews if isinstance(item, dict)] +sessions = [item.get("sessionID") for item in reviews if isinstance(item, dict)] +if len(actors) != len(reviews) or len(set(actors)) != len(reviews) or not all(actors): + fail("reviewers must use distinct non-empty actors") +if len(sessions) != len(reviews) or len(set(sessions)) != len(reviews) or not all(sessions): + fail("reviewers must use distinct non-empty sessions") + +criterion_ids = sorted(item.get("id") for item in scope.get("criteria", []) if isinstance(item, dict)) +shortcut_ids = sorted(item.get("id") for item in scope.get("forbiddenShortcuts", []) if isinstance(item, dict)) +levels = {"not_required": -1, "known": 0, "rediscovery": 1, "minor": 2, "publication": 3, "major": 4} +floor = scope.get("noveltyFloor") +if not criterion_ids or not shortcut_ids or floor not in levels: + fail("contract semantic scope is incomplete") +threshold = protocol.get("minConfidence") +if isinstance(threshold, bool) or not isinstance(threshold, (int, float)) or not math.isfinite(threshold): + fail("contract minimum confidence is invalid") + +incorrect = [] +uncertain = [] +technical = [] +for review in reviews: + if not isinstance(review, dict): + fail("every review must be an object") + if review.get("correctness") not in {"passed", "failed", "inconclusive"}: + fail("review correctness is invalid") + if review.get("alignment") not in {"intended", "reasonable_alternative", "misinterpreted", "ambiguous"}: + fail("review alignment is invalid") + confidence = review.get("confidence") + if ( + review.get("novelty") not in levels + or isinstance(confidence, bool) + or not isinstance(confidence, (int, float)) + or not math.isfinite(confidence) + or confidence < 0 + or confidence > 1 + ): + fail("review novelty or confidence is invalid") + if not isinstance(review.get("vacuous"), bool): + fail("review vacuity judgment is invalid") + criteria = review.get("criteria") + shortcuts = review.get("shortcuts") + if not isinstance(criteria, list) or sorted(item.get("id") for item in criteria if isinstance(item, dict)) != criterion_ids: + fail("review criteria do not match the frozen scope") + if not isinstance(shortcuts, list) or sorted(item.get("id") for item in shortcuts if isinstance(item, dict)) != shortcut_ids: + fail("review shortcuts do not match the frozen scope") + if any(item.get("status") not in {"passed", "failed", "inconclusive"} for item in criteria): + fail("review criterion status is invalid") + if any(not isinstance(item.get("observed"), bool) for item in shortcuts): + fail("review shortcut judgment is invalid") + evidence_sets = [review.get("evidence"), *[item.get("evidence") for item in criteria], *[item.get("evidence") for item in shortcuts]] + if any(not isinstance(items, list) or not items or not all(isinstance(ref, str) and ref for ref in items) for items in evidence_sets): + fail("every review judgment needs observable evidence") + literature = review.get("literatureRefs") + if floor != "not_required" and ( + not isinstance(literature, list) or not literature or not all(isinstance(ref, str) and ref for ref in literature) + ): + fail("novelty review needs frozen-scope literature evidence") + actor = review["actor"] + if review["correctness"] == "failed": + incorrect.append(f"{actor}:correctness_failed") + if review["correctness"] == "inconclusive" or review["alignment"] == "ambiguous" or review["confidence"] < threshold: + uncertain.append(f"{actor}:uncertain") + if any(item.get("status") == "inconclusive" for item in criteria): + uncertain.append(f"{actor}:criterion_inconclusive") + if review["alignment"] == "misinterpreted" or review.get("vacuous"): + technical.append(f"{actor}:intent_or_vacuity") + if any(item.get("status") == "failed" for item in criteria) or any(item.get("observed") for item in shortcuts): + technical.append(f"{actor}:criterion_or_shortcut") + if levels[review["novelty"]] < levels[floor]: + technical.append(f"{actor}:below_novelty_floor") + +status = "failed" if incorrect else "ambiguous" if uncertain else "technical_only" if technical else "meaningful" +print(json.dumps({ + "valid": True, + "derivedStatus": status, + "reviewers": len(reviews), + "subject": subject, + "objectiveSHA256": scope["objectiveSHA256"], + "failures": [*incorrect, *uncertain, *technical], +}, sort_keys=True)) diff --git a/backend/cli/skills/research/design-replay-interventions/SKILL.md b/backend/cli/skills/research/design-replay-interventions/SKILL.md new file mode 100644 index 00000000..05f4785a --- /dev/null +++ b/backend/cli/skills/research/design-replay-interventions/SKILL.md @@ -0,0 +1,61 @@ +--- +name: design-replay-interventions +description: Build and validate an evaluator-owned OpenScience controlled intervention plan for an evolved candidate. Use when an intervention-study-v1 contract requires same-condition replay, constants-only retuning, component ablation, repair, or model, context, evaluator, or split transfer evidence before final candidate promotion. +--- + +# Design Replay Interventions + +Run this skill in the evaluator process after the candidate's evolution trace is recorded and before its final evaluation. It validates exact one-difference pairs and creates the token-free body for `POST /harness/interventions`. + +## Workflow + +1. Freeze `intervention-study-v1` in the run contract before search. Choose required families, pair limits, a 95% confidence policy, direction-aware thresholds, and the exact SHA-256 of `scripts/design_interventions.py`. +2. Select the candidate and its exact `evolutionReceiptID`. Never accept candidate-authored intervention plans, transforms, contexts, or outcome evidence. +3. Build at least the contract's `minPairs` pairs per required family. Use distinct seeds or cases. Each pair has exact control and arm artifacts, model, context, evaluator, split manifest, environment, and budget commitments. +4. Make only the family-permitted change: + - `replay`: no target or condition change; execute the same cell independently. + - `retune`, `ablation`, `repair`: change only the artifact; the arm must be the study candidate. + - `model_transfer`, `context_transfer`, `evaluator_transfer`, `split_transfer`: change only the named condition; both cells use the study candidate. +5. Run the builder. It rejects missing families, sparse indexes, duplicated pairs, wrong rule modes, excess pairs, and every uncommitted extra difference. +6. Inject `evaluatorToken` only into the authenticated initialization request in memory. Never write it to the plan, pair report, evidence, or logs. +7. Execute both cells independently and record each exact target with `POST /harness/interventions/:candidateID/observations`. +8. Call the assessment only after every frozen outcome is recorded. The backend recomputes paired effects, t intervals, replay stability, tuning gap, component dependence, recovery, and transfer robustness. +9. Reference the passing `interventionReceiptID` on the later final candidate evaluation when `requiredForPromotion` is true. + +## Commands + +Print the immutable validator commitment: + +```bash +python scripts/design_interventions.py commitments +``` + +Validate a protocol and pair specification, then build a token-free request and target ledger: + +```bash +python scripts/design_interventions.py build \ + --contract intervention-contract.json \ + --spec intervention-pairs.json \ + --output intervention-initialize.json \ + --report intervention-targets.json +``` + +See [references/spec-schema.json](references/spec-schema.json) for the exact input shape. + +## Interpretation + +- Replay passes only when every absolute paired score difference stays within its frozen threshold. +- Retuning, ablation, and repair use a direction-aware lower 95% confidence bound above `min_effect`, with no regressing pair. +- Transfer passes only when no pair exceeds `max_regression` and the lower 95% confidence bound remains above the negative threshold. +- Failed or missing executions produce a failed receipt, not a silently smaller sample. +- These results qualify causal or robustness claims. They never become candidate fitness, a benchmark score, or proof of scientific novelty. + +## Fail-Closed Rules + +- Freeze the matrix before the candidate's final evaluation. +- Bind the exact candidate artifact and prior evolution receipt. +- Keep pair indexes contiguous from zero within every family. +- Use a content-addressed `change` artifact to identify the transform or execution protocol for every pair. +- Do not reuse a score or evidence item across nominally independent repetitions. +- Do not infer a semantic constants-only edit or valid ablation from a filename. The evaluator must generate and verify transforms outside the candidate sandbox. +- Treat an inconclusive confidence interval as inconclusive, never as a pass. diff --git a/backend/cli/skills/research/design-replay-interventions/agents/openai.yaml b/backend/cli/skills/research/design-replay-interventions/agents/openai.yaml new file mode 100644 index 00000000..37f67845 --- /dev/null +++ b/backend/cli/skills/research/design-replay-interventions/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Design Replay Interventions" + short_description: "Freeze controlled replay and transfer evidence" + default_prompt: "Use $design-replay-interventions to build and validate an evaluator-owned controlled intervention plan for an evolved benchmark candidate." diff --git a/backend/cli/skills/research/design-replay-interventions/references/spec-schema.json b/backend/cli/skills/research/design-replay-interventions/references/spec-schema.json new file mode 100644 index 00000000..929f8564 --- /dev/null +++ b/backend/cli/skills/research/design-replay-interventions/references/spec-schema.json @@ -0,0 +1,105 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://syntheticsciences.ai/schemas/intervention-study-spec-v1.json", + "title": "OpenScience controlled intervention specification", + "type": "object", + "required": ["schemaVersion", "runID", "sessionID", "subject", "evolutionReceiptID", "pairs"], + "properties": { + "schemaVersion": { "const": 1 }, + "runID": { "type": "string", "minLength": 1 }, + "sessionID": { "type": "string", "minLength": 1 }, + "subject": { + "type": "object", + "required": ["type", "id", "artifact"], + "properties": { + "type": { "const": "candidate" }, + "id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "artifact": { "$ref": "#/$defs/artifact" } + }, + "additionalProperties": false + }, + "evolutionReceiptID": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "pairs": { + "type": "array", + "minItems": 3, + "maxItems": 256, + "items": { + "type": "object", + "required": ["family", "index", "control", "arm", "change"], + "properties": { + "family": { + "enum": ["replay", "retune", "ablation", "repair", "model_transfer", "context_transfer", "evaluator_transfer", "split_transfer"] + }, + "index": { "type": "integer", "minimum": 0, "maximum": 31 }, + "control": { "$ref": "#/$defs/target" }, + "arm": { "$ref": "#/$defs/target" }, + "change": { "$ref": "#/$defs/artifact" } + }, + "additionalProperties": false + } + } + }, + "$defs": { + "artifact": { + "type": "object", + "required": ["uri", "sha256"], + "properties": { + "uri": { "type": "string", "minLength": 1 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + }, + "additionalProperties": false + }, + "target": { + "type": "object", + "required": ["artifact", "condition"], + "properties": { + "artifact": { "$ref": "#/$defs/artifact" }, + "condition": { + "type": "object", + "required": ["seed", "model", "context", "evaluator", "split", "environment", "budget"], + "properties": { + "seed": { "type": "integer" }, + "model": { "$ref": "#/$defs/model" }, + "context": { "$ref": "#/$defs/artifact" }, + "evaluator": { "$ref": "#/$defs/evaluator" }, + "split": { "$ref": "#/$defs/split" }, + "environment": { "$ref": "#/$defs/artifact" }, + "budget": { "$ref": "#/$defs/artifact" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "model": { + "type": "object", + "required": ["provider", "name", "version"], + "properties": { + "provider": { "type": "string", "minLength": 1 }, + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "required": ["name", "version", "source"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "source": { "enum": ["benchmark", "gate", "external"] } + }, + "additionalProperties": false + }, + "split": { + "type": "object", + "required": ["name", "manifest"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "manifest": { "$ref": "#/$defs/artifact" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false +} diff --git a/backend/cli/skills/research/design-replay-interventions/scripts/design_interventions.py b/backend/cli/skills/research/design-replay-interventions/scripts/design_interventions.py new file mode 100755 index 00000000..2c8511ff --- /dev/null +++ b/backend/cli/skills/research/design-replay-interventions/scripts/design_interventions.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Validate and build evaluator-owned OpenScience intervention plans.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +FAMILIES = { + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer", +} +MODES = { + "replay": "max_absolute_effect", + "retune": "min_effect", + "ablation": "min_effect", + "repair": "min_effect", + "model_transfer": "max_regression", + "context_transfer": "max_regression", + "evaluator_transfer": "max_regression", + "split_transfer": "max_regression", +} +HASH = set("0123456789abcdef") + + +class Invalid(ValueError): + pass + + +def canonical(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def sha(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def file_sha(path: Path) -> str: + return sha(path.read_bytes()) + + +def load(path: Path, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise Invalid(f"cannot read {label} {path}: {exc}") from exc + if not isinstance(value, dict): + raise Invalid(f"{label} must be a JSON object") + return value + + +def text(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise Invalid(f"{label} must be a non-empty string") + return value + + +def digest(value: Any, label: str) -> str: + item = text(value, label) + if len(item) != 64 or set(item) - HASH: + raise Invalid(f"{label} must be a lowercase SHA-256") + return item + + +def integer(value: Any, label: str, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum: + raise Invalid(f"{label} must be an integer from {minimum} to {maximum}") + return value + + +def artifact(value: Any, label: str) -> dict[str, str]: + if not isinstance(value, dict) or set(value) != {"uri", "sha256"}: + raise Invalid(f"{label} must contain only uri and sha256") + return {"uri": text(value.get("uri"), f"{label}.uri"), "sha256": digest(value.get("sha256"), f"{label}.sha256")} + + +def protocol(path: Path) -> dict[str, Any]: + value = load(path, "contract") + expected = { + "protocolVersion", + "validatorSHA256", + "requiredForPromotion", + "minPairs", + "maxPairs", + "maxTotalPairs", + "confidence", + "required", + "rules", + } + if set(value) != expected: + raise Invalid(f"contract must contain exactly {sorted(expected)}") + if value.get("protocolVersion") != "intervention-study-v1": + raise Invalid("contract.protocolVersion must be intervention-study-v1") + if digest(value.get("validatorSHA256"), "contract.validatorSHA256") != file_sha(Path(__file__).resolve()): + raise Invalid("contract validatorSHA256 does not match this exact script") + if value.get("confidence") != 0.95: + raise Invalid("contract.confidence must be 0.95") + if not isinstance(value.get("requiredForPromotion"), bool): + raise Invalid("contract.requiredForPromotion must be boolean") + minimum = integer(value.get("minPairs"), "contract.minPairs", 3, 32) + maximum = integer(value.get("maxPairs"), "contract.maxPairs", 3, 32) + total = integer(value.get("maxTotalPairs"), "contract.maxTotalPairs", 3, 256) + if minimum > maximum: + raise Invalid("contract.maxPairs cannot be smaller than minPairs") + required = value.get("required") + if not isinstance(required, list) or not required or any(item not in FAMILIES for item in required): + raise Invalid("contract.required must contain known intervention families") + if required != sorted(set(required)): + raise Invalid("contract.required must be unique and sorted") + rules = value.get("rules") + if not isinstance(rules, list) or len(rules) != len(required): + raise Invalid("contract.rules must cover every required family exactly once") + seen = [] + for rule in rules: + if not isinstance(rule, dict) or set(rule) != {"family", "mode", "threshold"}: + raise Invalid("each contract rule must contain family, mode, and threshold") + family = rule.get("family") + if family not in required or rule.get("mode") != MODES[family]: + raise Invalid(f"contract rule mode is invalid for {family}") + threshold = rule.get("threshold") + if isinstance(threshold, bool) or not isinstance(threshold, (int, float)) or not math.isfinite(threshold): + raise Invalid(f"contract rule threshold is invalid for {family}") + if threshold < 0: + raise Invalid(f"contract rule threshold must be nonnegative for {family}") + seen.append(family) + if seen != required: + raise Invalid("contract.rules must be family-sorted and match required") + if maximum * len(required) > total: + raise Invalid("contract.maxTotalPairs cannot fit every required family") + return value + + +def model(value: Any, label: str) -> dict[str, str]: + if not isinstance(value, dict) or set(value) != {"provider", "name", "version"}: + raise Invalid(f"{label} must contain provider, name, and version") + return {key: text(value.get(key), f"{label}.{key}") for key in ["provider", "name", "version"]} + + +def evaluator(value: Any, label: str) -> dict[str, str]: + if not isinstance(value, dict) or set(value) != {"name", "version", "source"}: + raise Invalid(f"{label} must contain name, version, and source") + source = value.get("source") + if source not in {"benchmark", "gate", "external"}: + raise Invalid(f"{label}.source is invalid") + return {"name": text(value.get("name"), f"{label}.name"), "version": text(value.get("version"), f"{label}.version"), "source": source} + + +def target(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != {"artifact", "condition"}: + raise Invalid(f"{label} must contain artifact and condition") + condition = value.get("condition") + expected = {"seed", "model", "context", "evaluator", "split", "environment", "budget"} + if not isinstance(condition, dict) or set(condition) != expected: + raise Invalid(f"{label}.condition must contain exactly {sorted(expected)}") + split = condition.get("split") + if not isinstance(split, dict) or set(split) != {"name", "manifest"}: + raise Invalid(f"{label}.condition.split must contain name and manifest") + return { + "artifact": artifact(value.get("artifact"), f"{label}.artifact"), + "condition": { + "seed": integer(condition.get("seed"), f"{label}.condition.seed", -(2**53), 2**53), + "model": model(condition.get("model"), f"{label}.condition.model"), + "context": artifact(condition.get("context"), f"{label}.condition.context"), + "evaluator": evaluator(condition.get("evaluator"), f"{label}.condition.evaluator"), + "split": { + "name": text(split.get("name"), f"{label}.condition.split.name"), + "manifest": artifact(split.get("manifest"), f"{label}.condition.split.manifest"), + }, + "environment": artifact(condition.get("environment"), f"{label}.condition.environment"), + "budget": artifact(condition.get("budget"), f"{label}.condition.budget"), + }, + } + + +def changes(control: dict[str, Any], arm: dict[str, Any]) -> list[str]: + fields = ["artifact", "model", "context", "evaluator", "split", "environment", "budget", "seed"] + result = [] + for field in fields: + left = control["artifact"] if field == "artifact" else control["condition"][field] + right = arm["artifact"] if field == "artifact" else arm["condition"][field] + if left != right: + result.append(field) + return result + + +def pair(value: Any, subject_artifact: dict[str, str], label: str) -> dict[str, Any]: + expected = {"family", "index", "control", "arm", "change"} + if not isinstance(value, dict) or set(value) != expected: + raise Invalid(f"{label} must contain exactly {sorted(expected)}") + family = value.get("family") + if family not in FAMILIES: + raise Invalid(f"{label}.family is invalid") + control = target(value.get("control"), f"{label}.control") + arm = target(value.get("arm"), f"{label}.arm") + changed = changes(control, arm) + if family == "replay": + if changed or arm["artifact"] != subject_artifact: + raise Invalid(f"{label}: replay must repeat the exact study subject and condition") + elif family in {"retune", "ablation", "repair"}: + if changed != ["artifact"] or arm["artifact"] != subject_artifact: + raise Invalid(f"{label}: {family} may change only the artifact and its arm must be the study subject") + else: + field = family.removesuffix("_transfer") + if changed != [field] or control["artifact"] != subject_artifact or arm["artifact"] != subject_artifact: + raise Invalid(f"{label}: {family} may change only {field} while evaluating the study subject") + return { + "family": family, + "index": integer(value.get("index"), f"{label}.index", 0, 31), + "control": control, + "arm": arm, + "change": artifact(value.get("change"), f"{label}.change"), + } + + +def build(contract_path: Path, spec_path: Path) -> tuple[dict[str, Any], dict[str, Any]]: + frozen = protocol(contract_path) + spec = load(spec_path, "spec") + expected = {"schemaVersion", "runID", "sessionID", "subject", "evolutionReceiptID", "pairs"} + if set(spec) != expected: + raise Invalid(f"spec must contain exactly {sorted(expected)}") + if spec.get("schemaVersion") != 1: + raise Invalid("spec.schemaVersion must be 1") + subject = spec.get("subject") + if not isinstance(subject, dict) or set(subject) != {"type", "id", "artifact"} or subject.get("type") != "candidate": + raise Invalid("spec.subject must be an exact candidate subject") + candidate = { + "type": "candidate", + "id": digest(subject.get("id"), "spec.subject.id"), + "artifact": artifact(subject.get("artifact"), "spec.subject.artifact"), + } + values = spec.get("pairs") + if not isinstance(values, list): + raise Invalid("spec.pairs must be an array") + pairs = [pair(value, candidate["artifact"], f"spec.pairs[{index}]") for index, value in enumerate(values)] + pairs.sort(key=lambda item: (item["family"], item["index"])) + if len({sha(canonical(item)) for item in pairs}) != len(pairs): + raise Invalid("spec.pairs must be unique") + families = sorted({item["family"] for item in pairs}) + if families != frozen["required"]: + raise Invalid("spec.pairs must cover exactly the required families") + for family in families: + items = [item for item in pairs if item["family"] == family] + if not frozen["minPairs"] <= len(items) <= frozen["maxPairs"]: + raise Invalid(f"{family} violates the frozen pair bounds") + if [item["index"] for item in items] != list(range(len(items))): + raise Invalid(f"{family} indexes must be contiguous from zero") + if len(pairs) > frozen["maxTotalPairs"]: + raise Invalid("spec.pairs exceeds contract.maxTotalPairs") + request = { + "schemaVersion": 1, + "runID": text(spec.get("runID"), "spec.runID"), + "sessionID": text(spec.get("sessionID"), "spec.sessionID"), + "subject": candidate, + "evolutionReceiptID": digest(spec.get("evolutionReceiptID"), "spec.evolutionReceiptID"), + "validator": { + "name": "design-replay-interventions", + "version": 1, + "scriptSHA256": file_sha(Path(__file__).resolve()), + }, + "pairs": pairs, + } + report = { + "schemaVersion": 1, + "candidateID": candidate["id"], + "families": {family: len([item for item in pairs if item["family"] == family]) for family in families}, + "targets": [ + { + "family": item["family"], + "index": item["index"], + "controlSHA256": sha(canonical(item["control"])), + "armSHA256": sha(canonical(item["arm"])), + "changeSHA256": item["change"]["sha256"], + } + for item in pairs + ], + } + return request, report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("commitments") + builder = commands.add_parser("build") + builder.add_argument("--contract", type=Path, required=True) + builder.add_argument("--spec", type=Path, required=True) + builder.add_argument("--output", type=Path, required=True) + builder.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + if args.command == "commitments": + print(json.dumps({"validatorSHA256": file_sha(Path(__file__).resolve())}, sort_keys=True)) + return 0 + request, report = build(args.contract, args.spec) + args.output.write_bytes(canonical(request) + b"\n") + args.report.write_bytes(canonical(report) + b"\n") + print(json.dumps({"output": str(args.output), "report": str(args.report), "pairs": len(request["pairs"])}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Invalid as exc: + print(json.dumps({"error": str(exc)}), file=sys.stderr) + raise SystemExit(2) from exc diff --git a/backend/cli/skills/research/evolve-meta-harness/SKILL.md b/backend/cli/skills/research/evolve-meta-harness/SKILL.md new file mode 100644 index 00000000..4e027572 --- /dev/null +++ b/backend/cli/skills/research/evolve-meta-harness/SKILL.md @@ -0,0 +1,45 @@ +--- +name: evolve-meta-harness +description: Evolve an OpenScience prompt, memory, skill, tool, middleware, subagent, or scaffold as a versioned, trace-grounded harness and qualify it on frozen search plus unseen model/task cells before sealed promotion. Use when meta-harness-v1 is bound, when refining agent instructions from execution failures, or when activation, adherence, phase drift, context cost, cross-model transfer, rollback, and full candidate-history evidence must be audited. +--- + +# Evolve Meta Harness + +Run this skill across two isolated roles: an updater may inspect only frozen search evidence and propose session-local deltas; the qualifier owns unseen models/tasks, adherence labels, and the one-shot receipt. Never let either role edit the benchmark evaluator, hidden inputs, qualification validator, protected roots, or baseline. + +## Workflow + +1. Freeze `meta-harness-v1` before search. Commit the immutable baseline artifact and manifests; exact mutable component roots; protected roots; archive and trace schemas; updater and judge identities; sorted search/held-out model-task matrices; and numeric promotion thresholds. +2. Preserve the baseline. Save each accepted delta as a new content-addressed snapshot with its parent hash. Use atomic writes and explicit rollback revisions. Local/session scope is the default; global promotion requires independent qualification. +3. Retain every search candidate, including failures and unevaluated proposals. Store full source, scores, and raw traces in the filesystem archive. A summary, reflection, or compressed memory is not a substitute for trace bytes. +4. For every refinement, cite an archived trace message, diagnose implementation versus fundamental failure, state the root cause, enumerate exact file/component changes, predict search-task flips or protected passing cells, and state the expected outcome before evaluation. +5. Terminate search before qualification. Request `POST /harness/meta/selection` with the qualifier capability. Do not accept a caller-selected candidate. +6. Run the exact baseline and selected candidate across every frozen model-task pair. Keep held-out models/tasks inaccessible to the updater. On activation-required candidate cells, record whether the harness loaded plus judge counts at `loaded`, `midpoint`, `pre_final`, and `final_validation`. +7. Build a token-free body with `scripts/build_submission.ts`. Inject `metaToken` only into the authenticated request in memory, then call `POST /harness/meta/receipts` once. +8. Proceed to sealed confirmation only when the backend-derived receipt is `passed`. Treat diagnostic gains, activation, adherence, drift, prediction precision, and loaded benefit as qualification evidence—not the official benchmark score. + +## Build + +```bash +bun skills/research/evolve-meta-harness/scripts/build_submission.ts \ + --protocol meta-protocol.json \ + --selection meta-selection.json \ + --archive archive-input.json \ + --refinements refinements.json \ + --cells qualification-cells.json \ + --candidate-manifest \ + --output meta-submission.json +``` + +`archive-input.json` contains `{ "uri": "...", "entries": [...] }`. The builder fixes the archive policy from the protocol, hashes the sorted index and complete archive, validates the final body with the same runtime schema, and intentionally omits the capability token. + +Read [references/qualification-contract.md](references/qualification-contract.md) before constructing cells or refinements. Read [references/source-mechanisms.md](references/source-mechanisms.md) when changing the architecture or claiming provenance from upstream systems. + +## Fail Closed + +- Do not adapt on held-out model/task outcomes, even indirectly through memory, reflection, or branch selection. +- Do not overwrite snapshots or repair a frozen receipt. A failed or inconclusive first qualification remains canonical. +- Reject missing candidates, summary-only traces, stale parents, duplicate predictions, unsorted matrices, fabricated source hashes, and mutable/protected root overlap. +- Count a required action as followed only when trace evidence supports it. Use `requiredUnobserved` or `insufficientEvidence` instead of optimistic inference. +- Require complete paired cells for every baseline/candidate cross product. Missing scores remain inconclusive; they are never zero or passing. +- Report the model-harness pair. Never attribute a harness gain to the model alone. diff --git a/backend/cli/skills/research/evolve-meta-harness/agents/openai.yaml b/backend/cli/skills/research/evolve-meta-harness/agents/openai.yaml new file mode 100644 index 00000000..a1a898bf --- /dev/null +++ b/backend/cli/skills/research/evolve-meta-harness/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Evolve Meta Harness" + short_description: "Qualify harness refinements before promotion" + default_prompt: "Use $evolve-meta-harness to build and qualify a trace-grounded harness refinement lineage." diff --git a/backend/cli/skills/research/evolve-meta-harness/references/qualification-contract.md b/backend/cli/skills/research/evolve-meta-harness/references/qualification-contract.md new file mode 100644 index 00000000..a7032c98 --- /dev/null +++ b/backend/cli/skills/research/evolve-meta-harness/references/qualification-contract.md @@ -0,0 +1,36 @@ +# Qualification contract + +## Bound before search + +`meta-harness-v1` commits the baseline artifact and manifest; mutable component roots; protected manifest and roots; validator, archive-schema, and trace-schema hashes; distinct updater and adherence-judge identities; disjoint sorted search and held-out model IDs plus weights/config commitments and task IDs plus content commitments; and all promotion thresholds. Each split must contain an activation-required task. The bound beneficiary model belongs to the search set, never the held-out set. + +The qualifier capability must differ from optimization evaluation, evaluator audit, semantic audit, and sealed confirmation. The adherence judge must differ from the updater, optimization evaluator, and claim evaluator. + +## Archive entries + +One entry is required for every search candidate, sorted by candidate ID. For this protocol the search artifact is the exact evolution-source snapshot, not a mutable pointer or separately packaged surrogate. An evaluated entry includes that source hash; exact result-metric, result, and evaluation hashes; and a complete raw trace with the frozen schema. An unevaluated entry has no result or trace fields. Hidden task content and evaluator implementation are excluded from the trace archive. + +Evolution capture must cover every mutable and protected root. The candidate manifest is derived from the selected evolution snapshot's sorted `{path, sha256}` pairs. The protected subset is derived from the same bytes and must reproduce the frozen protected-manifest hash; echoing a trusted-looking hash without matching source files fails. + +The index hash is SHA-256 of JavaScript `JSON.stringify(entries)`. The archive hash is SHA-256 of `JSON.stringify` over the archive object without its `sha256` field, in protocol field order. Use the builder rather than hand-computing these hashes. + +## Refinements + +Revisions are contiguous from one. Revision one descends from the frozen baseline artifact; each later revision descends from the preceding snapshot; the last snapshot equals the backend-selected candidate artifact. Changes are sorted, unique, confined to exactly one declared mutable root, and labeled with the matching component type. + +Every revision includes a trigger, failure diagnosis (`implementation`, `fundamental`, or `inconclusive`), root cause, expected outcome, at least one archived trace citation, and sorted search-only predictions. Predictions are unique across the lineage. Held-out cells cannot appear in refinement evidence or predictions. + +## Qualification cells + +Provide one baseline and one candidate cell for every cross product: + +- every search model × search task; +- every held-out model × held-out task. + +Sort by `split`, `modelID`, `taskID`, then `role`. Every cell echoes the frozen model and task commitments; reusing an ID for changed bytes is rejected. Completed cells require numeric score and pass verdict. Failed or inconclusive cells publish neither. Every cell includes context tokens, output hash, complete trace, and evidence. + +Candidate cells add `loaded` and phase observations. If an activation-required completed cell loaded the harness, all four canonical phases are required. Counts distinguish followed, commission violation, omission violation, required-but-unobserved, not applicable, and insufficient evidence. + +## Backend-derived firewall + +Direction-normalized gains use candidate minus baseline for maximize metrics and baseline minus candidate for minimize metrics. Promotion requires search and held-out mean gains, per-model regression bounds, activation and adherence floors, phase-drift and context ceilings, prediction precision, and risk-regression limits. Any failed cell or numeric threshold breach fails. Missing or insufficient evidence is inconclusive. Both states block sealed confirmation permanently for that session receipt. diff --git a/backend/cli/skills/research/evolve-meta-harness/references/source-mechanisms.md b/backend/cli/skills/research/evolve-meta-harness/references/source-mechanisms.md new file mode 100644 index 00000000..141fad72 --- /dev/null +++ b/backend/cli/skills/research/evolve-meta-harness/references/source-mechanisms.md @@ -0,0 +1,25 @@ +# Source mechanism ledger + +Pinned source revisions used for `meta-harness-v1` design: + +| Source | Revision | Adopted mechanism | +|---|---|---| +| PrimeIntellect-ai/prime-agent | `0859d06a5da2c7642adb4130cdadf7e8aa445835` | Immutable base plus local-first versioned supplemental prompt/memory/skill/subagent state; evidence-backed refinement; atomic saves and rollback; serialized updates | +| InternScience/MLEvolve | `7d8403c899c40f01941c0429f1c4ef51e82ae41c` | Progressive graph search, exploration/exploitation control, stagnation-aware fusion, and success/failure memory | +| EvoScientist/EvoScientist | `0c21a01f6fdb4852ac26909c00fff50d098617b4` | Guarded skill proposals, read/write tiers, and implementation-versus-fundamental failure diagnosis | +| EvoScientist/EvoSkills | `2e474118106f86c29082a6466b995ba59236614c` | Confidence-tagged reusable skill memory | +| Agentic Harness Engineering | `8b2a55d97590363fe50c3cc6b5e833b020a4bb4c` | Full raw traces; trace-cited root cause and targeted fix; predicted task flips and risk tasks; falsification on the next iteration | +| Meta-Harness | `44b9942127847f7421db70d8c7e48407f09a3c70` | Updater/beneficiary separation, held-out task/model transfer, activation and adherence measurement, model-harness pair reporting | +| A-Evolve | `c9d4789f2be499589d543aa08e74d05d10d93177` | Continual harness refinement and evaluator-owned promotion checks | +| HarnessBench | `1025086a446653702b80cfb48babbeec35db6b2c` | Harness evaluation as an empirical object rather than prompt aesthetics | + +OpenScience already supplied persistent runtime goals, heartbeats, compaction, subagents, adaptive graph search, Pareto/island search, reservations, evolution provenance, evaluator isolation, and sealed confirmation. Those mechanisms were integrated rather than duplicated. + +Deliberately excluded: + +- candidate-authored or summary-only evidence; +- hidden-model/task feedback during search; +- global self-modification before independent transfer qualification; +- diagnostics as benchmark fitness; +- prompt-only evaluation without activation/adherence measurement; +- mutable receipts or post-hoc prediction edits. diff --git a/backend/cli/skills/research/evolve-meta-harness/scripts/build_submission.ts b/backend/cli/skills/research/evolve-meta-harness/scripts/build_submission.ts new file mode 100644 index 00000000..39a0c8ba --- /dev/null +++ b/backend/cli/skills/research/evolve-meta-harness/scripts/build_submission.ts @@ -0,0 +1,64 @@ +#!/usr/bin/env bun + +import path from "path" +import { HarnessContract } from "../../../../src/session/harness/contract" +import { HarnessMeta } from "../../../../src/session/harness/meta" + +const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") +const args = Bun.argv.slice(2) +const options = Object.fromEntries( + args.flatMap((value, index) => (value.startsWith("--") && args[index + 1] ? [[value.slice(2), args[index + 1]!]] : [])), +) +const required = (name: string) => { + const value = options[name] + if (!value) throw new Error(`Missing --${name}`) + return value +} +const read = async (name: string) => JSON.parse(await Bun.file(path.resolve(required(name))).text()) as unknown +const hash = (name: string) => { + const value = required(name) + if (!/^[a-f0-9]{64}$/.test(value)) throw new Error(`--${name} must be a lowercase SHA-256`) + return value +} + +const protocol = HarnessContract.MetaHarness.parse(await read("protocol")) +const selection = HarnessMeta.Selection.parse(await read("selection")) +const source = (await read("archive")) as { uri?: unknown; entries?: unknown } +if (typeof source.uri !== "string" || !Array.isArray(source.entries)) { + throw new Error(`--archive must contain a URI and entries array`) +} +const entries = source.entries.toSorted((left, right) => { + const a = (left as { candidateID?: string }).candidateID ?? "" + const b = (right as { candidateID?: string }).candidateID ?? "" + return a.localeCompare(b) +}) +const archiveBase = { + uri: source.uri, + schemaSHA256: protocol.archiveSchemaSHA256, + indexSHA256: digest(entries), + contents: protocol.archive.contents, + query: protocol.archive.query, + complete: true as const, + hiddenContent: protocol.archive.hiddenContent, + evaluatorContent: protocol.archive.evaluatorContent, + entries, +} +const archive = { ...archiveBase, sha256: digest(archiveBase) } +const input = HarnessMeta.Submit.parse({ + schemaVersion: 1, + sessionID: selection.sourceSessionID, + metaToken: "token-injected-only-at-request-time-000000000000000000", + selectionID: selection.selectionID, + candidateArtifactSHA256: selection.candidateArtifact.sha256, + candidateManifestSHA256: hash("candidate-manifest"), + protectedManifestSHA256: protocol.protected.manifestSHA256, + validatorSHA256: protocol.validatorSHA256, + archive, + refinements: await read("refinements"), + cells: await read("cells"), + evaluatedAt: Math.max(Date.now(), selection.selectedAt), +}) +const output = structuredClone(input) as Record +delete output.metaToken +await Bun.write(path.resolve(required("output")), JSON.stringify(output, null, 2) + "\n") +process.stdout.write(`${JSON.stringify({ output: path.resolve(required("output")), archiveSHA256: archive.sha256 })}\n`) diff --git a/backend/cli/skills/research/operate-adaptive-search/SKILL.md b/backend/cli/skills/research/operate-adaptive-search/SKILL.md new file mode 100644 index 00000000..098c1b3c --- /dev/null +++ b/backend/cli/skills/research/operate-adaptive-search/SKILL.md @@ -0,0 +1,32 @@ +--- +name: operate-adaptive-search +description: Execute an OpenScience adaptive-search-v1 recommendation or reservation lease. Use when a benchmark optimization run supplies a strategy, target island, verified lineage, controller snapshot, or agentic variation mandate and the next worker must produce one compliant candidate artifact without overriding evaluator-owned routing or fitness. +--- + +# Operate Adaptive Search + +Treat the lease as the authorized search action. Produce one new runnable artifact; never rewrite its lineage, target island, controller fields, or mandate. + +## Workflow + +1. Save the supplied recommendation JSON and run `python3 scripts/validate_lease.py `. Stop if it fails. +2. Read [references/lease-contract.md](references/lease-contract.md) when a field or strategy is ambiguous. +3. Load only the leased parents, inspirations, and bounded context. Official verified results are evidence; observations and screening scores are not fitness. +4. Execute the strategy: + - `seed`: build an independent baseline from the task contract. + - `explore`: pursue an orthogonal mechanism. When `control.explore` is true, prefer a stepwise redesign or full implementation over a cosmetic diff. + - `exploit`: retain the parent premise and make the smallest evidence-backed improvement. + - `migrate`: adapt the inspiration's useful mechanism to the target parent; do not copy its artifact unchanged. + - `fuse`: reconcile the two parents into one coherent implementation and resolve incompatibilities explicitly. + - `diverge`: perform meta-analysis of the verified trajectory, state a qualitatively different tactic, then implement it. Parameter-only retuning does not satisfy divergence. +5. Honor the variation mandate's operator. If strategy and operator appear in tension, satisfy both through the narrowest coherent interpretation; do not edit either assignment. +6. Run task-permitted local checks. Debug and revise within the lease, but return exactly one new artifact for external evaluation. +7. Report the implemented hypothesis, changed mechanism, local evidence, artifact URI/hash, and unresolved risks. Do not claim a benchmark improvement before the evaluator verifies it. + +## Integrity rules + +- Never inspect hidden tests, release answers, evaluator internals, or benchmark leaderboards to choose the change. +- Never feed intervention outcomes, self-reported scores, or non-final fidelity results into routing. +- Treat intensity and reward fields as controller diagnostics, not permission to change compute budgets. +- Preserve reproducibility: pin dependencies, record commands, and keep the returned artifact content-distinct from every context artifact. +- Reject stale or malformed leases instead of guessing a route. diff --git a/backend/cli/skills/research/operate-adaptive-search/agents/openai.yaml b/backend/cli/skills/research/operate-adaptive-search/agents/openai.yaml new file mode 100644 index 00000000..945c17e9 --- /dev/null +++ b/backend/cli/skills/research/operate-adaptive-search/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Operate Adaptive Search" + short_description: "Execute verified adaptive search leases" + default_prompt: "Use $operate-adaptive-search to execute this OpenScience adaptive search lease and return one compliant candidate artifact." diff --git a/backend/cli/skills/research/operate-adaptive-search/references/lease-contract.md b/backend/cli/skills/research/operate-adaptive-search/references/lease-contract.md new file mode 100644 index 00000000..57df0db8 --- /dev/null +++ b/backend/cli/skills/research/operate-adaptive-search/references/lease-contract.md @@ -0,0 +1,26 @@ +# Adaptive lease contract + +An `adaptive-search-v1` lease is evaluator-derived and content-addressed. Its controller snapshot is replayed from final verified candidate events at the lease revision. + +## Controller fields + +- `eventCount`: verified final events visible when the lease was issued. +- `selectedIsland`: island selected by minimum-visit warmup or decayed-reward UCB. +- `targetIsland`: island authorized for this candidate. Portfolio leases may target a different valid island than the serial selection. +- `visits`: verified final evaluations attributed to the target island. +- `accumulatedImprovement`: EMA of squared, direction-aware, locally normalized positive gains. It decays on every non-improvement and failed evaluation. +- `rewardMean`: decayed globally normalized gain divided by decayed visits. It routes budget; it is not fitness. +- `intensity`: exploration probability derived inversely from accumulated improvement. +- `draw`: deterministic content-derived draw. `explore` must equal `draw < intensity`. +- `globalStagnation`: true only after the patience window and when every active island signal is below the frozen threshold. +- `policySHA256`: commitment to the server-standardized controller policy. + +The primary score and declared secondary objectives remain the only fitness. Migration does not receive reward until its newly evaluated artifact earns an improvement. + +## Modes + +- `single-pass`: independent seed construction. +- `diff`: focused modification of a verified parent. +- `stepwise`: plan, implement, test, diagnose, and revise; use for fusion, migration, exploration, and meta-guided divergence. + +The lease ID commits revision, strategy, mode, sorted lineage, target island, context, and controller snapshot. The candidate ID additionally commits branch, proposal, artifact, and reservation provenance. diff --git a/backend/cli/skills/research/operate-adaptive-search/scripts/validate_lease.py b/backend/cli/skills/research/operate-adaptive-search/scripts/validate_lease.py new file mode 100755 index 00000000..45ccf18c --- /dev/null +++ b/backend/cli/skills/research/operate-adaptive-search/scripts/validate_lease.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +import hashlib +import json +import re +import sys +from pathlib import Path + + +def fail(message: str) -> None: + raise SystemExit(f"invalid adaptive lease: {message}") + + +if len(sys.argv) != 2: + raise SystemExit("usage: validate_lease.py ") + +path = Path(sys.argv[1]) +raw = sys.stdin.buffer.read() if sys.argv[1] == "-" else path.read_bytes() +value = json.loads(raw) +required = { + "id", + "revision", + "strategy", + "mode", + "parentIDs", + "inspirationIDs", + "targetIsland", + "contextIDs", + "reasons", + "control", +} +missing = required - value.keys() +if missing: + fail(f"missing fields: {', '.join(sorted(missing))}") +if not re.fullmatch(r"[a-f0-9]{64}", value["id"]): + fail("id must be a lowercase sha256") +if value["strategy"] not in {"seed", "explore", "exploit", "fuse", "migrate", "diverge"}: + fail("unknown strategy") +if value["mode"] not in {"single-pass", "stepwise", "diff"}: + fail("unknown generation mode") +if not isinstance(value["revision"], int) or value["revision"] < 0: + fail("revision must be a nonnegative integer") +if not isinstance(value["targetIsland"], int) or value["targetIsland"] < 0: + fail("targetIsland must be a nonnegative integer") +for field, limit in (("parentIDs", 2), ("inspirationIDs", 2), ("contextIDs", 6)): + items = value[field] + if not isinstance(items, list) or len(items) > limit or len(items) != len(set(items)): + fail(f"{field} must be a unique list of at most {limit} ids") + if any(not isinstance(item, str) or not re.fullmatch(r"[a-f0-9]{64}", item) for item in items): + fail(f"{field} contains a non-sha256 id") +if set(value["parentIDs"]) & set(value["inspirationIDs"]): + fail("parents and inspirations must be distinct") + +control = value["control"] +fields = { + "protocolVersion", + "policySHA256", + "eventCount", + "stalled", + "targetIsland", + "visits", + "accumulatedImprovement", + "rewardMean", + "intensity", + "draw", + "explore", + "globalStagnation", +} +if not isinstance(control, dict) or fields - control.keys(): + fail("controller snapshot is incomplete") +if control["protocolVersion"] != "adaptive-search-v1": + fail("unsupported controller protocol") +if not re.fullmatch(r"[a-f0-9]{64}", control["policySHA256"]): + fail("policySHA256 must be a lowercase sha256") +if control["targetIsland"] != value["targetIsland"]: + fail("controller and lease target islands differ") +for field in ("eventCount", "stalled", "visits"): + if not isinstance(control[field], int) or control[field] < 0: + fail(f"{field} must be a nonnegative integer") +for field in ("accumulatedImprovement", "rewardMean"): + if not isinstance(control[field], (int, float)) or control[field] < 0: + fail(f"{field} must be nonnegative") +for field in ("intensity", "draw"): + if not isinstance(control[field], (int, float)) or not 0 <= control[field] <= 1: + fail(f"{field} must be in [0, 1]") +if control["explore"] != (control["draw"] < control["intensity"]): + fail("explore does not match the deterministic intensity draw") +if value["strategy"] == "seed" and value["parentIDs"]: + fail("seed cannot have parents") +if value["strategy"] in {"exploit", "diverge"} and len(value["parentIDs"]) != 1: + fail(f"{value['strategy']} requires one parent") +if value["strategy"] == "fuse" and len(value["parentIDs"]) != 2: + fail("fuse requires two parents") +if value["strategy"] == "migrate" and (len(value["parentIDs"]) != 1 or len(value["inspirationIDs"]) != 1): + fail("migrate requires one parent and one inspiration") + +summary = { + "valid": True, + "leaseSHA256": hashlib.sha256(raw).hexdigest(), + "strategy": value["strategy"], + "mode": value["mode"], + "targetIsland": value["targetIsland"], + "eventCount": control["eventCount"], + "explore": control["explore"], + "globalStagnation": control["globalStagnation"], +} +print(json.dumps(summary, sort_keys=True)) diff --git a/backend/cli/skills/research/operate-proof-blueprint/SKILL.md b/backend/cli/skills/research/operate-proof-blueprint/SKILL.md new file mode 100644 index 00000000..83da4b70 --- /dev/null +++ b/backend/cli/skills/research/operate-proof-blueprint/SKILL.md @@ -0,0 +1,53 @@ +--- +name: operate-proof-blueprint +description: Operate evaluator-owned, verifier-grounded Lean proof search as a bounded content-addressed AND/OR blueprint. Use when a formal-proof-v1 benchmark should try direct proofs, compile decomposition sketches, share repeated lemmas, lease parallel subgoals, retain failed attempts, or refine blocked branches without treating search progress as final proof evidence. +--- + +# Operate a proof blueprint + +Keep the evaluator in control of leases, compiler execution, reviewer evidence, +and submissions. The proving agent may propose source and decompositions but +must never possess the evaluator token or report its own verifier success. + +Read [references/protocol.md](references/protocol.md) before configuring limits +or interpreting graph status. + +## Freeze the search protocol + +1. Pin the graph schema, the same Lean kernel used by `formal-proof-v1`, a + sketch validator, reviewer executable, and reviewer rubric. Treat the + reviewer only as a search heuristic. +2. Set hard node, depth, parallelism, direct-attempt, refinement, and lease + limits before search. +3. Run `bun scripts/preflight.ts protocol blueprint-manifest.json` and put the + emitted `blueprint` inside the formal-proof contract. +4. Have the evaluator call `POST /harness/proofs/blueprints` once. Repeating + initialization is idempotent only for the same contract. + +## Advance the frontier + +1. Have the evaluator request work from `POST /harness/proofs/blueprints/leases`. + Never fabricate, reuse, or transfer a lease. +2. Try a direct proof or refutation first. Run the frozen compiler against the + exact leased declaration and retain its transcript and failure feedback. +3. After direct failure, propose a decomposition. Compile a sketch proving the + parent while leaving placeholders only for the newly introduced child + declarations. Run the frozen reviewer for relevance, lower difficulty, and + plausibility. +4. Run `bun scripts/preflight.ts attempt preflight.json lease.json evidence.json`. + Inject the evaluator token only at the authenticated boundary and submit the + emitted payload to `POST /harness/proofs/blueprints/attempts`. +5. Continue with newly leased deepest-ready goals. When a branch blocks, add a + new alternative; never rewrite a proved goal, delete a rejected attempt, or + silently change a lemma signature. + +The script hashes private source, binaries, feedback, and transcripts and emits +only content identifiers. It derives placeholder declarations from child specs +and emits no bearer capability. + +## Finish with proof verification + +Blueprint `proved` means the search graph has a compiler-grounded route. It is +not a proof receipt. Build the exact root artifact and use +`$verify-formal-proof`; only a passing `formal-proof-v1` receipt may appear as +`proofReceiptID` in a passing final evaluation. diff --git a/backend/cli/skills/research/operate-proof-blueprint/agents/openai.yaml b/backend/cli/skills/research/operate-proof-blueprint/agents/openai.yaml new file mode 100644 index 00000000..7cb123cb --- /dev/null +++ b/backend/cli/skills/research/operate-proof-blueprint/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Operate Proof Blueprint" + short_description: "Run bounded verifier-grounded proof search" + default_prompt: "Use $operate-proof-blueprint to lease and advance a verifier-grounded AND/OR Lean proof search." diff --git a/backend/cli/skills/research/operate-proof-blueprint/references/protocol.md b/backend/cli/skills/research/operate-proof-blueprint/references/protocol.md new file mode 100644 index 00000000..d3450e5d --- /dev/null +++ b/backend/cli/skills/research/operate-proof-blueprint/references/protocol.md @@ -0,0 +1,43 @@ +# Proof blueprint protocol + +OpenScience models proof search as a monotone bipartite AND/OR DAG, following +the verifier-grounded decomposition pattern in +[LEAP](https://arxiv.org/abs/2606.03303). A goal is an OR node: a direct +compiler-accepted result or any closed decomposition may solve it. A +decomposition is an AND node: every child must be proved. The exact tuple +`(statementSHA256, declaration, module)` identifies a goal, so repeated lemmas +share work rather than spawning independent copies. + +The server derives status and scheduling: + +- The root exactly matches the frozen `formal-proof-v1` statement. +- A direct attempt precedes decomposition. +- An accepted sketch proves the parent assuming only the declared child goals; + its placeholder list must equal those child declarations exactly. +- The graph must remain reachable, acyclic, and within frozen node/depth limits. +- An open accepted branch suspends its parent while deepest-ready children run. +- A blocked branch returns its parent to the frontier for a bounded alternative. +- Every verifier/reviewer rejection consumes its lease and remains in the + contiguous attempt history. +- Stale, consumed, cross-session, and substituted-verifier leases fail closed. + +This preserves the useful failure-driven refinement in +[Goedel-Architect](https://arxiv.org/abs/2606.06468) without allowing a failed +helper to be silently repaired in place. Independent compiler calls and +statement-safety separation follow +[AlphaProof Nexus](https://arxiv.org/abs/2605.22763). Representation may evolve +only through a new frozen protocol; verifier authority remains fixed, as in +[Self-Modifying Lean Proof Agents](https://arxiv.org/abs/2607.17352). + +Endpoints are evaluator-capability protected: + +| Method | Path | Effect | +|---|---|---| +| `POST` | `/harness/proofs/blueprints` | Initialize the exact root | +| `POST` | `/harness/proofs/blueprints/status` | Read derived graph state | +| `POST` | `/harness/proofs/blueprints/leases` | Atomically lease ready goals | +| `POST` | `/harness/proofs/blueprints/attempts` | Consume a lease and retain an outcome | + +The reviewer cannot certify correctness. Blueprint closure cannot certify the +root. Final authority always remains `formal-proof-v1`, including its source, +axiom, fresh-recheck, and external-crosscheck policies. diff --git a/backend/cli/skills/research/operate-proof-blueprint/scripts/preflight.ts b/backend/cli/skills/research/operate-proof-blueprint/scripts/preflight.ts new file mode 100644 index 00000000..2b1e2a5d --- /dev/null +++ b/backend/cli/skills/research/operate-proof-blueprint/scripts/preflight.ts @@ -0,0 +1,228 @@ +#!/usr/bin/env bun + +import fs from "fs/promises" +import path from "path" + +const mode = process.argv[2] +const first = process.argv[3] +const second = process.argv[4] +const third = process.argv[5] +if (!mode || !first || (mode === "attempt" && (!second || !third))) { + throw new Error( + "Usage: bun scripts/preflight.ts protocol | attempt ", + ) +} + +const hash = (value: Uint8Array | string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const record = (value: unknown, label: string) => { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`) + return value as Record +} +const fields = (value: Record, label: string, allowed: string[]) => { + const unknown = Object.keys(value).filter((key) => !allowed.includes(key)) + if (unknown.length) throw new Error(`${label} has unknown fields: ${unknown.join(", ")}`) +} +const text = (value: unknown, label: string) => { + if (typeof value !== "string" || !value.trim()) throw new Error(`${label} must be a non-empty string`) + return value +} +const integer = (value: unknown, label: string, min: number, max = Number.MAX_SAFE_INTEGER) => { + if (!Number.isInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer from ${min} to ${max}`) + } + return value as number +} +const boolean = (value: unknown, label: string) => { + if (typeof value !== "boolean") throw new Error(`${label} must be a boolean`) + return value +} +const target = (root: string, value: unknown, label: string) => { + const input = text(value, label) + if (path.isAbsolute(input)) throw new Error(`${label} must be relative to its evidence directory`) + const file = path.resolve(root, input) + if (file !== root && !file.startsWith(`${root}${path.sep}`)) throw new Error(`${label} escapes its evidence directory`) + return file +} +const bytes = async (root: string, value: unknown, label: string) => { + const file = target(root, value, label) + const boundary = await fs.realpath(root) + const real = await fs.realpath(file) + if (real !== boundary && !real.startsWith(`${boundary}${path.sep}`)) { + throw new Error(`${label} resolves outside its evidence directory`) + } + return new Uint8Array(await Bun.file(real).arrayBuffer()) +} +const digest = async (root: string, value: unknown, label: string) => hash(await bytes(root, value, label)) + +async function protocol(file: string) { + const input = record(await Bun.file(path.resolve(file)).json(), "manifest") + fields(input, "manifest", [ + "graphSchemaPath", + "compilerPath", + "sketchValidatorPath", + "reviewerPath", + "reviewerPromptPath", + "maxNodes", + "maxDepth", + "maxParallel", + "maxAttemptsPerGoal", + "maxRefinementsPerGoal", + "leaseDurationMs", + ]) + const root = path.dirname(path.resolve(file)) + const blueprint = { + protocolVersion: "proof-blueprint-v1" as const, + graphSchemaSHA256: await digest(root, input.graphSchemaPath, "graphSchemaPath"), + compilerArtifactSHA256: await digest(root, input.compilerPath, "compilerPath"), + sketchValidatorArtifactSHA256: await digest(root, input.sketchValidatorPath, "sketchValidatorPath"), + reviewerArtifactSHA256: await digest(root, input.reviewerPath, "reviewerPath"), + reviewerPromptSHA256: await digest(root, input.reviewerPromptPath, "reviewerPromptPath"), + nodePolicy: "and-or-monotone-v1" as const, + failurePolicy: "preserve-and-refine" as const, + memoization: "goal-sha256" as const, + finalAuthority: "formal-proof-v1" as const, + directAttemptFirst: true as const, + verifiedSketchRequired: true as const, + completeFailureHistoryRequired: true as const, + maxNodes: integer(input.maxNodes, "maxNodes", 2, 512), + maxDepth: integer(input.maxDepth, "maxDepth", 1, 32), + maxParallel: integer(input.maxParallel, "maxParallel", 1, 32), + maxAttemptsPerGoal: integer(input.maxAttemptsPerGoal, "maxAttemptsPerGoal", 1, 16), + maxRefinementsPerGoal: integer(input.maxRefinementsPerGoal, "maxRefinementsPerGoal", 0, 16), + leaseDurationMs: integer(input.leaseDurationMs, "leaseDurationMs", 1_000, 3_600_000), + } + if (blueprint.maxParallel > blueprint.maxNodes) throw new Error("maxParallel cannot exceed maxNodes") + const artifacts = [ + blueprint.graphSchemaSHA256, + blueprint.compilerArtifactSHA256, + blueprint.sketchValidatorArtifactSHA256, + blueprint.reviewerArtifactSHA256, + blueprint.reviewerPromptSHA256, + ] + if (new Set(artifacts).size !== artifacts.length) { + throw new Error("schema, compiler, validator, reviewer, and reviewer prompt must be distinct artifacts") + } + return { blueprint } +} + +async function attempt(preflight: string, leaseFile: string, evidenceFile: string) { + const frozen = record(await Bun.file(path.resolve(preflight)).json(), "preflight") + fields(frozen, "preflight", ["blueprint"]) + const blueprint = record(frozen.blueprint, "preflight.blueprint") + const lease = record(await Bun.file(path.resolve(leaseFile)).json(), "lease") + fields(lease, "lease", ["id", "goalID", "revision", "ordinal", "status", "issuedAt", "expiresAt"]) + if (lease.status !== "open") throw new Error("lease must be open") + const input = record(await Bun.file(path.resolve(evidenceFile)).json(), "evidence") + fields(input, "evidence", [ + "sessionID", + "kind", + "artifactPath", + "claim", + "informalPlanPath", + "children", + "compiler", + "validator", + "review", + ]) + const root = path.dirname(path.resolve(evidenceFile)) + const compiler = record(input.compiler, "compiler") + fields(compiler, "compiler", [ + "artifactPath", + "statementMatched", + "exitCode", + "warnings", + "transcriptPath", + "feedbackPath", + "startedAt", + "endedAt", + ]) + const verification = { + compilerArtifactSHA256: await digest(root, compiler.artifactPath, "compiler.artifactPath"), + statementMatched: boolean(compiler.statementMatched, "compiler.statementMatched"), + exitCode: integer(compiler.exitCode, "compiler.exitCode", -2147483648, 2147483647), + warnings: integer(compiler.warnings, "compiler.warnings", 0), + transcriptSHA256: await digest(root, compiler.transcriptPath, "compiler.transcriptPath"), + feedbackSHA256: await digest(root, compiler.feedbackPath, "compiler.feedbackPath"), + startedAt: integer(compiler.startedAt, "compiler.startedAt", Number(lease.issuedAt), Number(lease.expiresAt)), + endedAt: integer(compiler.endedAt, "compiler.endedAt", Number(lease.issuedAt), Number(lease.expiresAt)), + } + if (verification.startedAt > verification.endedAt) throw new Error("compiler interval is reversed") + if (verification.compilerArtifactSHA256 !== text(blueprint.compilerArtifactSHA256, "blueprint.compilerArtifactSHA256")) { + throw new Error("compiler artifact does not match the frozen blueprint") + } + const base = { + sessionID: text(input.sessionID, "sessionID"), + leaseID: text(lease.id, "lease.id"), + artifactSHA256: await digest(root, input.artifactPath, "artifactPath"), + } + if (input.kind === "direct") { + if (!(["proof", "refutation", "failure"] as unknown[]).includes(input.claim)) { + throw new Error("claim must be proof, refutation, or failure") + } + return { submission: { ...base, kind: "direct" as const, claim: input.claim, verification } } + } + if (input.kind !== "decomposition") throw new Error("kind must be direct or decomposition") + if (!Array.isArray(input.children) || !input.children.length || input.children.length > 16) { + throw new Error("children must contain 1 to 16 goal specifications") + } + const children = await Promise.all( + input.children.map(async (value, index) => { + const item = record(value, `children[${index}]`) + fields(item, `children[${index}]`, ["statementPath", "declaration", "module"]) + return { + statementSHA256: await digest(root, item.statementPath, `children[${index}].statementPath`), + declaration: text(item.declaration, `children[${index}].declaration`), + module: text(item.module, `children[${index}].module`), + } + }), + ) + const validator = record(input.validator, "validator") + fields(validator, "validator", ["artifactPath", "transcriptPath"]) + const review = record(input.review, "review") + fields(review, "review", ["artifactPath", "promptPath", "relevant", "easier", "plausible", "transcriptPath"]) + const validatorArtifactSHA256 = await digest(root, validator.artifactPath, "validator.artifactPath") + const reviewerArtifactSHA256 = await digest(root, review.artifactPath, "review.artifactPath") + const promptSHA256 = await digest(root, review.promptPath, "review.promptPath") + if ( + validatorArtifactSHA256 !== + text(blueprint.sketchValidatorArtifactSHA256, "blueprint.sketchValidatorArtifactSHA256") || + reviewerArtifactSHA256 !== text(blueprint.reviewerArtifactSHA256, "blueprint.reviewerArtifactSHA256") || + promptSHA256 !== text(blueprint.reviewerPromptSHA256, "blueprint.reviewerPromptSHA256") + ) { + throw new Error("validator, reviewer, or reviewer prompt does not match the frozen blueprint") + } + return { + submission: { + ...base, + kind: "decomposition" as const, + informalPlanSHA256: await digest(root, input.informalPlanPath, "informalPlanPath"), + children, + verification: { + ...verification, + validatorArtifactSHA256, + placeholderDeclarations: children.map((item) => item.declaration).toSorted((a, b) => a.localeCompare(b)), + validatorTranscriptSHA256: await digest(root, validator.transcriptPath, "validator.transcriptPath"), + }, + review: { + reviewerArtifactSHA256, + promptSHA256, + relevant: boolean(review.relevant, "review.relevant"), + easier: boolean(review.easier, "review.easier"), + plausible: boolean(review.plausible, "review.plausible"), + transcriptSHA256: await digest(root, review.transcriptPath, "review.transcriptPath"), + }, + }, + frozen: { + compilerArtifactSHA256: text(blueprint.compilerArtifactSHA256, "blueprint.compilerArtifactSHA256"), + sketchValidatorArtifactSHA256: text( + blueprint.sketchValidatorArtifactSHA256, + "blueprint.sketchValidatorArtifactSHA256", + ), + reviewerArtifactSHA256: text(blueprint.reviewerArtifactSHA256, "blueprint.reviewerArtifactSHA256"), + reviewerPromptSHA256: text(blueprint.reviewerPromptSHA256, "blueprint.reviewerPromptSHA256"), + }, + } +} + +const result = mode === "protocol" ? await protocol(first) : await attempt(first, second!, third!) +process.stdout.write(`${JSON.stringify(result, null, 2)}\n`) diff --git a/backend/cli/skills/research/record-human-ai-autonomy/SKILL.md b/backend/cli/skills/research/record-human-ai-autonomy/SKILL.md new file mode 100644 index 00000000..f685a84c --- /dev/null +++ b/backend/cli/skills/research/record-human-ai-autonomy/SKILL.md @@ -0,0 +1,52 @@ +--- +name: record-human-ai-autonomy +description: Preflight and record a complete evaluator-owned human-AI interaction trace, bind it to an exact benchmark run or candidate artifact, and obtain a backend-derived essentially-autonomous, collaborative, or primarily-human receipt. Use when reporting autonomous scientific benchmark results, publishing Human-AI Interaction cards, or preventing hidden human hints from being mislabeled as agent-only performance. +--- + +# Record human-AI autonomy + +Capture the interaction stream outside the candidate sandbox. Keep raw prompts, +outputs, and evaluator capabilities private; expose only commitments and retained +evidence references to OpenScience. + +Read [references/protocol.md](references/protocol.md) before classifying events. + +## Freeze the protocol + +1. Prepare a manifest with `claimedLevel`, recorder name/version and local + executable path, local trace-schema and classification-policy paths, + `maxEvents`, and the disclosure policy. +2. Run `bun scripts/preflight.ts protocol protocol-manifest.json`. +3. Bind the emitted `protocol` as `autonomy` before the agent starts. Use + `intervention: human_reprompted` for collaborative or primarily-human claims. + +## Capture and submit + +1. Record the frozen benchmark problem and every benchmark, human, and agent + interaction in order. Retain the raw append-only log outside the agent's + readable workspace. +2. Classify each event as `problem`, `auxiliary`, `essential`, `core`, or + `unclear`. Never force an uncertain contribution into a passing class. +3. Reference local content and artifact files in a private trace manifest. Run + `bun scripts/preflight.ts submission preflight.json private-trace.json`. +4. Inject the evaluator capability only at the authenticated boundary and send + `output.submission` to `POST /harness/autonomy/receipts`. +5. Cite the returned `receiptID` as `autonomyReceiptID` in the final evaluation. + +The script hashes raw content, the final artifact, the recorder, and the whole +raw log; it never emits raw prompts or the evaluator token. + +## Integrity rules + +- The trace must be contiguous, monotonic, complete, and enclose candidate + creation. A candidate receipt must match its registered artifact SHA-256. +- Human `essential` or `core` input plus substantive agent input derives + `human_ai_collaboration`; substantive human input with only auxiliary AI + derives `primarily_human`; substantive AI without essential human input + derives `essentially_autonomous`. +- A problem statement or auxiliary exposition does not by itself downgrade an + autonomous result. Any `unclear` contribution makes the receipt inconclusive. +- One run or candidate gets one canonical receipt. An unfavorable trace cannot + be replaced after inspection. +- Hashes authenticate bytes and ordering, not semantic labels. A qualified + evaluator must retain the raw interaction evidence for audit. diff --git a/backend/cli/skills/research/record-human-ai-autonomy/agents/openai.yaml b/backend/cli/skills/research/record-human-ai-autonomy/agents/openai.yaml new file mode 100644 index 00000000..034a06d7 --- /dev/null +++ b/backend/cli/skills/research/record-human-ai-autonomy/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Record Human-AI Autonomy" + short_description: "Capture auditable human-AI contribution traces" + default_prompt: "Use $record-human-ai-autonomy to validate and preflight an evaluator-owned human-AI interaction trace." diff --git a/backend/cli/skills/research/record-human-ai-autonomy/references/protocol.md b/backend/cli/skills/research/record-human-ai-autonomy/references/protocol.md new file mode 100644 index 00000000..1bc538af --- /dev/null +++ b/backend/cli/skills/research/record-human-ai-autonomy/references/protocol.md @@ -0,0 +1,43 @@ +# Human-AI autonomy protocol + +The three output levels follow the contribution axis proposed in Google +DeepMind's Aletheia report, [Towards Autonomous Mathematics +Research](https://arxiv.org/abs/2602.10177): + +- `essentially_autonomous`: the agent generated the core scientific content + without essential human intervention. Posing the problem, exposition edits, + and genuinely minor corrections may remain auxiliary. +- `human_ai_collaboration`: human and agent contributions are both essential to + the scientific result. +- `primarily_human`: the core scientific content is human-generated and the AI + contribution is minor or auxiliary. + +## Actors + +- `benchmark`: frozen task delivery, evaluator feedback, or other protocol-owned + communication. This is not human scientific help. +- `human`: any person interacting with the agent or changing its artifact. +- `agent`: the evaluated system, including its declared worker topology. + +## Contribution classes + +- `problem`: only the original `problem_statement`; never an agent event. +- `auxiliary`: formatting, exposition, administrative clarification, or a minor + correction that does not supply a scientific step. +- `essential`: a contribution without which a material strategy, inference, + experiment, or validation would be absent. +- `core`: authorship of the central scientific construction or result. +- `unclear`: evidence is insufficient to distinguish the above. This must + remain inconclusive. + +Use the event kinds `problem_statement`, `clarification`, +`resource_provision`, `strategy`, `technical_correction`, `artifact_edit`, +`candidate_selection`, `evaluation_feedback`, `exposition`, and `other`. +The kind describes the interaction; the contribution class determines the +derived level. + +Every event needs a retained evidence reference. The private raw log must cover +the entire interval, including rejected suggestions, post-generation edits, +human candidate selection, and feedback that caused a retry. Hash commitments +do not prove that an omitted event never happened; completeness depends on the +evaluator-controlled recorder and operating boundary. diff --git a/backend/cli/skills/research/record-human-ai-autonomy/scripts/preflight.ts b/backend/cli/skills/research/record-human-ai-autonomy/scripts/preflight.ts new file mode 100644 index 00000000..f493f69a --- /dev/null +++ b/backend/cli/skills/research/record-human-ai-autonomy/scripts/preflight.ts @@ -0,0 +1,235 @@ +#!/usr/bin/env bun + +import path from "path" + +const mode = process.argv[2] +const first = process.argv[3] +const second = process.argv[4] +if (!mode || !first || (mode === "submission" && !second)) { + throw new Error( + "Usage: bun scripts/preflight.ts protocol | submission ", + ) +} + +const levels = ["essentially_autonomous", "human_ai_collaboration", "primarily_human"] as const +const actors = ["benchmark", "human", "agent"] as const +const contributions = ["problem", "auxiliary", "essential", "core", "unclear"] as const +const kinds = [ + "problem_statement", + "clarification", + "resource_provision", + "strategy", + "technical_correction", + "artifact_edit", + "candidate_selection", + "evaluation_feedback", + "exposition", + "other", +] as const +const hash = (value: Uint8Array | string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const record = (value: unknown, label: string) => { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`) + return value as Record +} +const fields = (value: Record, label: string, allowed: string[]) => { + const unknown = Object.keys(value).filter((key) => !allowed.includes(key)) + if (unknown.length) throw new Error(`${label} has unknown fields: ${unknown.join(", ")}`) +} +const text = (value: unknown, label: string) => { + if (typeof value !== "string" || !value.trim()) throw new Error(`${label} must be a non-empty string`) + return value +} +const integer = (value: unknown, label: string, min = 1, max = Number.MAX_SAFE_INTEGER) => { + if (!Number.isInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer from ${min} to ${max}`) + } + return value as number +} +const choice = (value: unknown, label: string, values: T) => { + const item = text(value, label) + if (!values.includes(item as T[number])) throw new Error(`${label} must be one of ${values.join(", ")}`) + return item as T[number] +} +const target = (root: string, value: unknown, label: string) => path.resolve(root, text(value, label)) +const bytes = async (root: string, value: unknown, label: string) => { + const file = target(root, value, label) + const source = Bun.file(file) + if (!(await source.exists())) throw new Error(`${label} does not exist: ${file}`) + return new Uint8Array(await source.arrayBuffer()) +} +const digest = async (root: string, value: unknown, label: string) => hash(await bytes(root, value, label)) + +async function protocol(file: string) { + const input = record(await Bun.file(path.resolve(file)).json(), "manifest") + fields(input, "manifest", [ + "claimedLevel", + "recorder", + "traceSchemaPath", + "classificationPolicyPath", + "maxEvents", + "disclosure", + ]) + const recorder = record(input.recorder, "recorder") + fields(recorder, "recorder", ["name", "version", "artifactPath"]) + const root = path.dirname(path.resolve(file)) + return { + protocolVersion: "human-ai-autonomy-v1" as const, + claimedLevel: choice(input.claimedLevel, "claimedLevel", levels), + recorder: { + name: text(recorder.name, "recorder.name"), + version: text(recorder.version, "recorder.version"), + artifactSHA256: await digest(root, recorder.artifactPath, "recorder.artifactPath"), + source: "evaluator_runtime" as const, + }, + traceSchemaSHA256: await digest(root, input.traceSchemaPath, "traceSchemaPath"), + classificationPolicySHA256: await digest(root, input.classificationPolicyPath, "classificationPolicyPath"), + maxEvents: integer(input.maxEvents, "maxEvents", 2, 10_000), + rawRetention: "required" as const, + disclosure: choice(input.disclosure, "disclosure", ["evaluator_retained", "public_essential_after_release"]), + completeTraceRequired: true as const, + uncertaintyPolicy: "inconclusive" as const, + } +} + +async function submission(preflight: string, file: string) { + const frozen = record(await Bun.file(path.resolve(preflight)).json(), "preflight") + fields(frozen, "preflight", ["protocol"]) + const protocol = record(frozen.protocol, "preflight.protocol") + const input = record(await Bun.file(path.resolve(file)).json(), "trace") + fields(input, "trace", [ + "sessionID", + "subject", + "artifactPath", + "rawLogPath", + "startedAt", + "endedAt", + "events", + ]) + const root = path.dirname(path.resolve(file)) + const subject = record(input.subject, "subject") + fields(subject, "subject", ["type", "id"]) + const type = choice(subject.type, "subject.type", ["run", "candidate"]) + const startedAt = integer(input.startedAt, "startedAt") + const endedAt = integer(input.endedAt, "endedAt") + if (endedAt < startedAt) throw new Error("endedAt must not precede startedAt") + if (!Array.isArray(input.events) || input.events.length < 2) throw new Error("events must contain at least two entries") + const source = input.events + const maxEvents = integer(protocol.maxEvents, "preflight.protocol.maxEvents", 2, 10_000) + if (source.length > maxEvents) throw new Error(`events exceed the frozen maxEvents ${maxEvents}`) + const events = await Promise.all( + source.map(async (value, index) => { + const event = record(value, `events[${index}]`) + fields(event, `events[${index}]`, [ + "sequence", + "at", + "actor", + "kind", + "contribution", + "contentPath", + "artifactBeforePath", + "artifactAfterPath", + "evidence", + ]) + const sequence = integer(event.sequence, `events[${index}].sequence`) + if (sequence !== index + 1) throw new Error("event sequence must be contiguous from one") + const at = integer(event.at, `events[${index}].at`) + if (at < startedAt || at > endedAt) throw new Error(`events[${index}].at falls outside the trace interval`) + if (index && at < integer(record(source[index - 1], `events[${index - 1}]`).at, `events[${index - 1}].at`)) { + throw new Error("event time must be monotonic") + } + const actor = choice(event.actor, `events[${index}].actor`, actors) + const kind = choice(event.kind, `events[${index}].kind`, kinds) + const contribution = choice(event.contribution, `events[${index}].contribution`, contributions) + if (kind === "problem_statement" && contribution !== "problem") { + throw new Error("problem_statement must use the problem contribution") + } + if ((!index && kind !== "problem_statement") || (index > 0 && kind === "problem_statement")) { + throw new Error("trace requires exactly one initial problem_statement") + } + if (contribution === "problem" && kind !== "problem_statement") { + throw new Error("only problem_statement may use the problem contribution") + } + if (contribution === "problem" && actor === "agent") throw new Error("agent cannot pose the frozen problem") + if (kind === "exposition" && !["auxiliary", "unclear"].includes(contribution)) { + throw new Error("exposition cannot be essential or core") + } + if (!Array.isArray(event.evidence) || !event.evidence.length || event.evidence.length > 32) { + throw new Error(`events[${index}].evidence must contain one to 32 references`) + } + const evidence = event.evidence.map((item, offset) => text(item, `events[${index}].evidence[${offset}]`)) + return { + sequence, + at, + actor, + kind, + contribution, + contentSHA256: await digest(root, event.contentPath, `events[${index}].contentPath`), + ...(event.artifactBeforePath + ? { artifactBeforeSHA256: await digest(root, event.artifactBeforePath, `events[${index}].artifactBeforePath`) } + : {}), + ...(event.artifactAfterPath + ? { artifactAfterSHA256: await digest(root, event.artifactAfterPath, `events[${index}].artifactAfterPath`) } + : {}), + evidence, + } + }), + ) + const artifactSHA256 = await digest(root, input.artifactPath, "artifactPath") + const transitions = events.filter((event) => event.artifactAfterSHA256) + if ( + transitions.some( + (event, index) => + Boolean(index) && event.artifactBeforeSHA256 !== transitions[index - 1]!.artifactAfterSHA256, + ) + ) { + throw new Error("artifact transitions must form one continuous chain") + } + if (transitions.at(-1)?.artifactAfterSHA256 !== artifactSHA256) { + throw new Error("the last artifact transition must bind artifactPath") + } + const human = events.filter( + (event) => event.actor === "human" && (event.contribution === "essential" || event.contribution === "core"), + ).length + const agent = events.filter( + (event) => event.actor === "agent" && (event.contribution === "essential" || event.contribution === "core"), + ).length + const unclear = events.filter((event) => event.contribution === "unclear").length + const derivedLevel = unclear + ? undefined + : human && agent + ? "human_ai_collaboration" + : human + ? "primarily_human" + : agent + ? "essentially_autonomous" + : undefined + return { + submission: { + sessionID: text(input.sessionID, "sessionID"), + subject: { type, id: text(subject.id, "subject.id") }, + artifactSHA256, + trace: { + owner: "evaluator_runtime" as const, + complete: true as const, + recorderArtifactSHA256: text(protocol.recorder && record(protocol.recorder, "protocol.recorder").artifactSHA256, "protocol.recorder.artifactSHA256"), + schemaSHA256: text(protocol.traceSchemaSHA256, "protocol.traceSchemaSHA256"), + classificationPolicySHA256: text(protocol.classificationPolicySHA256, "protocol.classificationPolicySHA256"), + rawLogSHA256: await digest(root, input.rawLogPath, "rawLogPath"), + startedAt, + endedAt, + events, + }, + }, + preview: { + claimedLevel: text(protocol.claimedLevel, "protocol.claimedLevel"), + derivedLevel, + unclearEvents: unclear, + humanSubstantiveEvents: human, + agentSubstantiveEvents: agent, + }, + } +} + +if (mode === "protocol") console.log(JSON.stringify({ protocol: await protocol(first) }, null, 2)) +else if (mode === "submission") console.log(JSON.stringify(await submission(first, second!), null, 2)) +else throw new Error(`Unknown mode ${mode}`) diff --git a/backend/cli/skills/research/run-clean-room-synthesis/SKILL.md b/backend/cli/skills/research/run-clean-room-synthesis/SKILL.md new file mode 100644 index 00000000..587ec879 --- /dev/null +++ b/backend/cli/skills/research/run-clean-room-synthesis/SKILL.md @@ -0,0 +1,66 @@ +--- +name: run-clean-room-synthesis +description: Preflight and record evaluator-owned clean-room scientific conclusion synthesis with a salted hidden atomic-fact manifest, complete retrieval trace, independent decomposition and precision/recall judges, and backend-derived factual F1. Use for long-form systematic-review synthesis where answer-key retrieval and judge failures must not be scored as genuine reasoning. +--- + +# Run clean-room scientific synthesis + +Keep the hidden reference, atomic facts, salt, evaluator capability, judge prompts, +and unfiltered tool outputs in the external evaluator process. The candidate sees +the public question and only tool results the clean-room filter allows. + +## Preflight + +1. Prepare a private manifest with the public `query`, `referenceTextPath`, one + to 2,048 `{id, text}` reference facts, and `factSaltPath` pointing to at + least 32 private random bytes. Use opaque fact IDs. +2. Add the ISO publication `cutoff`, the canonical subset of `google_search`, + `paper_search`, and `web_browse`, the maximum tool-event budget, and local + `traceSchemaPath` and `filterPolicyPath` files. +3. Add decomposer, precision-judge, and recall-judge identities. Each identity + names a version plus local `promptPath` and `configPath`; all three + prompt commitments must differ, while every configuration is also frozen. +4. Set the minimum generated facts and predeclared factual precision, recall, + and F1 thresholds. Run + `bun scripts/preflight.ts private-manifest.json > preflight.json`. +5. Bind `preflight.json.protocol` as `synthesis`, set the benchmark metric to + maximized `factual_f1`, and set `contamination.publicDataCutoff` to the same + cutoff. Also bind an evaluator audit that tests `wrong_answer`, + `unsupported_claim`, and `data_leakage` faults. + +The output exposes only salted reference commitments and content hashes. Keep +the salt and source material private so predictable clinical facts cannot be +recovered by dictionary attack. + +## Execute + +1. Record every result item returned by every declared retrieval tool in + contiguous order. Hash the request, response, and source. Supply the source + date and evaluator findings for forbidden domains and reference-title + matches. Missing dates, post-cutoff sources, repeated outputs, forbidden + domains, and reference-title matches must be blocked before reaching the + candidate. +2. Hash the final conclusion, decompose it with the frozen decomposer, and + retain a sorted manifest of generated atomic facts. +3. Label every generated fact `supported`, `contradicted`, `unsupported`, or + `judge_error`. Label every frozen reference fact `covered`, `missed`, or + `judge_error`. Judge/provider/format failures are errors, never ordinary + unsupported or missed facts. +4. Submit `POST /harness/syntheses/receipts` with the passing evaluator-audit + receipt. OpenScience recomputes every clean-room decision, the trace hash, + factual precision `(supported / total) * (1 - contradicted / total)`, recall, + and harmonic-mean F1. +5. Cite the canonical receipt in `synthesisReceiptID`. A passing final score + must exactly equal the receipt's backend-derived F1. + +## Integrity rules + +- One run or candidate can have only one canonical synthesis receipt. Failed + or unfavorable receipts cannot be replaced after inspection. +- Any decomposition or fact-judge error makes the receipt `inconclusive`. +- Authentication and commitments prevent drift and forgery; semantic truth is + established by the separately qualified evaluator, not by hashes alone. +- A clean-room receipt measures synthesis against the frozen reference. It is + not evidence that a clinical recommendation is safe for deployment. +- If the official benchmark lacks an external-candidate grading entrypoint, + preserve that upstream blocker instead of claiming a native run succeeded. diff --git a/backend/cli/skills/research/run-clean-room-synthesis/agents/openai.yaml b/backend/cli/skills/research/run-clean-room-synthesis/agents/openai.yaml new file mode 100644 index 00000000..dfae1d5a --- /dev/null +++ b/backend/cli/skills/research/run-clean-room-synthesis/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Clean-Room Synthesis" + short_description: "Measure atomic factuality without answer-key leakage" + default_prompt: "Use $run-clean-room-synthesis to preflight and record a clean-room scientific conclusion synthesis evaluation." diff --git a/backend/cli/skills/research/run-clean-room-synthesis/scripts/preflight.ts b/backend/cli/skills/research/run-clean-room-synthesis/scripts/preflight.ts new file mode 100755 index 00000000..019079b1 --- /dev/null +++ b/backend/cli/skills/research/run-clean-room-synthesis/scripts/preflight.ts @@ -0,0 +1,137 @@ +#!/usr/bin/env bun + +import path from "path" + +const file = process.argv[2] +if (!file) throw new Error("Usage: bun scripts/preflight.ts ") + +const Hash = /^[a-f0-9]{64}$/ +const ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/ +const tools = ["google_search", "paper_search", "web_browse"] as const +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const record = (value: unknown, label: string) => { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`) + return value as Record +} +const fields = (value: Record, label: string, allowed: string[]) => { + const unknown = Object.keys(value).filter((key) => !allowed.includes(key)) + if (unknown.length) throw new Error(`${label} has unknown fields: ${unknown.join(", ")}`) +} +const text = (value: unknown, label: string) => { + if (typeof value !== "string" || !value.trim()) throw new Error(`${label} must be a non-empty string`) + return value +} +const integer = (value: unknown, label: string, min: number, max: number) => { + if (!Number.isInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer from ${min} to ${max}`) + } + return value as number +} +const number = (value: unknown, label: string, min: number, max: number) => { + if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) { + throw new Error(`${label} must be a finite number from ${min} to ${max}`) + } + return value +} +const target = (value: unknown, label: string) => + path.resolve(path.dirname(path.resolve(file)), text(value, label)) +const bytes = async (value: unknown, label: string) => { + const filename = target(value, label) + const source = Bun.file(filename) + if (!(await source.exists())) throw new Error(`${label} does not exist: ${filename}`) + return source.text() +} +const identity = async (value: unknown, label: string) => { + const input = record(value, label) + fields(input, label, ["name", "version", "promptPath", "configPath"]) + return { + name: text(input.name, `${label}.name`), + version: text(input.version, `${label}.version`), + promptSHA256: hash(await bytes(input.promptPath, `${label}.promptPath`)), + configSHA256: hash(await bytes(input.configPath, `${label}.configPath`)), + } +} + +const input = record(await Bun.file(path.resolve(file)).json(), "manifest") +fields(input, "manifest", [ + "query", + "referenceTextPath", + "referenceFacts", + "factSaltPath", + "cutoff", + "tools", + "traceSchemaPath", + "filterPolicyPath", + "maxToolEvents", + "decomposer", + "judges", + "minGeneratedFacts", + "minPrecision", + "minRecall", + "minF1", +]) +const query = text(input.query, "query") +const reference = await bytes(input.referenceTextPath, "referenceTextPath") +const salt = await bytes(input.factSaltPath, "factSaltPath") +if (new TextEncoder().encode(salt).length < 32) throw new Error("factSaltPath must contain at least 32 bytes") +if (!Array.isArray(input.referenceFacts) || !input.referenceFacts.length || input.referenceFacts.length > 2_048) { + throw new Error("referenceFacts must contain one to 2048 entries") +} +const facts = input.referenceFacts + .map((value, index) => { + const fact = record(value, `referenceFacts[${index}]`) + fields(fact, `referenceFacts[${index}]`, ["id", "text"]) + const id = text(fact.id, `referenceFacts[${index}].id`) + if (!ID.test(id)) throw new Error(`referenceFacts[${index}].id must be an opaque safe identifier`) + const content = text(fact.text, `referenceFacts[${index}].text`) + return { id, commitment: hash(JSON.stringify({ kind: "reference_fact", id, content, salt })) } + }) + .toSorted((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)) +if (new Set(facts.map((fact) => fact.id)).size !== facts.length) throw new Error("reference fact IDs must be unique") +if (new Set(facts.map((fact) => fact.commitment)).size !== facts.length) { + throw new Error("reference facts must produce unique commitments") +} +const cutoff = text(input.cutoff, "cutoff") +const date = /^\d{4}-\d{2}-\d{2}$/.test(cutoff) ? new Date(`${cutoff}T00:00:00Z`) : undefined +if (!date || Number.isNaN(date.valueOf()) || date.toISOString().slice(0, 10) !== cutoff) { + throw new Error("cutoff must be an ISO calendar date") +} +if (!Array.isArray(input.tools) || !input.tools.length || input.tools.length > tools.length) { + throw new Error("tools must contain one to three entries") +} +const selected = input.tools.map((value, index) => text(value, `tools[${index}]`)) +if (new Set(selected).size !== selected.length || selected.some((value) => !tools.includes(value as (typeof tools)[number]))) { + throw new Error("tools must be unique supported clean-room tools") +} +const ordered = selected.toSorted((left, right) => tools.indexOf(left as (typeof tools)[number]) - tools.indexOf(right as (typeof tools)[number])) +if (JSON.stringify(selected) !== JSON.stringify(ordered)) throw new Error("tools must use canonical order") +const judges = record(input.judges, "judges") +fields(judges, "judges", ["precision", "recall"]) +const decomposer = await identity(input.decomposer, "decomposer") +const precision = await identity(judges.precision, "judges.precision") +const recall = await identity(judges.recall, "judges.recall") +const actors = [decomposer, precision, recall].map((item) => item.promptSHA256) +if (new Set(actors).size !== actors.length) { + throw new Error("decomposer, precision judge, and recall judge must use distinct prompt commitments") +} +const protocol = { + protocolVersion: "scientific-synthesis-v1" as const, + querySHA256: hash(query), + referenceSHA256: hash(JSON.stringify({ kind: "reference_text", reference, salt })), + referenceFactsSHA256: hash(JSON.stringify(facts)), + referenceFactCount: facts.length, + cutoff, + tools: selected, + traceSchemaSHA256: hash(await bytes(input.traceSchemaPath, "traceSchemaPath")), + filterPolicySHA256: hash(await bytes(input.filterPolicyPath, "filterPolicyPath")), + maxToolEvents: integer(input.maxToolEvents, "maxToolEvents", 1, 10_000), + decomposer, + judges: { precision, recall }, + minGeneratedFacts: integer(input.minGeneratedFacts, "minGeneratedFacts", 1, 512), + minPrecision: number(input.minPrecision, "minPrecision", 0, 1), + minRecall: number(input.minRecall, "minRecall", 0, 1), + minF1: number(input.minF1, "minF1", 0, 1), + cleanRoomRequired: true as const, + judgeFailurePolicy: "inconclusive" as const, +} +console.log(JSON.stringify({ protocol, referenceManifest: facts }, null, 2)) diff --git a/backend/cli/skills/research/run-proactive-evaluation/SKILL.md b/backend/cli/skills/research/run-proactive-evaluation/SKILL.md new file mode 100644 index 00000000..51f21235 --- /dev/null +++ b/backend/cli/skills/research/run-proactive-evaluation/SKILL.md @@ -0,0 +1,51 @@ +--- +name: run-proactive-evaluation +description: Build, execute, calibrate, and seal OpenScience proactive-audit-v2 evaluations with a backend-derived score-history Gaussian-process prior. Use when costly held-out population evaluation should use ProEval-style Bayesian-quadrature selection, when negative transfer must fail closed, or when a passing final evaluation requires a content-addressed active-audit receipt without exposing hidden cases. +--- + +# Run Proactive Evaluation + +Keep source scores, hidden cases, and the evaluator capability outside the candidate-producing agent. Use the script to commit the exact transfer pool before binding a run. + +## Prepare the frozen pool + +Create evaluator-private JSONL with `id`, `hidden`, `sourceLosses`, `stratum`, and optional `weight`. Every loss vector must follow the same source-model order. Create a token-free protocol JSON containing: + +- unique `sourceModels` (at least three); +- `selectionSHA256` for the source-profile selection artifact; +- `selectionMethod`: `pca-gmm-profile-v1` or `holdout-embedding-gmm-v1`; +- `calibrationSamples`; and +- `maxCalibrationMAE`. + +Run inside the secret-owning evaluator environment: + +```bash +bun scripts/preflight.ts \ + --input private-probes.jsonl \ + --protocol transfer-protocol.json \ + --out proactive-audit.json +``` + +The output contains only commitments, source losses, strata, weights, the exact `poolSHA256`, and a backend-compatible `sourceManifestSHA256` derived from the ordered source IDs and score matrix. It never contains hidden bytes or a capability. Do not edit the generated probes or transfer object. + +## Bind and execute + +1. Put the generated `transfer` object under `audit.transfer` in the immutable run contract. Set `promotionRequired: true` only for `performance` or `hybrid` mode. +2. Initialize `POST /harness/audits` with the generated `probes`, the exact run/candidate artifact hash, and the in-memory evaluator capability. +3. Call `/selection` once per round. Resolve the returned commitment to the private hidden case. Calibration selections must report `phase: calibration`. +4. Evaluate the frozen subject and submit one threshold-consistent loss, failure label, and evidence record to `/observations`. +5. Continue after `transfer.status: accepted`. If it becomes `rejected`, preserve `abstain: true`; remaining `fallback` selections are diagnostic and cannot qualify promotion. +6. Stop only at persisted terminal state. Seal it with `POST /harness/audits/:auditID/receipt`. +7. Add the returned `receiptID` as `auditReceiptID` on the ordinary authenticated final evaluation. Never use the audit estimate as the official benchmark score. + +## Integrity rules + +- Never provide `features` or `priorLoss` in v2. OpenScience derives the empirical mean and covariance features from `sourceLosses`. +- Never change the pool, source order, manifests, selection artifact, thresholds, subject artifact, or failure definition after binding. +- Never mix generated, synthesized, adaptively authored, or manually cherry-picked cases into the committed population pool. Run them as a separate failure-discovery stream. +- Treat a missing, corrupt, mismatched, future, abstaining, or unqualified receipt as a failed promotion gate. +- Treat byte-identical selection, observation, and sealing retries only as transport recovery. + +## Stop conditions + +Stop before evaluation on pool-digest mismatch, source dimension drift, fewer than three source models, hidden-byte exposure, capability exposure, unresolvable commitments, or subject substitution. Stop promotion when calibration rejects transfer, the terminal estimate abstains, or the receipt does not match the exact contract and subject. diff --git a/backend/cli/skills/research/run-proactive-evaluation/agents/openai.yaml b/backend/cli/skills/research/run-proactive-evaluation/agents/openai.yaml new file mode 100644 index 00000000..9a01f234 --- /dev/null +++ b/backend/cli/skills/research/run-proactive-evaluation/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Proactive Evaluation" + short_description: "Calibrate and seal transfer-qualified audits" + default_prompt: "Use $run-proactive-evaluation to preflight and run a transfer-qualified active audit." diff --git a/backend/cli/skills/research/run-proactive-evaluation/scripts/preflight.ts b/backend/cli/skills/research/run-proactive-evaluation/scripts/preflight.ts new file mode 100755 index 00000000..659a36da --- /dev/null +++ b/backend/cli/skills/research/run-proactive-evaluation/scripts/preflight.ts @@ -0,0 +1,104 @@ +#!/usr/bin/env bun + +const hash = /^[a-f0-9]{64}$/ +const digest = (value: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex") +const require = (condition: boolean, message: string) => { + if (!condition) throw new Error(message) +} + +function flag(name: string) { + const index = Bun.argv.indexOf(name) + require(index >= 0 && Boolean(Bun.argv[index + 1]), `Missing ${name}`) + return Bun.argv[index + 1]! +} + +function record(value: unknown, name: string) { + require(Boolean(value) && typeof value === "object" && !Array.isArray(value), `${name} must be an object`) + return value as Record +} + +function exact(value: Record, name: string, allowed: string[], required: string[]) { + const extra = Object.keys(value).filter((key) => !allowed.includes(key)) + const missing = required.filter((key) => !(key in value)) + require(!extra.length, `${name} has unknown fields: ${extra.join(", ")}`) + require(!missing.length, `${name} is missing fields: ${missing.join(", ")}`) +} + +function number(value: unknown, name: string, minimum: number, maximum: number) { + require(typeof value === "number" && Number.isFinite(value) && value >= minimum && value <= maximum, `${name} is invalid`) + return value as number +} + +const input = flag("--input") +const protocolFile = flag("--protocol") +const out = flag("--out") +const protocol = record(JSON.parse(await Bun.file(protocolFile).text()), "protocol") +const fields = [ + "sourceModels", + "selectionSHA256", + "selectionMethod", + "calibrationSamples", + "maxCalibrationMAE", +] +exact(protocol, "protocol", fields, fields) +require(Array.isArray(protocol.sourceModels), "sourceModels must be an array") +const models = protocol.sourceModels as unknown[] +require(models.length >= 3 && models.length <= 64, "sourceModels must contain 3 to 64 entries") +require(models.every((item) => typeof item === "string" && item.length > 0 && item.length <= 240), "sourceModels are invalid") +require(new Set(models).size === models.length, "sourceModels must be unique") +require(hash.test(String(protocol.selectionSHA256)), "selectionSHA256 is invalid") +require( + protocol.selectionMethod === "pca-gmm-profile-v1" || protocol.selectionMethod === "holdout-embedding-gmm-v1", + "selectionMethod is invalid", +) +const calibration = number(protocol.calibrationSamples, "calibrationSamples", 2, 64) +require(Number.isInteger(calibration), "calibrationSamples must be an integer") +const threshold = number(protocol.maxCalibrationMAE, "maxCalibrationMAE", Number.MIN_VALUE, 1) +const lines = (await Bun.file(input).text()) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) +const privateRows = lines.map((line, index) => record(JSON.parse(line), `probe ${index + 1}`)) +require(privateRows.length >= 2 && privateRows.length <= 2_000, "probe pool must contain 2 to 2,000 records") +const probes = privateRows + .map((row, index) => { + exact(row, `probe ${index + 1}`, ["id", "hidden", "sourceLosses", "stratum", "weight"], [ + "id", + "hidden", + "sourceLosses", + "stratum", + ]) + require(typeof row.id === "string" && row.id.length > 0 && row.id.length <= 240, `probe ${index + 1} id is invalid`) + require(typeof row.stratum === "string" && row.stratum.length > 0 && row.stratum.length <= 120, `probe ${index + 1} stratum is invalid`) + require(Array.isArray(row.sourceLosses) && row.sourceLosses.length === models.length, `probe ${index + 1} source dimension drifted`) + const losses = (row.sourceLosses as unknown[]).map((loss, offset) => + number(loss, `probe ${index + 1} sourceLosses[${offset}]`, 0, 1), + ) + const weight = row.weight === undefined ? 1 : number(row.weight, `probe ${index + 1} weight`, Number.MIN_VALUE, 1_000) + return { + id: row.id as string, + commitment: digest(row.hidden), + sourceLosses: losses, + stratum: row.stratum as string, + weight, + } + }) + .toSorted((left, right) => left.id.localeCompare(right.id)) +require(new Set(probes.map((probe) => probe.id)).size === probes.length, "probe ids must be unique") +require(new Set(probes.map((probe) => probe.commitment)).size === probes.length, "hidden probe commitments must be unique") +require(calibration <= probes.length, "calibrationSamples exceed the probe pool") +const transfer = { + protocolVersion: "score-history-prior-v1", + poolSHA256: digest(probes), + sourceManifestSHA256: digest({ + sourceModels: models, + scores: probes.map((probe) => ({ id: probe.id, sourceLosses: probe.sourceLosses })), + }), + selectionSHA256: protocol.selectionSHA256, + selectionMethod: protocol.selectionMethod, + sourceModels: models, + calibrationSamples: calibration, + maxCalibrationMAE: threshold, +} +await Bun.write(out, `${JSON.stringify({ schemaVersion: 1, transfer, probes }, null, 2)}\n`) +console.log(JSON.stringify({ valid: true, tokenFree: true, probes: probes.length, poolSHA256: transfer.poolSHA256, out })) diff --git a/backend/cli/skills/research/run-replicated-evaluation/SKILL.md b/backend/cli/skills/research/run-replicated-evaluation/SKILL.md new file mode 100644 index 00000000..82c841b2 --- /dev/null +++ b/backend/cli/skills/research/run-replicated-evaluation/SKILL.md @@ -0,0 +1,24 @@ +--- +name: run-replicated-evaluation +description: Prepare, execute, and preflight an evaluator-owned OpenScience replicated-evaluation receipt over a frozen stratum-by-independent-cluster grid. Use when a bound replicated-evaluation-v1 contract requires mean, median, IQM, or pass-rate aggregation with a confidence bound before a run or candidate may receive its final benchmark evaluation. +--- + +# Run Replicated Evaluation + +Operate this skill in the evaluator process. Keep the evaluator capability outside agent-visible artifacts. + +1. Read the immutable harness contract and locate `replication`. Stop if it is absent. +2. Read [references/protocol.md](references/protocol.md) before constructing a payload. +3. Materialize every declared `stratum × cluster` unit exactly once. Treat `clusterKind` as the highest independent sampling unit; never turn repeated measurements, checkpoints, timesteps, or metrics from one cluster into extra clusters. +4. Execute all units under the exact `replication.environmentSHA256` commitment. Preserve failed and inconclusive units instead of replacing or omitting them. +5. Record the matching stratum and cluster commitment hashes plus one immutable output hash, environment hash, timestamp, and evidence set per unit. Numeric estimators require a score only for passed units. `pass_rate` uses statuses and accepts no submitted numeric score. +6. Run: + + ```bash + python3 scripts/preflight.py contract.json observations.json + ``` + +7. Submit the complete body to `POST /harness/replications/receipts` with the bound evaluator capability. Treat the preflight as advisory; only the backend receipt is authoritative. Exact retries return the frozen receipt; changed retries for the same subject are forbidden. +8. Reference the returned `receiptID` in the final evaluation. Set the final score and bound metric to `receipt.statistics.estimate`. A passing evaluation requires `receipt.status == "passed"`. + +Do not select a favorable subset, retry only failed units, change strata or clusters after seeing results, report the best replicate, or replace the conservative bound with the point estimate. A failed or inconclusive receipt remains durable evidence. diff --git a/backend/cli/skills/research/run-replicated-evaluation/agents/openai.yaml b/backend/cli/skills/research/run-replicated-evaluation/agents/openai.yaml new file mode 100644 index 00000000..84eb5bfb --- /dev/null +++ b/backend/cli/skills/research/run-replicated-evaluation/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Replicated Evaluation" + short_description: "Freeze units and promote conservative bounds" + default_prompt: "Use $run-replicated-evaluation to prepare and validate a complete uncertainty-aware benchmark receipt." diff --git a/backend/cli/skills/research/run-replicated-evaluation/references/protocol.md b/backend/cli/skills/research/run-replicated-evaluation/references/protocol.md new file mode 100644 index 00000000..6ec8676d --- /dev/null +++ b/backend/cli/skills/research/run-replicated-evaluation/references/protocol.md @@ -0,0 +1,64 @@ +# Replicated evaluation protocol + +## Frozen design + +`replicated-evaluation-v1` defines an exact crossed design: + +- `sampling.strata`: benchmark tasks, folds, sites, datasets, or other declared conditions. +- `sampling.clusters`: independent seeds, trials, reproductions, operators, or laboratories. +- admissible units: the complete Cartesian product of the two axes. +- `commitmentSHA256`: evaluator-owned commitments to each frozen axis configuration. +- `environmentSHA256`: evaluator-owned commitment to the exact runtime, dependencies, tools, and simulator configuration used by every unit. + +Use one stratum when only independent repetitions are needed. Use several strata when performance must aggregate across tasks or conditions. Declare at least five genuinely independent clusters for numeric bootstrap estimators and at least three for pass-rate Wilson intervals. If two observations share the highest-level source of randomness or execution, they belong to one cluster. + +## Aggregation + +For `mean`, `median`, and `iqm`, every unit must pass and carry a finite score. The backend draws strata with replacement, then draws clusters independently with replacement inside each selected stratum using the frozen deterministic seed. It recomputes the estimator for each draw and takes the 2.5% and 97.5% percentile endpoints. IQM is the mean of the empirical quantile function between 25% and 75%. Use 50,000 resamples for publication or leaderboard reports when runtime permits; 1,000 is the contract minimum. + +For `pass_rate`, declare exactly one stratum so each cluster is one independent Bernoulli outcome. Observations carry only `passed`, `failed`, or `inconclusive`. The backend computes the pass fraction and a 95% Wilson score interval. An inconclusive unit makes the receipt inconclusive. For pass fractions across several tasks, encode `0`/`1` as numeric scores and use the stratified-bootstrap mean instead. + +Promotion uses the conservative endpoint: + +- `maximize` or `pass`: lower endpoint must be at least the frozen target. +- `minimize`: upper endpoint must be at most the frozen target. +- when `maxIntervalWidth` is set, the interval must also be no wider than that limit. + +The final evaluation score must exactly equal the backend estimate. The best unit, median seed chosen after inspection, and point estimate cannot authorize promotion. + +## Submission shape + +```json +{ + "sessionID": "bound-session", + "evaluatorToken": "inject out-of-band at request time; never persist in the preflight file", + "subject": { "type": "run", "id": "bound-run" }, + "observations": [ + { + "stratumID": "task-0", + "clusterID": "seed-0", + "stratumSHA256": "must equal the stratum commitmentSHA256", + "clusterSHA256": "must equal the cluster commitmentSHA256", + "status": "passed", + "score": 0.83, + "outputSHA256": "64 lowercase hex characters", + "environmentSHA256": "must equal replication.environmentSHA256", + "evidence": ["artifact:task-0/seed-0/result.json"], + "evaluatedAt": 1780000000000 + } + ] +} +``` + +Use `{ "type": "candidate", "id": "" }` only after that immutable candidate exists. Every observation must occur after subject creation and before receipt recording. Record the receipt before the subject's final evaluation. + +One subject can freeze only one receipt. An exact submission retry is idempotent; changed observations are rejected instead of creating a favorable alternate receipt. Use a new immutable run or candidate subject for a genuinely predeclared new experiment. + +## Failure interpretation + +- Missing or extra unit, or changed axis commitment: invalid request; restore the frozen grid and committed configuration. +- Duplicate unit or axis commitment: invalid independence claim; correct the design, not the score. +- Numeric failed/inconclusive unit: fail-closed receipt with no aggregate. +- Pass-rate inconclusive unit: inconclusive receipt. +- Bound misses target or interval is too wide: failed receipt; gather a new predeclared run rather than editing this receipt. +- Receipt hash, protocol, environment, session, subject, metric, timestamp, or derived-statistic mismatch: reject as drift, tampering, or replay. diff --git a/backend/cli/skills/research/run-replicated-evaluation/scripts/preflight.py b/backend/cli/skills/research/run-replicated-evaluation/scripts/preflight.py new file mode 100755 index 00000000..44ba7ab4 --- /dev/null +++ b/backend/cli/skills/research/run-replicated-evaluation/scripts/preflight.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +import json +import math +import re +import sys +from pathlib import Path + + +def fail(message: str) -> None: + raise SystemExit(message) + + +if len(sys.argv) != 3: + fail("usage: preflight.py ") + +contract = json.loads(Path(sys.argv[1]).read_text()) +payload = json.loads(Path(sys.argv[2]).read_text()) +protocol = contract.get("replication") +if not isinstance(protocol, dict) or protocol.get("protocolVersion") != "replicated-evaluation-v1": + fail("contract does not contain replicated-evaluation-v1") + +sampling = protocol.get("sampling", {}) +strata = sampling.get("strata", []) +clusters = sampling.get("clusters", []) +if isinstance(payload, dict) and "evaluatorToken" in payload: + fail("preflight artifacts must remain token-free; inject the capability only at request time") +if isinstance(payload, dict) and payload.get("sessionID", contract.get("sessionID")) != contract.get("sessionID"): + fail("submission session does not match the contract") +if isinstance(payload, dict) and payload.get("subject", {}).get("type") == "run": + if payload["subject"].get("id") != contract.get("runID"): + fail("run subject does not match the contract") +observations = payload.get("observations", payload) if isinstance(payload, dict) else payload +if not isinstance(observations, list): + fail("observations must be a JSON array or an object containing observations") + +expected = {(item["id"], cluster["id"]) for item in strata for cluster in clusters} +stratum_hashes = {item["id"]: item.get("commitmentSHA256") for item in strata} +cluster_hashes = {item["id"]: item.get("commitmentSHA256") for item in clusters} +actual = [(item.get("stratumID"), item.get("clusterID")) for item in observations] +if len(actual) != len(set(actual)): + fail("duplicate stratum-cluster observation") +missing = sorted(expected - set(actual)) +extra = sorted(set(actual) - expected) +if missing or extra: + fail(f"frozen grid mismatch: missing={missing} extra={extra}") + +estimator = protocol.get("estimator") +environment = protocol.get("environmentSHA256") +if not re.fullmatch(r"[a-f0-9]{64}", str(environment or "")): + fail("replication protocol requires a frozen environmentSHA256") +if estimator != "pass_rate" and len(clusters) < 5: + fail("numeric bootstrap requires at least five independent clusters") +if estimator == "pass_rate" and len(clusters) < 3: + fail("pass-rate evaluation requires at least three independent clusters") +if estimator == "pass_rate" and len(strata) != 1: + fail("Wilson pass-rate evaluation requires one Bernoulli stratum") +for item in observations: + status = item.get("status") + score = item.get("score") + if status not in {"passed", "failed", "inconclusive"}: + fail(f"invalid status for {item.get('stratumID')}/{item.get('clusterID')}") + if estimator == "pass_rate" and score is not None: + fail("pass_rate observations cannot contain scores") + if estimator != "pass_rate" and status == "passed" and ( + isinstance(score, bool) or not isinstance(score, (int, float)) or not math.isfinite(score) + ): + fail("passing numeric observations require scores") + if estimator != "pass_rate" and status != "passed" and score is not None: + fail("non-passing numeric observations cannot contain scores") + if not isinstance(item.get("evidence"), list) or not item["evidence"]: + fail("every observation requires evidence") + for field in ("stratumSHA256", "clusterSHA256", "outputSHA256", "environmentSHA256"): + if not re.fullmatch(r"[a-f0-9]{64}", str(item.get(field, ""))): + fail(f"every observation requires a valid {field}") + if item.get("stratumSHA256") != stratum_hashes[item.get("stratumID")]: + fail("observation changed a frozen stratum commitment") + if item.get("clusterSHA256") != cluster_hashes[item.get("clusterID")]: + fail("observation changed a frozen cluster commitment") + if item.get("environmentSHA256") != environment: + fail("every observation must match the frozen environmentSHA256") + if isinstance(item.get("evaluatedAt"), bool) or not isinstance(item.get("evaluatedAt"), int) or item["evaluatedAt"] <= 0: + fail("every observation requires a positive evaluatedAt timestamp") + +print( + json.dumps( + { + "valid": True, + "units": len(observations), + "strata": len(strata), + "clusters": len(clusters), + "estimator": estimator, + "statuses": { + status: sum(item.get("status") == status for item in observations) + for status in ("passed", "failed", "inconclusive") + }, + }, + sort_keys=True, + ) +) diff --git a/backend/cli/skills/research/run-sealed-confirmation/SKILL.md b/backend/cli/skills/research/run-sealed-confirmation/SKILL.md new file mode 100644 index 00000000..a40b3d56 --- /dev/null +++ b/backend/cli/skills/research/run-sealed-confirmation/SKILL.md @@ -0,0 +1,43 @@ +--- +name: run-sealed-confirmation +description: Run the evaluator-owned sealed-confirmation-v1 workflow for an OpenScience optimize session. Use when a terminal adaptive search must confirm exactly one backend-selected winner on a distinct held-out or release claim split, validate a claim-result envelope, or submit/read a capability-protected confirmation receipt without leaking claim feedback into optimization. +--- + +# Run Sealed Confirmation + +Treat optimization scores as provisional. Confirm only the immutable terminal selection returned by the backend, exactly once, through the claim evaluator capability. + +## Workflow + +1. Require a bound `sealed-confirmation-v1` contract. Stop if the optimization and claim splits or manifests are not distinct. +2. Request `POST /harness/confirmations/selection` from the evaluator host. Keep `confirmationToken` in the secret-owning transport; never write it to a file, log, prompt, notebook, or command line. +3. Stop if search is not terminal. Accept only the returned candidate artifact; never submit a candidate ID or replace the artifact. +4. Run the frozen claim validator once against the committed claim manifest and environment. Do not return partial metrics, per-example feedback, hidden inputs, or repair hints to the optimization process. +5. Create a token-free result JSON and validate it: + +```bash +python3 scripts/preflight.py \ + --protocol confirmation-protocol.json \ + --selection confirmation-selection.json \ + --result claim-result.json \ + --out confirmation-payload.json +``` + +6. Let the secret-owning transport add `confirmationToken` in memory and send the payload to `POST /harness/confirmations/receipts`. +7. Preserve the returned receipt ID. Do not rerun the claim split, resume search, capture claim hindsight, or generate a learned skill from the result. + +Read [references/protocol.md](references/protocol.md) before constructing API envelopes or diagnosing a rejected submission. + +## Result rules + +- Use `outcome: completed` only with a finite score and the exact bound metric equal to that score. +- Use `outcome: failed` or `inconclusive` without a score or bound metric. +- Echo the selection's candidate artifact hash and the frozen claim manifest, validator, and environment hashes. +- Include at least one blocking check and evidence. The backend derives pass/fail from checks, direction, and target. +- Treat an identical retry as transport recovery only. A changed result after the first canonical receipt is forbidden. + +## Stop conditions + +- Stop before claim evaluation if selection is unavailable, nonterminal, malformed, or contradicts the protocol. +- Stop on any commitment drift, candidate substitution, timestamp before selection, token exposure, or request for claim feedback during search. +- If the first receipt is failed or inconclusive, record it as final evidence. Do not open the holdout for repair. diff --git a/backend/cli/skills/research/run-sealed-confirmation/agents/openai.yaml b/backend/cli/skills/research/run-sealed-confirmation/agents/openai.yaml new file mode 100644 index 00000000..6933a8e7 --- /dev/null +++ b/backend/cli/skills/research/run-sealed-confirmation/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Sealed Confirmation" + short_description: "Confirm one terminal winner on hidden data" + default_prompt: "Use $run-sealed-confirmation to validate and submit a one-shot claim evaluation for the backend-selected winner." diff --git a/backend/cli/skills/research/run-sealed-confirmation/references/protocol.md b/backend/cli/skills/research/run-sealed-confirmation/references/protocol.md new file mode 100644 index 00000000..0428f982 --- /dev/null +++ b/backend/cli/skills/research/run-sealed-confirmation/references/protocol.md @@ -0,0 +1,42 @@ +# Sealed confirmation protocol + +## Trust boundary + +The optimization evaluator may be queried repeatedly and may return detailed feedback. Its results remain provisional. The claim evaluator has a distinct identity and capability, receives one terminal backend selection, and returns no search feedback. + +The backend enforces: + +- optimization split `development` or `validation`; +- claim split `held_out` or `release`; +- distinct committed manifests; +- a numeric metric, direction, and target shared by optimization and claim evaluation; +- one verified candidate selected from a terminal search state; +- one canonical content-addressed receipt per session; +- report quality derived only from the claim receipt for confirmation-enabled contracts. + +## Protected APIs + +`POST /harness/confirmations/selection` + +```json +{ + "sessionID": "session-id", + "confirmationToken": "in-memory secret" +} +``` + +The response binds `contractSHA256`, `protocolSHA256`, terminal search revision and stop reason, candidate ID and artifact, durable optimization-evaluation and search-result hashes, and selection time. Do not copy the candidate ID into a receipt submission; the server derives it. + +`POST /harness/confirmations/receipts` + +The token-free payload produced by `scripts/preflight.py` needs `confirmationToken` added only inside the authenticated transport. A completed submission contains the score and exact bound metric. Failed or inconclusive submissions contain neither. + +`POST /harness/confirmations/receipts/:receiptID` + +Use the same claim evaluator capability to read and validate the canonical receipt. Optimization evaluator credentials cannot access it. + +## Information firewall + +Never place claim inputs, per-example outcomes, notes, or feedback in an optimization evaluation, search observation, retrospective-memory record, coalition prompt, or learned-skill proposal. The terminal report may expose only the derived aggregate status, score, target result, evaluator identity, and receipt ID. + +Exact retries recover transport failures. Any changed output, metric, timestamp, evidence, or check after the first receipt is a second holdout attempt and must be rejected. diff --git a/backend/cli/skills/research/run-sealed-confirmation/scripts/preflight.py b/backend/cli/skills/research/run-sealed-confirmation/scripts/preflight.py new file mode 100755 index 00000000..1afe2021 --- /dev/null +++ b/backend/cli/skills/research/run-sealed-confirmation/scripts/preflight.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import math +from pathlib import Path + + +HASH = 64 + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def load(path: str) -> dict: + value = json.loads(Path(path).read_text()) + require(isinstance(value, dict), f"{path} must contain one JSON object") + return value + + +def exact(value: dict, name: str, allowed: set[str], required: set[str]) -> None: + require(set(value) <= allowed, f"{name} has unknown fields: {sorted(set(value) - allowed)}") + require(required <= set(value), f"{name} is missing fields: {sorted(required - set(value))}") + + +def sha(value: object) -> str: + data = json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode() + return hashlib.sha256(data).hexdigest() + + +def valid_hash(value: object) -> bool: + return isinstance(value, str) and len(value) == HASH and all(char in "0123456789abcdef" for char in value) + + +def secrets(value: object, location: str = "$") -> None: + if isinstance(value, dict): + for key, item in value.items(): + normalized = key.lower().replace("_", "").replace("-", "") + require( + not any(word in normalized for word in ("token", "secret", "apikey")), + f"secret field at {location}.{key}", + ) + require(key not in {"feedback", "notes"}, f"forbidden claim field at {location}.{key}") + secrets(item, f"{location}.{key}") + return + if isinstance(value, list): + for index, item in enumerate(value): + secrets(item, f"{location}[{index}]") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate a token-free sealed confirmation result") + parser.add_argument("--protocol", required=True) + parser.add_argument("--selection", required=True) + parser.add_argument("--result", required=True) + parser.add_argument("--out", required=True) + args = parser.parse_args() + + protocol = load(args.protocol) + selection = load(args.selection) + result = load(args.result) + secrets(protocol) + secrets(selection) + secrets(result) + + exact( + protocol, + "protocol", + {"protocolVersion", "optimization", "claim", "selection", "exposure", "failurePolicy"}, + {"protocolVersion", "optimization", "claim", "selection", "exposure", "failurePolicy"}, + ) + require(protocol["protocolVersion"] == "sealed-confirmation-v1", "unsupported protocol version") + require(protocol["optimization"]["split"] in {"development", "validation"}, "invalid optimization split") + require(protocol["claim"]["split"] in {"held_out", "release"}, "invalid claim split") + require( + protocol["optimization"]["manifestSHA256"] != protocol["claim"]["manifestSHA256"], + "manifests must differ", + ) + require( + protocol["selection"] == {"rule": "terminal-verified-best-v1", "subjects": 1}, + "invalid selection policy", + ) + require( + protocol["exposure"] + == {"policy": "terminal-receipt-only", "searchFeedback": False, "memoryCapture": False}, + "invalid exposure policy", + ) + require(protocol["failurePolicy"] == "fail-closed", "confirmation must fail closed") + for key in ("manifestSHA256", "validatorSHA256", "environmentSHA256"): + require(valid_hash(protocol["claim"][key]), f"claim.{key} must be a lowercase SHA-256") + + fields = { + "schemaVersion", + "protocolVersion", + "selectionID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "searchRevision", + "stopReason", + "candidateID", + "candidateArtifact", + "candidateCreatedAt", + "optimizationResultSHA256", + "optimizationEvaluationSHA256", + "selectedAt", + } + exact(selection, "selection", fields, fields) + require(selection["protocolVersion"] == "terminal-verified-best-selection-v1", "invalid selection version") + stable = dict(selection) + stable.pop("selectionID") + require(sha(stable) == selection["selectionID"], "selection content hash is invalid") + require(selection["protocolSHA256"] == sha(protocol), "selection does not bind this protocol") + require(valid_hash(selection["candidateArtifact"]["sha256"]), "candidate artifact hash is invalid") + + allowed = { + "candidateSHA256", + "manifestSHA256", + "validatorSHA256", + "environmentSHA256", + "outcome", + "score", + "metrics", + "checks", + "evidence", + "usage", + "outputSHA256", + "evaluatedAt", + } + exact(result, "result", allowed, allowed - {"score", "usage"}) + require( + result["candidateSHA256"] == selection["candidateArtifact"]["sha256"], + "candidate substitution detected", + ) + for key in ("manifestSHA256", "validatorSHA256", "environmentSHA256"): + require(result[key] == protocol["claim"][key], f"frozen {key} changed") + require(result["outcome"] in {"completed", "failed", "inconclusive"}, "invalid outcome") + require(isinstance(result["checks"], list) and result["checks"], "checks must be non-empty") + require(any(item.get("blocking") is True for item in result["checks"]), "a blocking check is required") + require(isinstance(result["evidence"], list) and result["evidence"], "evidence must be non-empty") + require(valid_hash(result["outputSHA256"]), "outputSHA256 is invalid") + require( + isinstance(result["evaluatedAt"], int) and result["evaluatedAt"] >= selection["selectedAt"], + "evaluation predates selection", + ) + + metric = protocol["claim"]["metric"] + completed = result["outcome"] == "completed" + score = result.get("score") + require( + not completed or isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(score), + "completed result needs a finite score", + ) + require(completed or score is None, "incomplete result cannot expose a score") + require(not completed or result["metrics"].get(metric) == score, "bound metric must equal score") + require(completed or metric not in result["metrics"], "incomplete result cannot expose the bound metric") + + payload = {"schemaVersion": 1, "sessionID": selection["sourceSessionID"], **result} + Path(args.out).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + direction = protocol["claim"]["direction"] + target = protocol["claim"]["target"] + reached = completed and (score >= target if direction == "maximize" else score <= target) + print(json.dumps({"valid": True, "tokenFree": True, "derivedTargetReached": reached, "out": args.out})) + + +if __name__ == "__main__": + main() diff --git a/backend/cli/skills/research/run-topic-aware-failure-discovery/SKILL.md b/backend/cli/skills/research/run-topic-aware-failure-discovery/SKILL.md new file mode 100644 index 00000000..1eff73a1 --- /dev/null +++ b/backend/cli/skills/research/run-topic-aware-failure-discovery/SKILL.md @@ -0,0 +1,66 @@ +--- +name: run-topic-aware-failure-discovery +description: Preflight and execute an evaluator-owned topic-aware adversarial failure-discovery stream with frozen topic, generator, validator, embedding, audit-pool, and budget commitments. Use when generated stress cases should discover diverse target-model failures without entering the official benchmark score or population estimate. +--- + +# Run topic-aware failure discovery + +Keep this workflow in the external evaluator process. Never expose topic definitions, +anchor contents, generated cases, expected answers, evaluator tokens, or target outputs to +the candidate agent. + +## Preflight + +1. Finish and seal the bound active audit. Use only its authenticated observed + failures as anchors. +2. Prepare a private JSON manifest containing: + - `sourcePoolSHA256` from the audit receipt; + - two to 64 topics as `{id, definition}`, using opaque IDs containing only + letters, digits, `.`, `_`, `:`, or `-`, plus `topicSaltPath` pointing to a + private file with at least 32 random bytes; + - topic-model, generator, correctness-validator, topic-validator, + novelty-validator, and embedding identities with local `promptPath` and + `configPath` files; the generator and three validators must use distinct + prompt/config commitment pairs, not merely different display names; + - embedding dimensions, attempt budget, anchors per attempt, the exact audit + failure threshold, and optional failure target. +3. Run `bun scripts/preflight.ts private-manifest.json > preflight.json`. +4. Inspect `preflight.json`. Topic commitments are salted, so keep the salt with + the private manifest for later opening or audit. The output contains opaque + topic IDs/commitments and frozen identity metadata/file hashes, but no topic + definitions or prompt/config bytes. Bind `preflight.json.protocol` as + `failureDiscovery` in the harness task. Keep the source manifest and + referenced files outside the candidate environment. + +## Execute + +1. Initialize `POST /harness/failure-streams` with the exact subject artifact + and terminal audit receipt. +2. Request `POST /harness/failure-streams/:streamID/selection`. Use exactly the + returned topic and anchors; never choose either client-side. +3. Generate one case in the returned topic while transposing the anchors' + failure pattern. Hash the canonical hidden case and generator output. +4. Run the three frozen validators independently: + - `correctness`: the generated task and answer are valid; + - `topic`: the case belongs to the selected topic; + - `novelty`: it is not a semantic restatement of prior cases. +5. Submit the attempt. A generation failure uses `generation.status=failed`, no + validators, and no target outcome. A generated case supplies all validators; + submit a target outcome only when all three pass. Supply an L2-normalized + embedding from the frozen embedding identity. +6. Repeat until the server reports `completed`, then seal the receipt. Cite its + ID in `failureDiscoveryReceiptID` only as robustness provenance. + +## Integrity rules + +- Every attempt consumes budget. Invalid, inconclusive, failed, or duplicate + cases earn zero reward. +- OpenScience forces every topic once, then recomputes UCB1 from the immutable + attempt journal. Treat a returned selection as a server lease, not advice. +- Never put generated cases into the active-audit pool, a benchmark evaluation + metric, candidate fitness, retrospective score memory, or a confirmation + split. +- The receipt proves protocol execution and records failure yield/diversity. It + does not prove state of the art or change the official score. +- If the generator and validator disagree, retain the failed/inconclusive + attempt. Do not repair its label after observing the target result. diff --git a/backend/cli/skills/research/run-topic-aware-failure-discovery/agents/openai.yaml b/backend/cli/skills/research/run-topic-aware-failure-discovery/agents/openai.yaml new file mode 100644 index 00000000..647d662f --- /dev/null +++ b/backend/cli/skills/research/run-topic-aware-failure-discovery/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Topic-Aware Failure Discovery" + short_description: "Discover validated failures without score leakage" + default_prompt: "Use $run-topic-aware-failure-discovery to execute a committed topic-aware adversarial evaluation stream." diff --git a/backend/cli/skills/research/run-topic-aware-failure-discovery/scripts/preflight.ts b/backend/cli/skills/research/run-topic-aware-failure-discovery/scripts/preflight.ts new file mode 100755 index 00000000..6fedfffb --- /dev/null +++ b/backend/cli/skills/research/run-topic-aware-failure-discovery/scripts/preflight.ts @@ -0,0 +1,149 @@ +#!/usr/bin/env bun + +import path from "path" + +const file = process.argv[2] +if (!file) throw new Error("Usage: bun scripts/preflight.ts ") + +const Hash = /^[a-f0-9]{64}$/ +const Topic = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$/ +const kinds = ["correctness", "topic", "novelty"] as const +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const record = (value: unknown, label: string) => { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`) + return value as Record +} +const fields = (value: Record, label: string, allowed: string[]) => { + const unknown = Object.keys(value).filter((key) => !allowed.includes(key)) + if (unknown.length) throw new Error(`${label} has unknown fields: ${unknown.join(", ")}`) +} +const text = (value: unknown, label: string) => { + if (typeof value !== "string" || !value.trim()) throw new Error(`${label} must be a non-empty string`) + return value +} +const integer = (value: unknown, label: string, min: number, max: number) => { + if (!Number.isInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer from ${min} to ${max}`) + } + return value as number +} +const number = (value: unknown, label: string, min: number, max: number) => { + if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) { + throw new Error(`${label} must be a finite number from ${min} to ${max}`) + } + return value +} +const bytes = async (value: unknown, label: string) => { + const target = path.resolve(path.dirname(path.resolve(file)), text(value, label)) + const source = Bun.file(target) + if (!(await source.exists())) throw new Error(`${label} does not exist: ${target}`) + return source.text() +} +const identity = async (value: unknown, label: string, extras: string[] = []) => { + const input = record(value, label) + fields(input, label, ["name", "version", "promptPath", "configPath", ...extras]) + return { + name: text(input.name, `${label}.name`), + version: text(input.version, `${label}.version`), + promptSHA256: hash(await bytes(input.promptPath, `${label}.promptPath`)), + configSHA256: hash(await bytes(input.configPath, `${label}.configPath`)), + } +} + +const input = record(await Bun.file(path.resolve(file)).json(), "manifest") +fields(input, "manifest", [ + "sourcePoolSHA256", + "topicSaltPath", + "topics", + "topicModel", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "exploration", + "failureThreshold", + "targetFailures", +]) +const sourcePoolSHA256 = text(input.sourcePoolSHA256, "sourcePoolSHA256") +if (!Hash.test(sourcePoolSHA256)) throw new Error("sourcePoolSHA256 must be a lowercase SHA-256 digest") +const salt = await bytes(input.topicSaltPath, "topicSaltPath") +if (new TextEncoder().encode(salt).length < 32) throw new Error("topicSaltPath must contain at least 32 bytes") +if (!Array.isArray(input.topics) || input.topics.length < 2 || input.topics.length > 64) { + throw new Error("topics must contain two to 64 entries") +} +const topics = input.topics + .map((value, index) => { + const topic = record(value, `topics[${index}]`) + fields(topic, `topics[${index}]`, ["id", "definition"]) + const id = text(topic.id, `topics[${index}].id`) + if (!Topic.test(id)) throw new Error(`topics[${index}].id must be an opaque safe identifier`) + const definition = text(topic.definition, `topics[${index}].definition`) + return { id, commitment: hash(JSON.stringify({ id, definition, salt })) } + }) + .toSorted((left, right) => (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)) +if (new Set(topics.map((topic) => topic.id)).size !== topics.length) throw new Error("topic IDs must be unique") +if (new Set(topics.map((topic) => topic.commitment)).size !== topics.length) { + throw new Error("topic definitions must produce unique commitments") +} + +const validators = record(input.validators, "validators") +fields(validators, "validators", [...kinds]) +const generator = await identity(input.generator, "generator") +const panel = await Promise.all( + kinds.map(async (kind) => ({ kind, identity: await identity(validators[kind], `validators.${kind}`) })), +) +const actors = [generator, ...panel.map((item) => item.identity)] +const actorIDs = actors.map((item) => `${item.promptSHA256}:${item.configSHA256}`) +if (new Set(actorIDs).size !== actorIDs.length) { + throw new Error("generator and validators must use distinct prompt/config commitments") +} + +const topicModel = record(input.topicModel, "topicModel") +const embedding = record(input.embedding, "embedding") +const budget = integer(input.budget, "budget", 2, 512) +if (budget < topics.length) throw new Error("budget must initialize every topic arm") +const targetFailures = + input.targetFailures === undefined ? undefined : integer(input.targetFailures, "targetFailures", 1, budget) +const protocol = { + protocolVersion: "topic-aware-failure-v1" as const, + sourcePoolSHA256, + topicModel: { + kind: text(topicModel.kind, "topicModel.kind"), + identity: await identity(topicModel, "topicModel", ["kind"]), + }, + topics, + generator, + validators: panel, + embedding: { + identity: await identity(embedding, "embedding", ["dimensions", "regularization"]), + dimensions: integer(embedding.dimensions, "embedding.dimensions", 2, 64), + regularization: + embedding.regularization === undefined + ? 1e-6 + : number(embedding.regularization, "embedding.regularization", Number.MIN_VALUE, 0.01), + }, + budget, + anchorsPerAttempt: integer(input.anchorsPerAttempt, "anchorsPerAttempt", 1, 8), + exploration: + input.exploration === undefined ? Math.SQRT2 : number(input.exploration, "exploration", Number.MIN_VALUE, 4), + failureThreshold: number(input.failureThreshold, "failureThreshold", 0, 1), + ...(targetFailures === undefined ? {} : { targetFailures }), +} +if (protocol.topicModel.kind !== "predefined" && protocol.topicModel.kind !== "bertopic") { + throw new Error("topicModel.kind must be predefined or bertopic") +} + +console.log( + JSON.stringify( + { + protocol, + commitments: { + topicManifestSHA256: hash(JSON.stringify(topics)), + sourcePoolSHA256, + }, + }, + null, + 2, + ), +) diff --git a/backend/cli/skills/research/run-verifier-routed-research/SKILL.md b/backend/cli/skills/research/run-verifier-routed-research/SKILL.md new file mode 100644 index 00000000..5b6eff34 --- /dev/null +++ b/backend/cli/skills/research/run-verifier-routed-research/SKILL.md @@ -0,0 +1,38 @@ +--- +name: run-verifier-routed-research +description: Execute one OpenScience verifier_loop work unit as a generator, targeted reviser, evidence investigator, or independent verifier. Use when coalition_ready returns a verifier-routed-v1 unit and the worker must honor clean-restart context isolation, structured severity verdicts, observable evidence, and the backend-owned attempt budget without choosing its own next route. +--- + +# Run Verifier-Routed Research + +Execute exactly the bound unit. The orchestration backend owns routing; worker text cannot accept a candidate or request its preferred retry. + +## Workflow + +1. Save the complete `coalition_ready` work object as JSON and run `python3 scripts/validate_unit.py `. Stop on validation failure. +2. Read [references/protocol.md](references/protocol.md) before handling a role or verdict format that is unfamiliar. +3. Use only the direct `context` supplied by the unit: + - `initial-candidate`: build an independent complete candidate. + - `clean-restart-*`: start from a blank solution using verifier summaries only as failure constraints. Do not reconstruct or minimally edit the rejected candidate. + - `targeted-revision-*`: correct the cited failed checks, preserve supported components, and return a complete replacement artifact. + - `evidence-investigation-*`: acquire evidence or counterexamples for inconclusive checks; do not edit the candidate. + - `repair-verification-*`: independently inspect the candidate and observable evidence without reading another verifier's verdict. +4. Stay within the unit's allocation. Use task-appropriate tools to create durable artifacts and evidence references. +5. Return one structured result with `summary`, `artifactRefs`, `evidenceRefs`, and actual `usage`. Verification units must also return `verdict` with `decision`, `severity`, `confidence`, and evidence-backed `checks`. + +## Verdict rules + +- `support` + `none`: every declared check passed. +- `reject` + `minor`: a localized correction can preserve the candidate's premise. +- `reject` + `critical`: the premise, interpretation, or global reasoning is invalid. +- `abstain` + `unknown`: available evidence cannot decide at least one material check. + +Never weaken a critical defect to obtain a revision, call missing evidence a minor defect, or claim support from confidence alone. Every check needs at least one observable evidence reference. + +## Integrity rules + +- Do not inspect hidden tests, evaluator internals, another verifier's output, or omitted ancestor artifacts. +- Do not add dependencies or resume another worker session. +- Treat summaries as provisional and evidence references as pointers, not proof by themselves. +- Report failure instead of fabricating artifacts, checks, resource usage, or a conclusive verdict. +- Do not claim benchmark improvement or scientific acceptance; only evaluator settlement can promote a result. diff --git a/backend/cli/skills/research/run-verifier-routed-research/agents/openai.yaml b/backend/cli/skills/research/run-verifier-routed-research/agents/openai.yaml new file mode 100644 index 00000000..d1f9fa5a --- /dev/null +++ b/backend/cli/skills/research/run-verifier-routed-research/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Run Verifier-Routed Research" + short_description: "Execute bounded verifier-routed research units" + default_prompt: "Use $run-verifier-routed-research to execute this OpenScience verifier-loop work unit and return one compliant structured result." diff --git a/backend/cli/skills/research/run-verifier-routed-research/references/protocol.md b/backend/cli/skills/research/run-verifier-routed-research/references/protocol.md new file mode 100644 index 00000000..5b8c0e05 --- /dev/null +++ b/backend/cli/skills/research/run-verifier-routed-research/references/protocol.md @@ -0,0 +1,55 @@ +# Verifier-routed-v1 protocol + +The backend runs bounded panels and derives routes with fixed precedence: + +1. Any `reject/critical` verdict causes a clean generator restart. +2. Otherwise, any `abstain/unknown` verdict causes evidence investigation. +3. Otherwise, any `reject/minor` verdict causes targeted revision. +4. Otherwise, unanimous `support/none` verdicts at or above the contract confidence threshold accept the candidate. +5. Any remaining low-confidence panel causes evidence investigation. + +The attempt ceiling is `orchestration.maxRounds`. Every possible candidate/action and verifier panel is budgeted before execution. Reaching the ceiling stops without another action. Only the terminal panel contributes to final consensus; earlier rejected or inconclusive panels remain provenance. + +## Context boundaries + +| Unit | Direct context | Forbidden context | +|---|---|---| +| Initial candidate | None | Prior attempts | +| Targeted revision | Candidate and the rejecting panel | Hidden reasoning and unrelated ancestors | +| Clean restart | Rejecting panel only | Rejected candidate artifact or summary | +| Evidence investigation | Candidate and inconclusive panel | Candidate edits | +| Verification | Current candidate and optional new investigation | Other verifier verdicts and stale panels | + +The DAG may retain transitive causal provenance even when a clean restart omits the rejected candidate from the worker's direct context. + +## Result envelope + +Non-verifier: + +```json +{ + "summary": "what was produced or observed", + "artifactRefs": ["artifact://..."], + "evidenceRefs": ["evidence://..."], + "usage": {"steps": 1, "tokens": 1000, "costUSD": 0.01, "wallTimeMs": 1000} +} +``` + +Verifier: + +```json +{ + "summary": "independent review", + "artifactRefs": [], + "evidenceRefs": ["evidence://review.json"], + "usage": {"steps": 1, "tokens": 1000}, + "verdict": { + "decision": "reject", + "severity": "minor", + "confidence": 0.84, + "checks": [ + {"id": "residual-check", "status": "failed", "evidenceRefs": ["evidence://residual.json"]} + ] + } +} +``` diff --git a/backend/cli/skills/research/run-verifier-routed-research/scripts/validate_unit.py b/backend/cli/skills/research/run-verifier-routed-research/scripts/validate_unit.py new file mode 100755 index 00000000..908cd164 --- /dev/null +++ b/backend/cli/skills/research/run-verifier-routed-research/scripts/validate_unit.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +import json +import re +import sys +from pathlib import Path + + +def fail(message: str) -> None: + raise SystemExit(f"invalid verifier-loop unit: {message}") + + +if len(sys.argv) != 2: + fail("usage: validate_unit.py ") + +data = json.loads(Path(sys.argv[1]).read_text()) +if not isinstance(data, dict): + fail("root must be an object") +if not re.fullmatch(r"[a-f0-9]{64}", str(data.get("id", ""))): + fail("id must be a sha256 digest") +if data.get("status") != "pending": + fail("work must be pending") + +role = data.get("role") +if role not in {"generation", "revision", "verification", "investigation"}: + fail("unsupported role") +label = data.get("label") +if not isinstance(label, str) or not label: + fail("label is required") +prompt = data.get("prompt") +if not isinstance(prompt, str) or f'role="{role}" topology="verifier_loop"' not in prompt: + fail("prompt is not bound to the declared verifier-loop role") + +context = data.get("context") +if not isinstance(context, list) or any(not isinstance(item, dict) for item in context): + fail("context must be an array of work results") +roles = [item.get("role") for item in context] +candidate = any(item in {"generation", "revision"} for item in roles) +reviews = sum(item == "verification" for item in roles) + +if role == "generation" and label == "initial-candidate" and context: + fail("initial candidate must not receive ancestor context") +if role == "generation" and label.startswith("clean-restart-") and (not context or reviews != len(context)): + fail("clean restart may receive verifier summaries only") +if role == "revision" and (not candidate or reviews < 1): + fail("targeted revision needs the candidate and rejecting panel") +if role == "investigation" and (not candidate or reviews < 1): + fail("investigation needs the candidate and inconclusive panel") +if role == "verification" and (not candidate or reviews): + fail("verification needs a candidate and cannot receive another verifier verdict") + +allocation = data.get("allocation") +if not isinstance(allocation, dict) or not allocation: + fail("allocation is required") +if any(not isinstance(value, (int, float)) or value < 0 for value in allocation.values()): + fail("allocation values must be nonnegative numbers") + +requirements = { + "generation": "return one complete candidate artifact", + "revision": "return one complete replacement artifact", + "investigation": "return evidence without editing the candidate", + "verification": "return an evidence-backed decision, severity, confidence, and checks", +} +print(json.dumps({"valid": True, "role": role, "label": label, "requirement": requirements[role]})) diff --git a/backend/cli/skills/research/scientific-ablation-design/SKILL.md b/backend/cli/skills/research/scientific-ablation-design/SKILL.md new file mode 100644 index 00000000..1051081a --- /dev/null +++ b/backend/cli/skills/research/scientific-ablation-design/SKILL.md @@ -0,0 +1,68 @@ +--- +name: scientific-ablation-design +description: Design and validate controlled ablations for a benchmark, agent architecture, model, or scientific pipeline. Use when attributing a measured improvement to orchestration, memory, search, tools, training choices, simulators, or another claimed mechanism. +--- + +# Scientific Ablation Design + +Turn “this component helped” into a predeclared, budget-matched contrast. + +## Define claims before results + +For every claimed mechanism, write: + +- one named factor; +- its full-system value and ablated value; +- the predicted direction and primary metric; +- the failure observation that would weaken the claim; and +- any interaction that cannot be identified by a one-factor contrast. + +Do not create ablations only for components that look favorable after the main run. + +## Match the experimental context + +Use the same evaluator version, held-out split, model, prompt/template, tool policy, candidate budget, wall-time/compute cap, seeds, stopping rule, and contamination policy for baseline and isolation arms. Change exactly one factor per isolation arm. + +Add interaction arms only when explicitly declared. Label multi-factor arms as interactions; do not report them as isolated causal evidence. + +## Validate the matrix + +Create a plan JSON: + +```json +{ + "metric":{"name":"score","direction":"maximize"}, + "baseline":{"id":"full","config":{"memory":"verified","search":"ucb"},"seeds":[1,2,3],"budget":{"candidates":30},"split":"held_out","evaluator":"eval-sha"}, + "claims":[{"id":"memory-value","factor":"memory","from":"verified","to":"none"}], + "arms":[{"id":"no-memory","config":{"memory":"none","search":"ucb"},"seeds":[1,2,3],"budget":{"candidates":30},"split":"held_out","evaluator":"eval-sha"}] +} +``` + +Run: + +```bash +python scripts/validate_ablation_plan.py ablations.json --output ablation-report.json +``` + +The validator rejects missing isolation arms, multiple changes disguised as one ablation, seed/budget/split/evaluator drift, duplicate IDs, and claims whose declared baseline value is false. + +## Execute and analyze + +1. Run arms in a randomized or interleaved order when shared infrastructure can drift. +2. Preserve all seeds and failures. +3. Compute paired seed-level differences when pairing is valid. +4. Report effect size and uncertainty, not only whether the mean changed. +5. Correct for multiplicity across many factors or predeclare one primary contrast. +6. Inspect quality-cost Pareto changes; a score gain bought by materially more compute is not an isolated method gain. +7. Keep the claim weakened when effects are unstable, interaction-dependent, or below practical relevance. + +## Guard against invalid attribution + +Do not accept: + +- a different evaluator or data split; +- unequal search or training budget; +- cherry-picked seeds; +- an arm that changes prompts, models, tools, and component simultaneously; +- a comparison to an obsolete baseline when a stronger matched parent exists; or +- internal reviewer preference as a substitute for the declared metric. diff --git a/backend/cli/skills/research/scientific-ablation-design/agents/openai.yaml b/backend/cli/skills/research/scientific-ablation-design/agents/openai.yaml new file mode 100644 index 00000000..14929387 --- /dev/null +++ b/backend/cli/skills/research/scientific-ablation-design/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Scientific Ablation Design" + short_description: "Design controlled benchmark ablations" + default_prompt: "Use $scientific-ablation-design to construct and validate a controlled ablation matrix for this claimed improvement." diff --git a/backend/cli/skills/research/scientific-ablation-design/scripts/validate_ablation_plan.py b/backend/cli/skills/research/scientific-ablation-design/scripts/validate_ablation_plan.py new file mode 100755 index 00000000..937d169e --- /dev/null +++ b/backend/cli/skills/research/scientific-ablation-design/scripts/validate_ablation_plan.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Validate one-factor benchmark ablations against a frozen baseline.""" + +import argparse +import hashlib +import json +import os +import sys +import tempfile +from pathlib import Path + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def context(run: dict) -> dict: + return {key: run.get(key) for key in ("seeds", "budget", "split", "evaluator")} + + +def write(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def validate_run(run: object, label: str) -> dict: + require(isinstance(run, dict), f"{label} must be an object") + require(isinstance(run.get("id"), str) and run["id"], f"{label}.id is required") + require(isinstance(run.get("config"), dict) and run["config"], f"{label}.config is required") + seeds = run.get("seeds") + require(isinstance(seeds, list) and seeds, f"{label}.seeds must be non-empty") + require(all(isinstance(seed, int) and not isinstance(seed, bool) for seed in seeds), f"{label}.seeds must be integers") + require(len(set(seeds)) == len(seeds), f"{label}.seeds must be unique") + require(isinstance(run.get("budget"), dict) and run["budget"], f"{label}.budget is required") + require(isinstance(run.get("split"), str) and run["split"], f"{label}.split is required") + require(isinstance(run.get("evaluator"), str) and run["evaluator"], f"{label}.evaluator is required") + return run + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("input", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + source = args.input.read_bytes() + data = json.loads(source) + require(isinstance(data, dict), "ablation plan must be an object") + metric = data.get("metric") + require(isinstance(metric, dict), "metric is required") + require(isinstance(metric.get("name"), str) and metric["name"], "metric.name is required") + require(metric.get("direction") in {"maximize", "minimize", "pass"}, "metric.direction is invalid") + + baseline = validate_run(data.get("baseline"), "baseline") + claims = data.get("claims") + arms = data.get("arms") + require(isinstance(claims, list) and claims, "at least one claim is required") + require(isinstance(arms, list) and arms, "at least one ablation arm is required") + arms = [validate_run(arm, f"arm[{index}]") for index, arm in enumerate(arms)] + identifiers = [baseline["id"], *[arm["id"] for arm in arms]] + require(len(set(identifiers)) == len(identifiers), "baseline and arm ids must be unique") + claim_ids = [claim.get("id") for claim in claims if isinstance(claim, dict)] + require(len(claim_ids) == len(claims) and all(isinstance(value, str) and value for value in claim_ids), "claim ids are required") + require(len(set(claim_ids)) == len(claim_ids), "claim ids must be unique") + + contrasts = [] + for index, claim in enumerate(claims): + require(isinstance(claim, dict), f"claim[{index}] must be an object") + factor = claim.get("factor") + require(isinstance(factor, str) and factor, f"claim {claim['id']} needs a factor") + require("from" in claim and "to" in claim, f"claim {claim['id']} needs explicit from and to values") + require(factor in baseline["config"], f"claim {claim['id']} factor is absent from the baseline") + require(baseline["config"][factor] == claim.get("from"), f"claim {claim['id']} baseline value does not match from") + matches = [] + for arm in arms: + keys = set(baseline["config"]) | set(arm["config"]) + changes = sorted(key for key in keys if baseline["config"].get(key) != arm["config"].get(key)) + if arm.get("interactionFactors") is not None: + interaction = arm["interactionFactors"] + require(isinstance(interaction, list) and len(interaction) >= 2, f"arm {arm['id']} interactionFactors is invalid") + require(sorted(interaction) == changes, f"arm {arm['id']} interactionFactors do not match its changes") + else: + require(len(changes) == 1, f"arm {arm['id']} changes {len(changes)} factors without an interaction declaration") + require(context(arm) == context(baseline), f"arm {arm['id']} drifts seed, budget, split, or evaluator") + if changes == [factor] and arm["config"].get(factor) == claim.get("to"): + matches.append(arm["id"]) + require(len(matches) == 1, f"claim {claim['id']} needs exactly one matching isolation arm") + contrasts.append({"claim": claim["id"], "baseline": baseline["id"], "arm": matches[0], "factor": factor}) + + report = { + "schemaVersion": 1, + "passed": True, + "inputSHA256": hashlib.sha256(source).hexdigest(), + "metric": metric, + "baseline": baseline["id"], + "contrasts": contrasts, + "seeds": baseline["seeds"], + "budget": baseline["budget"], + } + if args.output: + write(args.output, report) + print(json.dumps(report, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(json.dumps({"error": str(error)}), file=sys.stderr) + raise SystemExit(2) diff --git a/backend/cli/skills/research/trace-evolutionary-candidate/SKILL.md b/backend/cli/skills/research/trace-evolutionary-candidate/SKILL.md new file mode 100644 index 00000000..1bde7190 --- /dev/null +++ b/backend/cli/skills/research/trace-evolutionary-candidate/SKILL.md @@ -0,0 +1,74 @@ +--- +name: trace-evolutionary-candidate +description: Build an evaluator-owned, content-addressed source snapshot and exact parent-delta submission for an OpenScience optimization candidate. Use before recording a final candidate evaluation when the harness contract binds evolution-trace-v1, or when evolutionary search lineage, deterministic replay, source novelty, ancestral code reintroduction, or cycle diagnostics must be independently auditable. +--- + +# Trace Evolutionary Candidate + +Run this skill in the evaluator process, never in the candidate agent. It scans the contract-frozen source roots, creates a canonical manifest, verifies every local parent against its already-recorded snapshot, writes deterministic delta artifacts, and builds the token-free body for `POST /harness/evolution/receipts`. + +## Workflow + +1. Freeze `evolution-trace-v1` before search. Pin the exact SHA-256 of `scripts/trace_candidate.py` and [references/manifest-schema.json](references/manifest-schema.json), plus source roots, extensions, exclusions, and resource bounds. +2. Capture candidate and parent worktrees outside the candidate sandbox. Do not accept a candidate-authored manifest, diff, or line count. +3. Record every passing parent trace before admitting a child. Each parent specification must name its candidate artifact, receipt, snapshot artifact, and exact local root. +4. Run the builder. It rejects symlinks, invalid UTF-8, omitted or substituted parent snapshots, duplicate paths, out-of-scope files, and bound violations. It hashes exact non-empty line bytes; indentation and carriage returns are significant. +5. Inject `evaluatorToken` only into the authenticated request in memory. Never write it into the submission, report, manifest, or delta artifacts. +6. Record the returned receipt before the final candidate evaluation and reference it as `evolutionReceiptID`. +7. Treat `cycleDetected`, `reintroducedLines`, and novelty diagnostics as analysis only. They are not benchmark scores and cannot verify or promote a candidate. + +## Commands + +Print exact validator and manifest-schema commitments: + +```bash +python scripts/trace_candidate.py commitments +``` + +Build a root-candidate submission: + +```bash +python scripts/trace_candidate.py build \ + --contract contract-evolution.json \ + --subject candidate.json \ + --candidate-root ./candidate-worktree \ + --artifact-dir ./trace-artifacts \ + --run-id run-123 \ + --session-id session-123 \ + --output evolution-submission.json \ + --report evolution-diagnostics.json +``` + +For a descendant, repeat `--parent` once per declared parent: + +```bash +python scripts/trace_candidate.py build \ + --contract contract-evolution.json \ + --subject child.json \ + --candidate-root ./child-worktree \ + --parent parent-a.json \ + --parent parent-b.json \ + --artifact-dir ./trace-artifacts \ + --run-id run-123 \ + --session-id session-123 \ + --output evolution-submission.json +``` + +## Inputs + +- `contract-evolution.json`: the exact `HarnessContract.Evolution` value. +- `candidate.json`: `{ "type": "candidate", "id": "", "artifact": { "uri": "...", "sha256": "" } }`. +- `parent-*.json`: `{ "id": "", "artifact": {...}, "receiptID": "", "snapshot": { "uri": "...", "sha256": "" }, "root": "./exact-parent-worktree", "deltaURI": "optional stable URI" }`. +- `--candidate-root`: exact unpacked candidate artifact, captured by the evaluator. + +The output intentionally omits `evaluatorToken`. Its snapshot and delta URIs remain replay references; their SHA-256 values bind exact canonical bytes. + +## Fail-Closed Rules + +- Scan only contract-frozen roots and extensions, while honoring every committed exclusion. +- Reject symlinks instead of following them across the capture boundary. +- Reject invalid UTF-8 instead of silently treating binary data as source. +- Hash exact non-empty byte lines split only on LF. Do not trim whitespace or normalize CRLF. +- Require local parent manifests to match the immutable snapshot hashes in their specifications. +- Require one parent specification for every declared search parent and no others. +- Do not infer fitness, edit semantics, or scientific novelty. The backend recomputes structural deltas and ancestral reintroductions; the benchmark evaluator remains the sole fitness authority. diff --git a/backend/cli/skills/research/trace-evolutionary-candidate/agents/openai.yaml b/backend/cli/skills/research/trace-evolutionary-candidate/agents/openai.yaml new file mode 100644 index 00000000..c5adc069 --- /dev/null +++ b/backend/cli/skills/research/trace-evolutionary-candidate/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Trace Evolutionary Candidate" + short_description: "Capture replayable candidate lineage and cycles" + default_prompt: "Use $trace-evolutionary-candidate to build an evaluator-owned evolution trace submission for a candidate and its exact parents." diff --git a/backend/cli/skills/research/trace-evolutionary-candidate/references/manifest-schema.json b/backend/cli/skills/research/trace-evolutionary-candidate/references/manifest-schema.json new file mode 100644 index 00000000..a228ed04 --- /dev/null +++ b/backend/cli/skills/research/trace-evolutionary-candidate/references/manifest-schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://syntheticsciences.ai/schemas/evolution-source-manifest-v1.json", + "title": "OpenScience evaluator-owned evolution source manifest", + "type": "object", + "required": ["schemaVersion", "lineAlgorithm", "files"], + "properties": { + "schemaVersion": { "const": 1 }, + "lineAlgorithm": { "const": "sha256-exact-line-v1" }, + "files": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["path", "sha256", "bytes", "lineHashes"], + "properties": { + "path": { "type": "string", "minLength": 1, "maxLength": 1000 }, + "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "bytes": { "type": "integer", "minimum": 0 }, + "lineHashes": { + "type": "array", + "items": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/backend/cli/skills/research/trace-evolutionary-candidate/scripts/trace_candidate.py b/backend/cli/skills/research/trace-evolutionary-candidate/scripts/trace_candidate.py new file mode 100755 index 00000000..741f76f2 --- /dev/null +++ b/backend/cli/skills/research/trace-evolutionary-candidate/scripts/trace_candidate.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +"""Build evaluator-owned OpenScience evolutionary source provenance.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import time +from collections import Counter +from pathlib import Path, PurePosixPath +from typing import Any + + +HASH = set("0123456789abcdef") +ROOT = Path(__file__).resolve().parent.parent +SCHEMA = ROOT / "references" / "manifest-schema.json" +ALGORITHM = "sha256-exact-line-v1" + + +class Invalid(ValueError): + pass + + +def canonical(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def hash_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_hash(value: Any) -> bool: + return isinstance(value, str) and len(value) == 64 and set(value) <= HASH + + +def obj(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise Invalid(f"{label} must be a JSON object") + return value + + +def text(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise Invalid(f"{label} must be a non-empty string") + return value + + +def digest(value: Any, label: str) -> str: + if not is_hash(value): + raise Invalid(f"{label} must be a lowercase SHA-256") + return value + + +def integer(value: Any, label: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise Invalid(f"{label} must be an integer >= {minimum}") + return value + + +def load(path: Path, label: str) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise Invalid(f"cannot read {label} {path}: {exc}") from exc + + +def artifact(value: Any, label: str) -> dict[str, str]: + item = obj(value, label) + return { + "uri": text(item.get("uri"), f"{label}.uri"), + "sha256": digest(item.get("sha256"), f"{label}.sha256"), + } + + +def relative(value: Any, label: str, dot: bool = False) -> str: + item = text(value, label) + if dot and item == ".": + return item + pure = PurePosixPath(item) + if ( + pure.is_absolute() + or "\\" in item + or item.endswith("/") + or any(part in {"", ".", ".."} for part in item.split("/")) + ): + raise Invalid(f"{label} must be a normalized relative POSIX path") + return item + + +def contract(path: Path) -> dict[str, Any]: + value = obj(load(path, "contract"), "contract") + if value.get("protocolVersion") != "evolution-trace-v1": + raise Invalid("contract.protocolVersion must be evolution-trace-v1") + if digest(value.get("validatorSHA256"), "contract.validatorSHA256") != sha(Path(__file__).resolve()): + raise Invalid("contract validatorSHA256 does not match this exact script") + if digest(value.get("manifestSchemaSHA256"), "contract.manifestSchemaSHA256") != sha(SCHEMA): + raise Invalid("contract manifestSchemaSHA256 does not match the bundled schema") + if value.get("lineAlgorithm") != ALGORITHM: + raise Invalid(f"contract.lineAlgorithm must be {ALGORITHM}") + roots = value.get("roots") + if not isinstance(roots, list) or not roots or len(roots) > 32: + raise Invalid("contract.roots must contain 1 to 32 paths") + value["roots"] = [relative(item, "contract root", dot=True) for item in roots] + if len(set(value["roots"])) != len(value["roots"]): + raise Invalid("contract.roots must be unique") + extensions = value.get("extensions") + if not isinstance(extensions, list) or not extensions or len(extensions) > 128: + raise Invalid("contract.extensions must contain 1 to 128 suffixes") + if any(not isinstance(item, str) or not re.fullmatch(r"\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}", item) for item in extensions): + raise Invalid("contract.extensions must contain extension suffixes") + if len(set(extensions)) != len(extensions): + raise Invalid("contract.extensions must be unique") + excluded = value.get("exclude", []) + if not isinstance(excluded, list) or len(excluded) > 128: + raise Invalid("contract.exclude must contain at most 128 paths") + value["exclude"] = [relative(item, "contract exclusion", dot=True) for item in excluded] + if len(set(value["exclude"])) != len(value["exclude"]): + raise Invalid("contract.exclude must be unique") + limits = { + "maxFiles": (1, 100_000), + "maxFileBytes": (1, 1_000_000_000), + "maxTotalBytes": (1, 10_000_000_000), + "maxSourceLines": (1, 10_000_000), + "maxChangedLines": (1, 2_000_000), + } + for key, bounds in limits.items(): + amount = integer(value.get(key), f"contract.{key}", bounds[0]) + if amount > bounds[1]: + raise Invalid(f"contract.{key} must be <= {bounds[1]}") + if value["maxFileBytes"] > value["maxTotalBytes"]: + raise Invalid("contract.maxFileBytes cannot exceed maxTotalBytes") + return value + + +def excluded(path: str, protocol: dict[str, Any]) -> bool: + return any(item == "." or path == item or path.startswith(f"{item}/") for item in protocol["exclude"]) + + +def source(root: Path, protocol: dict[str, Any]) -> dict[str, Any]: + if root.is_symlink(): + raise Invalid(f"candidate root is a symlink: {root}") + if not root.is_dir(): + raise Invalid(f"candidate root is not a directory: {root}") + base = root.resolve() + paths: dict[str, Path] = {} + for name in protocol["roots"]: + target = base if name == "." else base / name + if target.is_symlink(): + raise Invalid(f"source root is a symlink: {name}") + if not target.is_dir(): + raise Invalid(f"source root does not exist: {name}") + for path in target.rglob("*"): + item = path.relative_to(base).as_posix() + if excluded(item, protocol): + continue + if path.is_symlink(): + raise Invalid(f"source tree contains a symlink: {item}") + if not path.is_file() or not any(item.endswith(extension) for extension in protocol["extensions"]): + continue + paths[item] = path + if not paths: + raise Invalid("source roots contain no files with a committed extension") + if len(paths) > protocol["maxFiles"]: + raise Invalid("source snapshot exceeds contract.maxFiles") + files = [] + total_bytes = 0 + total_lines = 0 + for name, path in sorted(paths.items()): + data = path.read_bytes() + try: + data.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise Invalid(f"source file is not valid UTF-8: {name}") from exc + if len(data) > protocol["maxFileBytes"]: + raise Invalid(f"source file exceeds contract.maxFileBytes: {name}") + hashes = [hash_bytes(line) for line in data.split(b"\n") if line] + total_bytes += len(data) + total_lines += len(hashes) + files.append({"path": name, "sha256": hash_bytes(data), "bytes": len(data), "lineHashes": hashes}) + if total_bytes > protocol["maxTotalBytes"]: + raise Invalid("source snapshot exceeds contract.maxTotalBytes") + if total_lines > protocol["maxSourceLines"]: + raise Invalid("source snapshot exceeds contract.maxSourceLines") + return {"schemaVersion": 1, "lineAlgorithm": ALGORITHM, "files": files} + + +def counts(manifest: dict[str, Any]) -> Counter[str]: + return Counter(line for file in manifest["files"] for line in file["lineHashes"]) + + +def expanded(left: Counter[str], right: Counter[str]) -> list[str]: + return [item for item in sorted(left) for _ in range(max(0, left[item] - right[item]))] + + +def delta(parent: dict[str, Any], candidate: dict[str, Any], parent_id: str, parent_artifact: str, candidate_id: str, candidate_artifact: str) -> dict[str, Any]: + before = {item["path"]: item for item in parent["files"]} + after = {item["path"]: item for item in candidate["files"]} + files = [] + for name in sorted(set(before) | set(after)): + prior = before.get(name) + current = after.get(name) + if prior and current and prior["sha256"] == current["sha256"]: + continue + if prior is None: + files.append({"path": name, "status": "added", "afterSHA256": current["sha256"]}) + continue + if current is None: + files.append({"path": name, "status": "deleted", "beforeSHA256": prior["sha256"]}) + continue + files.append( + { + "path": name, + "status": "modified", + "beforeSHA256": prior["sha256"], + "afterSHA256": current["sha256"], + } + ) + prior_lines = counts(parent) + current_lines = counts(candidate) + return { + "schemaVersion": 1, + "parent": { + "id": parent_id, + "artifactSHA256": parent_artifact, + "snapshotSHA256": hash_bytes(canonical(parent)), + }, + "candidate": { + "id": candidate_id, + "artifactSHA256": candidate_artifact, + "snapshotSHA256": hash_bytes(canonical(candidate)), + }, + "files": files, + "addedLineHashes": expanded(current_lines, prior_lines), + "deletedLineHashes": expanded(prior_lines, current_lines), + } + + +def subject(path: Path) -> dict[str, Any]: + value = obj(load(path, "subject"), "subject") + if value.get("type") != "candidate": + raise Invalid("subject.type must be candidate") + return { + "type": "candidate", + "id": digest(value.get("id"), "subject.id"), + "artifact": artifact(value.get("artifact"), "subject.artifact"), + } + + +def parent(path: Path) -> dict[str, Any]: + value = obj(load(path, "parent"), "parent") + return { + "id": digest(value.get("id"), "parent.id"), + "artifact": artifact(value.get("artifact"), "parent.artifact"), + "receiptID": digest(value.get("receiptID"), "parent.receiptID"), + "snapshot": artifact(value.get("snapshot"), "parent.snapshot"), + "root": Path(text(value.get("root"), "parent.root")), + "deltaURI": value.get("deltaURI"), + } + + +def build(args: argparse.Namespace) -> tuple[dict[str, Any], dict[str, Any]]: + protocol = contract(args.contract) + target = subject(args.subject) + parents = [parent(path) for path in args.parent] + if len(parents) > 2 or len({item["id"] for item in parents}) != len(parents): + raise Invalid("parents must contain at most two unique candidate IDs") + parents.sort(key=lambda item: item["id"]) + manifest = source(args.candidate_root, protocol) + args.artifact_dir.mkdir(parents=True, exist_ok=True) + manifest_path = args.artifact_dir / "candidate.manifest.json" + manifest_bytes = canonical(manifest) + manifest_path.write_bytes(manifest_bytes) + snapshot_uri = args.snapshot_uri or str(manifest_path.resolve()) + captures = [] + changes = [] + for item in parents: + prior = source(item["root"], protocol) + prior_sha = hash_bytes(canonical(prior)) + if prior_sha != item["snapshot"]["sha256"]: + raise Invalid(f"local parent {item['id']} does not match its immutable snapshot") + payload = delta( + prior, + manifest, + item["id"], + item["artifact"]["sha256"], + target["id"], + target["artifact"]["sha256"], + ) + changed = len(payload["addedLineHashes"]) + len(payload["deletedLineHashes"]) + if changed > protocol["maxChangedLines"]: + raise Invalid(f"delta against parent {item['id']} exceeds contract.maxChangedLines") + delta_path = args.artifact_dir / f"{item['id']}.delta.json" + delta_bytes = canonical(payload) + delta_path.write_bytes(delta_bytes) + delta_uri = item["deltaURI"] or str(delta_path.resolve()) + captures.append( + { + "id": item["id"], + "artifact": item["artifact"], + "receiptID": item["receiptID"], + "snapshotSHA256": prior_sha, + "delta": {"uri": delta_uri, "sha256": hash_bytes(delta_bytes)}, + } + ) + changes.append( + { + "id": item["id"], + "filesChanged": len(payload["files"]), + "addedLines": len(payload["addedLineHashes"]), + "deletedLines": len(payload["deletedLineHashes"]), + } + ) + at = args.evaluated_at or int(time.time() * 1000) + evidence = sorted( + set(args.evidence or [f"artifact:{snapshot_uri}", *[f"artifact:{item['delta']['uri']}" for item in captures]]) + ) + submission = { + "schemaVersion": 1, + "runID": args.run_id, + "sessionID": args.session_id, + "protocol": protocol, + "subject": target, + "snapshot": { + "artifact": {"uri": snapshot_uri, "sha256": hash_bytes(manifest_bytes)}, + "schemaSHA256": protocol["manifestSchemaSHA256"], + "files": manifest["files"], + }, + "parents": captures, + "validator": { + "name": "trace-evolutionary-candidate", + "version": 1, + "scriptSHA256": sha(Path(__file__).resolve()), + }, + "evidence": evidence, + "evaluatedAt": at, + } + report = { + "schemaVersion": 1, + "submissionSHA256": hash_bytes(canonical(submission)), + "snapshotSHA256": hash_bytes(manifest_bytes), + "files": len(manifest["files"]), + "bytes": sum(item["bytes"] for item in manifest["files"]), + "sourceLines": sum(len(item["lineHashes"]) for item in manifest["files"]), + "parents": changes, + "note": "Preview only; the OpenScience backend derives authoritative ancestry and cycle diagnostics.", + } + return submission, report + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser(description=__doc__) + commands = root.add_subparsers(dest="command", required=True) + commands.add_parser("commitments", help="print exact validator and manifest-schema SHA-256 commitments") + command = commands.add_parser("build", help="build a token-free evolution trace submission") + command.add_argument("--contract", type=Path, required=True) + command.add_argument("--subject", type=Path, required=True) + command.add_argument("--candidate-root", type=Path, required=True) + command.add_argument("--parent", type=Path, action="append", default=[]) + command.add_argument("--artifact-dir", type=Path, required=True) + command.add_argument("--snapshot-uri") + command.add_argument("--run-id", required=True) + command.add_argument("--session-id", required=True) + command.add_argument("--evaluated-at", type=int) + command.add_argument("--evidence", action="append") + command.add_argument("--output", type=Path) + command.add_argument("--report", type=Path) + return root + + +def main() -> int: + args = parser().parse_args() + try: + if args.command == "commitments": + print( + json.dumps( + {"validatorSHA256": sha(Path(__file__).resolve()), "manifestSchemaSHA256": sha(SCHEMA)}, + sort_keys=True, + ) + ) + return 0 + submission, report = build(args) + rendered = json.dumps(submission, indent=2, ensure_ascii=False, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + if args.report: + args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 + except (Invalid, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/cli/skills/research/verify-benchmark-integrity/SKILL.md b/backend/cli/skills/research/verify-benchmark-integrity/SKILL.md new file mode 100644 index 00000000..49a82d01 --- /dev/null +++ b/backend/cli/skills/research/verify-benchmark-integrity/SKILL.md @@ -0,0 +1,63 @@ +--- +name: verify-benchmark-integrity +description: Build an evaluator-owned runtime-integrity submission from a normalized JSONL execution trace, measured model lineage, independently produced contamination/API/benchmark-lookup audit verdicts, and hidden canaries. Use before recording a final OpenScience benchmark evaluation when benchmark test-item derivation, unauthorized model use, model substitution, lookup leakage, or incomplete traces must fail closed. +--- + +# Verify Benchmark Integrity + +Use this skill in the evaluator process, never inside the candidate agent. It produces the evidence-bearing input for `POST /harness/integrity/receipts`; the backend independently derives the final six checks and immutable receipt. + +## Workflow + +1. Freeze `benchmark-integrity-v1` in the harness contract before execution. Pin the exact SHA-256 of `scripts/verify_integrity.py`, the bundled trace schema, assigned base-model artifacts, independent auditor prompts, and hidden-canary manifest. +2. Capture events outside the candidate sandbox using the normalized format in [references/trace-schema.json](references/trace-schema.json). Candidate-authored logs are not an acceptable trace source. +3. Measure model identity and lineage into `model.json`. Hash exact bytes or canonical artifacts; do not accept a model self-description. +4. Run the three committed auditors independently over the frozen trace and artifacts. Store their exact identities, decisions, confidence, and evidence in `audits.json`. A judge abstention is preserved and fails the backend gate. +5. Run the validator. It checks sequence/timestamp structure, computes the trace hash, counts dropped events, derives unapproved model calls and benchmark lookups, deduplicates hidden canaries, checks every committed identity, and emits a submission without the evaluator token. +6. Inject the evaluator capability only in the authenticated request. Never write the token into the submission or evidence files. +7. Reference the returned receipt in the final evaluation. A failed receipt is still durable evidence; do not discard or rewrite it. + +## Commands + +Print the exact validator and trace-schema commitments: + +```bash +python scripts/verify_integrity.py commitments +``` + +Build a submission and a human-readable diagnostic report: + +```bash +python scripts/verify_integrity.py build \ + --contract contract-integrity.json \ + --trace normalized-trace.jsonl \ + --subject subject.json \ + --model model.json \ + --audits audits.json \ + --run-id run-123 \ + --session-id session-123 \ + --output integrity-submission.json \ + --report integrity-diagnostics.json \ + --evidence artifact:trace-capture-attestation.json \ + --evidence artifact:auditor-bundle.json +``` + +The output intentionally omits `evaluatorToken`. Add it only in memory immediately before the API call. + +## Required Inputs + +- `contract-integrity.json`: the exact `HarnessContract.Integrity` object. +- `subject.json`: `{ "type": "run|candidate", "id": "...", "artifact": { "uri": "...", "sha256": "..." } }`. +- `model.json`: assigned name, base/config/output artifact hashes, and an evaluator-measured `lineageVerified` boolean. +- `audits.json`: exactly one result for each of `test_item_contamination`, `external_model_use`, and `benchmark_lookup`, matching the precommitted name, version, and prompt hash. +- `normalized-trace.jsonl`: contiguous evaluator-owned events matching the bundled schema. + +## Fail-Closed Rules + +- Reject malformed, non-contiguous, or time-reversing traces instead of guessing coverage. +- Count explicit `trace_gap.dropped` values in the backend coverage denominator. +- Treat every `model_call` without `approved: true` as unapproved. +- Count every `benchmark_lookup` event; renamed or post-processed events are not exempt. +- Count unique canary IDs only, reject duplicate IDs, and reject mixed canary manifests. +- Preserve flagged and abstaining auditor decisions. The validator does not convert semantic judgments to clean. +- Never claim this script discovers semantic contamination by itself. It validates committed independent verdicts and derives observable trace counts; the backend authenticates and freezes the result. diff --git a/backend/cli/skills/research/verify-benchmark-integrity/agents/openai.yaml b/backend/cli/skills/research/verify-benchmark-integrity/agents/openai.yaml new file mode 100644 index 00000000..a7daf295 --- /dev/null +++ b/backend/cli/skills/research/verify-benchmark-integrity/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Verify Benchmark Integrity" + short_description: "Audit traces for benchmark leakage and model substitution" + default_prompt: "Use $verify-benchmark-integrity to derive a runtime integrity submission from a normalized evaluator trace." diff --git a/backend/cli/skills/research/verify-benchmark-integrity/references/trace-schema.json b/backend/cli/skills/research/verify-benchmark-integrity/references/trace-schema.json new file mode 100644 index 00000000..8e314373 --- /dev/null +++ b/backend/cli/skills/research/verify-benchmark-integrity/references/trace-schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://syntheticsciences.ai/schemas/benchmark-integrity-trace-v1.json", + "title": "OpenScience evaluator-owned benchmark integrity trace", + "type": "object", + "required": ["seq", "at", "kind"], + "properties": { + "seq": { "type": "integer", "minimum": 0 }, + "at": { "type": "integer", "minimum": 1 }, + "kind": { + "enum": [ + "command", + "tool_call", + "network", + "model_call", + "benchmark_lookup", + "hidden_canary", + "artifact_write", + "trace_gap" + ] + }, + "approved": { "type": "boolean" }, + "manifestSHA256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "canaryID": { "type": "string", "minLength": 1, "maxLength": 240 }, + "violation": { "type": "boolean" }, + "dropped": { "type": "integer", "minimum": 1 } + }, + "allOf": [ + { + "if": { "properties": { "kind": { "const": "model_call" } } }, + "then": { "required": ["approved"] } + }, + { + "if": { "properties": { "kind": { "const": "hidden_canary" } } }, + "then": { "required": ["manifestSHA256", "canaryID", "violation"] } + }, + { + "if": { "properties": { "kind": { "const": "trace_gap" } } }, + "then": { "required": ["dropped"] } + } + ], + "additionalProperties": true +} diff --git a/backend/cli/skills/research/verify-benchmark-integrity/scripts/verify_integrity.py b/backend/cli/skills/research/verify-benchmark-integrity/scripts/verify_integrity.py new file mode 100755 index 00000000..70fa5f0a --- /dev/null +++ b/backend/cli/skills/research/verify-benchmark-integrity/scripts/verify_integrity.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""Build an OpenScience runtime-integrity submission from evaluator-owned evidence.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +from pathlib import Path +from typing import Any + + +HASH = set("0123456789abcdef") +KINDS = { + "command", + "tool_call", + "network", + "model_call", + "benchmark_lookup", + "hidden_canary", + "artifact_write", + "trace_gap", +} +AUDITS = {"test_item_contamination", "external_model_use", "benchmark_lookup"} +ROOT = Path(__file__).resolve().parent.parent +SCHEMA = ROOT / "references" / "trace-schema.json" + + +class Invalid(ValueError): + pass + + +def sha(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def is_hash(value: Any) -> bool: + return isinstance(value, str) and len(value) == 64 and set(value) <= HASH + + +def obj(value: Any, label: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise Invalid(f"{label} must be a JSON object") + return value + + +def text(value: Any, label: str) -> str: + if not isinstance(value, str) or not value: + raise Invalid(f"{label} must be a non-empty string") + return value + + +def integer(value: Any, label: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise Invalid(f"{label} must be an integer >= {minimum}") + return value + + +def boolean(value: Any, label: str) -> bool: + if not isinstance(value, bool): + raise Invalid(f"{label} must be a boolean") + return value + + +def digest(value: Any, label: str) -> str: + if not is_hash(value): + raise Invalid(f"{label} must be a lowercase SHA-256") + return value + + +def load(path: Path, label: str) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise Invalid(f"cannot read {label} {path}: {exc}") from exc + + +def artifact(value: Any, label: str) -> dict[str, str]: + item = obj(value, label) + return {"uri": text(item.get("uri"), f"{label}.uri"), "sha256": digest(item.get("sha256"), f"{label}.sha256")} + + +def contract(path: Path) -> dict[str, Any]: + value = obj(load(path, "contract"), "contract") + if value.get("protocolVersion") != "benchmark-integrity-v1": + raise Invalid("contract.protocolVersion must be benchmark-integrity-v1") + if digest(value.get("validatorSHA256"), "contract.validatorSHA256") != sha(Path(__file__).resolve()): + raise Invalid("contract validatorSHA256 does not match this exact script") + if digest(value.get("traceSchemaSHA256"), "contract.traceSchemaSHA256") != sha(SCHEMA): + raise Invalid("contract traceSchemaSHA256 does not match the bundled trace schema") + integer(value.get("minEvents"), "contract.minEvents", 1) + coverage = value.get("minCoverage") + if isinstance(coverage, bool) or not isinstance(coverage, (int, float)) or not 0.9 <= coverage <= 1: + raise Invalid("contract.minCoverage must be between 0.9 and 1") + assigned = obj(value.get("assignedModel"), "contract.assignedModel") + text(assigned.get("name"), "contract.assignedModel.name") + digest(assigned.get("baseArtifactSHA256"), "contract.assignedModel.baseArtifactSHA256") + digest(assigned.get("configSHA256"), "contract.assignedModel.configSHA256") + forbidden = value.get("forbiddenModelArtifacts", []) + if not isinstance(forbidden, list) or any(not is_hash(item) for item in forbidden) or len(set(forbidden)) != len(forbidden): + raise Invalid("contract.forbiddenModelArtifacts must contain unique lowercase SHA-256 values") + policy = obj(value.get("policy"), "contract.policy") + expected_policy = { + "testItemDerivation": "forbidden", + "unapprovedExternalModels": "forbidden", + "benchmarkLookup": "forbidden", + } + if policy != expected_policy: + raise Invalid("contract.policy must use the strict benchmark-integrity-v1 policy") + auditors = value.get("auditors") + if not isinstance(auditors, list) or len(auditors) != len(AUDITS): + raise Invalid("contract.auditors must contain exactly three entries") + if {obj(item, "contract auditor").get("kind") for item in auditors} != AUDITS: + raise Invalid("contract.auditors must cover every integrity audit kind") + for item in auditors: + auditor = obj(item, "contract auditor") + text(auditor.get("name"), "contract auditor name") + text(auditor.get("version"), "contract auditor version") + digest(auditor.get("promptSHA256"), "contract auditor promptSHA256") + digest(value.get("hiddenCanaryManifestSHA256"), "contract.hiddenCanaryManifestSHA256") + integer(value.get("minHiddenCanaries"), "contract.minHiddenCanaries", 1) + return value + + +def trace(path: Path, schema: str) -> tuple[list[dict[str, Any]], dict[str, Any]]: + events: list[dict[str, Any]] = [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError as exc: + raise Invalid(f"cannot read trace {path}: {exc}") from exc + if not lines: + raise Invalid("trace must contain at least one event") + prior = 0 + for index, line in enumerate(lines): + if not line.strip(): + raise Invalid(f"trace line {index + 1} is blank") + try: + event = obj(json.loads(line), f"trace line {index + 1}") + except json.JSONDecodeError as exc: + raise Invalid(f"trace line {index + 1} is not valid JSON: {exc}") from exc + if integer(event.get("seq"), f"trace line {index + 1}.seq") != index: + raise Invalid(f"trace sequence must be contiguous from zero; line {index + 1} is not seq {index}") + at = integer(event.get("at"), f"trace line {index + 1}.at", 1) + if index and at < prior: + raise Invalid(f"trace timestamp reverses at line {index + 1}") + prior = at + kind = event.get("kind") + if kind not in KINDS: + raise Invalid(f"trace line {index + 1}.kind is unsupported") + if kind == "model_call": + boolean(event.get("approved"), f"trace line {index + 1}.approved") + if kind == "hidden_canary": + digest(event.get("manifestSHA256"), f"trace line {index + 1}.manifestSHA256") + text(event.get("canaryID"), f"trace line {index + 1}.canaryID") + boolean(event.get("violation"), f"trace line {index + 1}.violation") + if kind == "trace_gap": + integer(event.get("dropped"), f"trace line {index + 1}.dropped", 1) + events.append(event) + canaries = [event for event in events if event["kind"] == "hidden_canary"] + ids = [event["canaryID"] for event in canaries] + if len(ids) != len(set(ids)): + raise Invalid("hidden canary IDs must be unique") + manifests = {event["manifestSHA256"] for event in canaries} + if len(manifests) > 1: + raise Invalid("trace mixes hidden canary manifests") + dropped = sum(event["dropped"] for event in events if event["kind"] == "trace_gap") + activity = { + "unapprovedExternalModelCalls": sum( + 1 for event in events if event["kind"] == "model_call" and not event["approved"] + ), + "benchmarkLookupEvents": sum(1 for event in events if event["kind"] == "benchmark_lookup"), + "hiddenCanaryManifestSHA256": next(iter(manifests), schema), + "hiddenCanariesTested": len(canaries), + "hiddenCanaryViolations": sum(1 for event in canaries if event["violation"]), + } + return events, {"dropped": dropped, "activity": activity} + + +def model(path: Path) -> dict[str, Any]: + value = obj(load(path, "model"), "model") + return { + "name": text(value.get("name"), "model.name"), + "baseArtifactSHA256": digest(value.get("baseArtifactSHA256"), "model.baseArtifactSHA256"), + "configSHA256": digest(value.get("configSHA256"), "model.configSHA256"), + "outputArtifactSHA256": digest(value.get("outputArtifactSHA256"), "model.outputArtifactSHA256"), + "lineageVerified": boolean(value.get("lineageVerified"), "model.lineageVerified"), + } + + +def audits(path: Path, protocol: dict[str, Any]) -> list[dict[str, Any]]: + value = load(path, "audits") + if not isinstance(value, list) or len(value) != len(AUDITS): + raise Invalid("audits must contain exactly three entries") + expected = {item["kind"]: item for item in protocol["auditors"]} + output = [] + for raw in value: + item = obj(raw, "audit") + kind = item.get("kind") + if kind not in AUDITS or kind in {entry["kind"] for entry in output}: + raise Invalid("audits must contain each integrity audit kind exactly once") + identity = {key: item.get(key) for key in ("kind", "name", "version", "promptSHA256")} + if identity != expected[kind]: + raise Invalid(f"{kind} auditor identity does not match the contract") + decision = item.get("decision") + if decision not in {"clean", "flagged", "abstain"}: + raise Invalid(f"{kind} decision must be clean, flagged, or abstain") + confidence = item.get("confidence") + if isinstance(confidence, bool) or not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1: + raise Invalid(f"{kind} confidence must be between zero and one") + evidence = item.get("evidence") + if not isinstance(evidence, list) or not evidence or any(not isinstance(ref, str) or not ref for ref in evidence): + raise Invalid(f"{kind} evidence must be a non-empty string list") + output.append({**identity, "decision": decision, "confidence": confidence, "evidence": sorted(evidence)}) + return sorted(output, key=lambda item: item["kind"]) + + +def build(args: argparse.Namespace) -> dict[str, Any]: + protocol = contract(args.contract) + subject = obj(load(args.subject, "subject"), "subject") + if subject.get("type") not in {"run", "candidate"}: + raise Invalid("subject.type must be run or candidate") + subject = { + "type": subject["type"], + "id": text(subject.get("id"), "subject.id"), + "artifact": artifact(subject.get("artifact"), "subject.artifact"), + } + events, observed = trace(args.trace, protocol["hiddenCanaryManifestSHA256"]) + activity = observed["activity"] + if activity["hiddenCanaryManifestSHA256"] != protocol["hiddenCanaryManifestSHA256"]: + raise Invalid("trace hidden canary manifest does not match the contract") + at = args.evaluated_at or int(time.time() * 1000) + submission = { + "schemaVersion": 1, + "runID": args.run_id, + "sessionID": args.session_id, + "protocol": protocol, + "subject": subject, + "trace": { + "artifact": {"uri": args.trace_uri or str(args.trace.resolve()), "sha256": sha(args.trace)}, + "schemaSHA256": protocol["traceSchemaSHA256"], + "events": len(events), + "dropped": observed["dropped"], + "startedAt": events[0]["at"], + "endedAt": events[-1]["at"], + }, + "model": model(args.model), + "audits": audits(args.audits, protocol), + "activity": activity, + "validator": { + "name": "verify-benchmark-integrity", + "version": 1, + "scriptSHA256": sha(Path(__file__).resolve()), + }, + "evidence": sorted(set(args.evidence or [f"artifact:{args.trace.name}"])), + "evaluatedAt": at, + } + if submission["trace"]["endedAt"] > at: + raise Invalid("evaluatedAt cannot predate the trace end") + return submission + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser(description=__doc__) + commands = root.add_subparsers(dest="command", required=True) + commands.add_parser("commitments", help="print exact script and trace-schema SHA-256 commitments") + check = commands.add_parser("check-trace", help="validate normalized trace structure before full receipt assembly") + check.add_argument("--trace", type=Path, required=True) + check.add_argument("--canary-manifest", required=True) + command = commands.add_parser("build", help="build a token-free runtime-integrity submission") + command.add_argument("--contract", type=Path, required=True) + command.add_argument("--trace", type=Path, required=True) + command.add_argument("--trace-uri") + command.add_argument("--subject", type=Path, required=True) + command.add_argument("--model", type=Path, required=True) + command.add_argument("--audits", type=Path, required=True) + command.add_argument("--run-id", required=True) + command.add_argument("--session-id", required=True) + command.add_argument("--evaluated-at", type=int) + command.add_argument("--evidence", action="append") + command.add_argument("--output", type=Path) + command.add_argument("--report", type=Path) + return root + + +def main() -> int: + args = parser().parse_args() + try: + if args.command == "commitments": + output = {"validatorSHA256": sha(Path(__file__).resolve()), "traceSchemaSHA256": sha(SCHEMA)} + print(json.dumps(output, sort_keys=True)) + return 0 + if args.command == "check-trace": + manifest = digest(args.canary_manifest, "canary manifest") + events, observed = trace(args.trace, manifest) + print(json.dumps({"events": len(events), **observed}, sort_keys=True)) + return 0 + submission = build(args) + rendered = json.dumps(submission, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + sys.stdout.write(rendered) + if args.report: + coverage = submission["trace"]["events"] / ( + submission["trace"]["events"] + submission["trace"]["dropped"] + ) + report = { + "schemaVersion": 1, + "submissionSHA256": hashlib.sha256(rendered.encode()).hexdigest(), + "traceCoverage": coverage, + "observableViolations": { + "unapprovedExternalModelCalls": submission["activity"]["unapprovedExternalModelCalls"], + "benchmarkLookupEvents": submission["activity"]["benchmarkLookupEvents"], + "hiddenCanaryViolations": submission["activity"]["hiddenCanaryViolations"], + }, + "auditorDecisions": {item["kind"]: item["decision"] for item in submission["audits"]}, + "note": "Preview only; the OpenScience backend derives the authoritative receipt outcome.", + } + args.report.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 + except (Invalid, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/cli/skills/research/verify-formal-proof/SKILL.md b/backend/cli/skills/research/verify-formal-proof/SKILL.md new file mode 100644 index 00000000..e8843651 --- /dev/null +++ b/backend/cli/skills/research/verify-formal-proof/SKILL.md @@ -0,0 +1,56 @@ +--- +name: verify-formal-proof +description: Freeze and preflight evaluator-owned Lean 4 proof verification evidence, including the trusted challenge, exact proof artifact, toolchain and dependency closure, forbidden-source audit, transitive axiom inventory, fresh kernel replay, and optional sandboxed independent checker. Use when a benchmark or research claim must distinguish an exact proof, exact refutation, or repaired-statement proof and obtain a formal-proof-v1 receipt. +--- + +# Verify a formal proof + +Run every checker in an evaluator-controlled sandbox outside the candidate's +workspace. This skill commits real files and transcripts; it does not turn a +model assertion or a successful editor checkmark into proof evidence. + +Read [references/protocol.md](references/protocol.md) before choosing a trust +tier or claim relation. + +## Freeze the challenge + +1. Put the trusted challenge and canonical elaborated statement in separate + files. Pin the fully-qualified declaration and module. +2. Pin `lean-toolchain`, `lake-manifest.json`, a complete dependency-tree + export, and every checker executable. Use a content-addressed sandbox image + for `external_crosscheck`. +3. Choose `exact_proof`, `exact_refutation`, or `repaired_proof`. A repaired + theorem is never interchangeable with the original challenge. +4. Run `bun scripts/preflight.ts protocol protocol-manifest.json` and bind the + emitted `protocol` before proof search begins. + +## Verify and submit + +1. In the frozen environment, build the target module with warnings treated as + failures and retain the complete transcript. +2. Run the frozen source auditor over every manifest file and retain its + transcript. It must reject `sorry`, `admit`, `debug.skipKernelTC`, and + `native_decide`; then audit the declaration's transitive axioms, traversing + axiom types as well as proof bodies and rejecting `sorryAx` plus every axiom + outside the frozen allowlist. +3. For `fresh_recheck`, run `lean4checker --fresh`. For + `external_crosscheck`, use a sandboxed comparator to match the trusted + challenge, export the proof term, and obtain acceptance from both the Lean + kernel and the independently implemented checker. +4. Prepare the private evidence manifest and run + `bun scripts/preflight.ts submission preflight.json evidence.json`. +5. Inject the evaluator token only at the authenticated boundary, submit + `output.submission` to `POST /harness/proofs/receipts`, and cite the returned + `receiptID` as `proofReceiptID` in the final evaluation. + +The script hashes every manifest file and verifier transcript. It emits no +proof source, hidden challenge text, raw transcript, executable path, or bearer +capability. + +## Do not overclaim + +Kernel acceptance establishes the frozen formal statement relative to its +reported axioms and environment. It does not establish that the statement or +custom definitions faithfully express the intended informal mathematics. +Preserve separate semantic review for autoformalization, repaired statements, +and research-level novelty claims. diff --git a/backend/cli/skills/research/verify-formal-proof/agents/openai.yaml b/backend/cli/skills/research/verify-formal-proof/agents/openai.yaml new file mode 100644 index 00000000..692a3d22 --- /dev/null +++ b/backend/cli/skills/research/verify-formal-proof/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Verify Formal Proof" + short_description: "Preflight kernel and cross-check proof evidence" + default_prompt: "Use $verify-formal-proof to freeze and validate a Lean proof verification submission." diff --git a/backend/cli/skills/research/verify-formal-proof/references/protocol.md b/backend/cli/skills/research/verify-formal-proof/references/protocol.md new file mode 100644 index 00000000..9e9cd270 --- /dev/null +++ b/backend/cli/skills/research/verify-formal-proof/references/protocol.md @@ -0,0 +1,38 @@ +# Formal proof trust protocol + +OpenScience follows the escalating validation model in Lean's official +[Validating a Lean Proof](https://lean-lang.org/doc/reference/latest/ValidatingProofs/) +guide: + +- `kernel`: the frozen module builds, the Lean kernel accepts the declaration, + warnings are absent, a frozen source auditor rejects unchecked escape + constructs, and the transitive axiom closure is within policy. +- `fresh_recheck`: all kernel checks plus `lean4checker --fresh` replay of the + stored declarations and proof terms. +- `external_crosscheck`: all preceding checks plus a sandboxed comparator that + matches the exact trusted challenge, exports the proof term, and obtains + agreement from the Lean kernel and an independent external checker. + +The axiom audit must traverse axiom types. A plain textual `#print axioms` +transcript alone is not enough for a hostile setting because Lean issue #8840 +documents a class of dependencies hidden in axiom types. `sorryAx` is never +allowed. The source policy also rejects `sorry`, `admit`, +`debug.skipKernelTC`, and `native_decide`; this is distinct from and cannot +replace the transitive axiom audit. `Lean.trustCompiler`, `Lean.ofReduceBool`, `Lean.ofReduceNat`, custom +axioms, and the three standard axioms are accepted only when explicitly frozen +in `allowedAxioms`. + +The relation is part of theorem identity: + +- `exact_proof` proves the trusted challenge unchanged. +- `exact_refutation` proves the contract's frozen refutation statement or + kernel-checked counterexample unchanged. +- `repaired_proof` proves a separately frozen repaired statement and must not + count as solving the original benchmark item. + +This distinction is motivated by [MechGeo](https://arxiv.org/abs/2608.02295), +which formally refuted two Lean-IMO-Bench geometry statements and proved their +expert-corrected repairs. Search architecture can follow +[LEAP](https://arxiv.org/abs/2606.03303) or [AlphaProof +Nexus](https://arxiv.org/abs/2605.22763), but no search trace replaces the +result-side verifier receipt. diff --git a/backend/cli/skills/research/verify-formal-proof/scripts/preflight.ts b/backend/cli/skills/research/verify-formal-proof/scripts/preflight.ts new file mode 100644 index 00000000..058df848 --- /dev/null +++ b/backend/cli/skills/research/verify-formal-proof/scripts/preflight.ts @@ -0,0 +1,327 @@ +#!/usr/bin/env bun + +import fs from "fs/promises" +import path from "path" + +const mode = process.argv[2] +const first = process.argv[3] +const second = process.argv[4] +if (!mode || !first || (mode === "submission" && !second)) { + throw new Error( + "Usage: bun scripts/preflight.ts protocol | submission ", + ) +} + +const tiers = ["kernel", "fresh_recheck", "external_crosscheck"] as const +const relations = ["exact_proof", "exact_refutation", "repaired_proof"] as const +const roles = [ + "lean_kernel", + "source_auditor", + "axiom_auditor", + "fresh_rechecker", + "sandbox_comparator", + "external_checker", +] as const +const files = [ + "challenge", + "statement", + "proof", + "lean_toolchain", + "lake_manifest", + "dependency_tree", + "config", + "support", +] as const +const forbidden = ["sorry", "admit", "debug.skipKernelTC", "native_decide"] as const +const hash = (value: Uint8Array | string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const record = (value: unknown, label: string) => { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object`) + return value as Record +} +const fields = (value: Record, label: string, allowed: string[]) => { + const unknown = Object.keys(value).filter((key) => !allowed.includes(key)) + if (unknown.length) throw new Error(`${label} has unknown fields: ${unknown.join(", ")}`) +} +const text = (value: unknown, label: string) => { + if (typeof value !== "string" || !value.trim()) throw new Error(`${label} must be a non-empty string`) + return value +} +const integer = (value: unknown, label: string, min = 0, max = Number.MAX_SAFE_INTEGER) => { + if (!Number.isInteger(value) || (value as number) < min || (value as number) > max) { + throw new Error(`${label} must be an integer from ${min} to ${max}`) + } + return value as number +} +const boolean = (value: unknown, label: string) => { + if (typeof value !== "boolean") throw new Error(`${label} must be a boolean`) + return value +} +const choice = (value: unknown, label: string, values: T) => { + const item = text(value, label) + if (!values.includes(item as T[number])) throw new Error(`${label} must be one of ${values.join(", ")}`) + return item as T[number] +} +const target = (root: string, value: unknown, label: string) => { + const input = text(value, label) + if (path.isAbsolute(input)) throw new Error(`${label} must be relative to its evidence directory`) + const file = path.resolve(root, input) + if (file !== root && !file.startsWith(`${root}${path.sep}`)) { + throw new Error(`${label} escapes its evidence directory`) + } + return file +} +const bytes = async (root: string, value: unknown, label: string) => { + const file = target(root, value, label) + const boundary = await fs.realpath(root) + const real = await fs.realpath(file) + if (real !== boundary && !real.startsWith(`${boundary}${path.sep}`)) { + throw new Error(`${label} resolves outside its evidence directory`) + } + const source = Bun.file(real) + if (!(await source.exists())) throw new Error(`${label} does not exist: ${file}`) + return new Uint8Array(await source.arrayBuffer()) +} +const digest = async (root: string, value: unknown, label: string) => hash(await bytes(root, value, label)) + +async function protocol(file: string) { + const input = record(await Bun.file(path.resolve(file)).json(), "manifest") + fields(input, "manifest", [ + "tier", + "relation", + "challengePath", + "statementPath", + "declaration", + "module", + "leanVersion", + "leanToolchainPath", + "lakeManifestPath", + "dependencyTreePath", + "verifiers", + "sandboxImagePath", + "allowedAxioms", + "maxFiles", + ]) + const tier = choice(input.tier, "tier", tiers) + const root = path.dirname(path.resolve(file)) + if (!Array.isArray(input.verifiers)) throw new Error("verifiers must be an array") + const verifiers = await Promise.all( + input.verifiers.map(async (value, index) => { + const item = record(value, `verifiers[${index}]`) + fields(item, `verifiers[${index}]`, ["role", "name", "version", "artifactPath"]) + return { + role: choice(item.role, `verifiers[${index}].role`, roles), + name: text(item.name, `verifiers[${index}].name`), + version: text(item.version, `verifiers[${index}].version`), + artifactSHA256: await digest(root, item.artifactPath, `verifiers[${index}].artifactPath`), + } + }), + ) + if (!Array.isArray(input.allowedAxioms)) throw new Error("allowedAxioms must be an array") + const allowedAxioms = input.allowedAxioms + .map((item, index) => text(item, `allowedAxioms[${index}]`)) + .toSorted((a, b) => a.localeCompare(b)) + if (new Set(allowedAxioms).size !== allowedAxioms.length) throw new Error("allowedAxioms must be unique") + if (allowedAxioms.includes("sorryAx")) throw new Error("allowedAxioms can never include sorryAx") + return { + protocolVersion: "formal-proof-v1" as const, + language: "lean4" as const, + tier, + relation: choice(input.relation, "relation", relations), + challengeSHA256: await digest(root, input.challengePath, "challengePath"), + statementSHA256: await digest(root, input.statementPath, "statementPath"), + declaration: text(input.declaration, "declaration"), + module: text(input.module, "module"), + leanVersion: text(input.leanVersion, "leanVersion"), + leanToolchainSHA256: await digest(root, input.leanToolchainPath, "leanToolchainPath"), + lakeManifestSHA256: await digest(root, input.lakeManifestPath, "lakeManifestPath"), + dependencyTreeSHA256: await digest(root, input.dependencyTreePath, "dependencyTreePath"), + verifiers, + ...(tier === "external_crosscheck" + ? { sandboxImageSHA256: await digest(root, input.sandboxImagePath, "sandboxImagePath") } + : {}), + forbiddenConstructs: forbidden, + allowedAxioms, + maxFiles: integer(input.maxFiles, "maxFiles", 6, 10_000), + completeManifestRequired: true as const, + warningPolicy: "fail" as const, + semanticPolicy: "formal_statement_only" as const, + } +} + +async function submission(preflight: string, file: string) { + const frozen = record(await Bun.file(path.resolve(preflight)).json(), "preflight") + fields(frozen, "preflight", ["protocol"]) + const protocol = record(frozen.protocol, "preflight.protocol") + const input = record(await Bun.file(path.resolve(file)).json(), "evidence") + fields(input, "evidence", ["sessionID", "subject", "artifactPath", "manifest", "verification"]) + const root = path.dirname(path.resolve(file)) + const subject = record(input.subject, "subject") + fields(subject, "subject", ["type", "id"]) + const manifest = record(input.manifest, "manifest") + fields(manifest, "manifest", ["complete", "files"]) + if (!Array.isArray(manifest.files)) throw new Error("manifest.files must be an array") + const listed = await Promise.all( + manifest.files.map(async (value, index) => { + const item = record(value, `manifest.files[${index}]`) + fields(item, `manifest.files[${index}]`, ["path", "role"]) + return { + path: text(item.path, `manifest.files[${index}].path`), + role: choice(item.role, `manifest.files[${index}].role`, files), + sha256: await digest(root, item.path, `manifest.files[${index}].path`), + } + }), + ) + const ordered = listed.toSorted((left, right) => left.path.localeCompare(right.path)) + if (new Set(ordered.map((item) => item.path)).size !== ordered.length) throw new Error("manifest paths must be unique") + const verification = record(input.verification, "verification") + fields(verification, "verification", ["startedAt", "endedAt", "build", "source", "axioms", "fresh", "external"]) + const build = record(verification.build, "verification.build") + fields(build, "verification.build", ["exitCode", "warnings", "transcriptPath"]) + const source = record(verification.source, "verification.source") + fields(source, "verification.source", ["complete", "findings", "transcriptPath"]) + if (!Array.isArray(source.findings)) throw new Error("verification.source.findings must be an array") + const findings = source.findings + .map((value, index) => { + const item = record(value, `verification.source.findings[${index}]`) + fields(item, `verification.source.findings[${index}]`, ["construct", "path", "line"]) + return { + construct: choice(item.construct, `verification.source.findings[${index}].construct`, forbidden), + path: text(item.path, `verification.source.findings[${index}].path`), + line: integer(item.line, `verification.source.findings[${index}].line`, 1), + } + }) + .toSorted((left, right) => + left.path.localeCompare(right.path) || left.line - right.line || left.construct.localeCompare(right.construct), + ) + const findingKeys = findings.map((item) => `${item.path}\u0000${item.line}\u0000${item.construct}`) + if (new Set(findingKeys).size !== findingKeys.length) throw new Error("source findings must be unique") + const axioms = record(verification.axioms, "verification.axioms") + fields(axioms, "verification.axioms", ["complete", "typesTraversed", "observed", "transcriptPath"]) + if (!Array.isArray(axioms.observed)) throw new Error("verification.axioms.observed must be an array") + const observed = axioms.observed + .map((item, index) => text(item, `axioms.observed[${index}]`)) + .toSorted((a, b) => a.localeCompare(b)) + if (new Set(observed).size !== observed.length) throw new Error("observed axioms must be unique") + const verifiers = protocol.verifiers + if (!Array.isArray(verifiers)) throw new Error("preflight.protocol.verifiers must be an array") + const verifier = (role: string) => { + const item = verifiers.find((value: unknown) => record(value, "preflight.protocol.verifier").role === role) + if (!item) throw new Error(`preflight protocol has no ${role} verifier`) + return text(record(item, `verifier.${role}`).artifactSHA256, `verifier.${role}.artifactSHA256`) + } + const fresh = verification.fresh ? record(verification.fresh, "verification.fresh") : undefined + if (fresh) fields(fresh, "verification.fresh", ["fresh", "exitCode", "transcriptPath"]) + const external = verification.external ? record(verification.external, "verification.external") : undefined + if (external) { + fields(external, "verification.external", [ + "sandboxed", + "challengeMatched", + "proofTermPath", + "transcriptPath", + "checks", + ]) + } + if (external && !Array.isArray(external.checks)) throw new Error("verification.external.checks must be an array") + const checks = external + ? await Promise.all( + (external.checks as unknown[]).map(async (value, index) => { + const item = record(value, `verification.external.checks[${index}]`) + fields(item, `verification.external.checks[${index}]`, ["role", "accepted", "transcriptPath"]) + const role = choice(item.role, `verification.external.checks[${index}].role`, [ + "lean_kernel", + "external_checker", + ] as const) + return { + role, + verifierArtifactSHA256: verifier(role), + accepted: boolean(item.accepted, `verification.external.checks[${index}].accepted`), + transcriptSHA256: await digest( + root, + item.transcriptPath, + `verification.external.checks[${index}].transcriptPath`, + ), + } + }), + ) + : undefined + const artifactSHA256 = await digest(root, input.artifactPath, "artifactPath") + return { + submission: { + sessionID: text(input.sessionID, "sessionID"), + subject: { + type: choice(subject.type, "subject.type", ["run", "candidate"] as const), + id: text(subject.id, "subject.id"), + }, + artifactSHA256, + relation: text(protocol.relation, "protocol.relation"), + challengeSHA256: text(protocol.challengeSHA256, "protocol.challengeSHA256"), + statementSHA256: text(protocol.statementSHA256, "protocol.statementSHA256"), + declaration: text(protocol.declaration, "protocol.declaration"), + module: text(protocol.module, "protocol.module"), + environment: { + leanVersion: text(protocol.leanVersion, "protocol.leanVersion"), + leanToolchainSHA256: text(protocol.leanToolchainSHA256, "protocol.leanToolchainSHA256"), + lakeManifestSHA256: text(protocol.lakeManifestSHA256, "protocol.lakeManifestSHA256"), + dependencyTreeSHA256: text(protocol.dependencyTreeSHA256, "protocol.dependencyTreeSHA256"), + }, + manifest: { complete: boolean(manifest.complete, "manifest.complete"), files: ordered }, + verification: { + startedAt: integer(verification.startedAt, "verification.startedAt", 1), + endedAt: integer(verification.endedAt, "verification.endedAt", 1), + build: { + verifierArtifactSHA256: verifier("lean_kernel"), + exitCode: integer(build.exitCode, "verification.build.exitCode", -2147483648, 2147483647), + warnings: integer(build.warnings, "verification.build.warnings"), + transcriptSHA256: await digest(root, build.transcriptPath, "verification.build.transcriptPath"), + }, + source: { + verifierArtifactSHA256: verifier("source_auditor"), + complete: boolean(source.complete, "verification.source.complete"), + findings, + transcriptSHA256: await digest(root, source.transcriptPath, "verification.source.transcriptPath"), + }, + axioms: { + verifierArtifactSHA256: verifier("axiom_auditor"), + complete: boolean(axioms.complete, "verification.axioms.complete"), + typesTraversed: boolean(axioms.typesTraversed, "verification.axioms.typesTraversed"), + observed, + transcriptSHA256: await digest(root, axioms.transcriptPath, "verification.axioms.transcriptPath"), + }, + ...(fresh + ? { + fresh: { + verifierArtifactSHA256: verifier("fresh_rechecker"), + fresh: boolean(fresh.fresh, "verification.fresh.fresh"), + exitCode: integer(fresh.exitCode, "verification.fresh.exitCode", -2147483648, 2147483647), + transcriptSHA256: await digest(root, fresh.transcriptPath, "verification.fresh.transcriptPath"), + }, + } + : {}), + ...(external && checks + ? { + external: { + comparatorArtifactSHA256: verifier("sandbox_comparator"), + sandboxImageSHA256: text(protocol.sandboxImageSHA256, "protocol.sandboxImageSHA256"), + sandboxed: boolean(external.sandboxed, "verification.external.sandboxed"), + challengeMatched: boolean(external.challengeMatched, "verification.external.challengeMatched"), + proofTermSHA256: await digest(root, external.proofTermPath, "verification.external.proofTermPath"), + transcriptSHA256: await digest(root, external.transcriptPath, "verification.external.transcriptPath"), + checks, + }, + } + : {}), + }, + }, + preview: { + tier: protocol.tier, + relation: protocol.relation, + files: ordered.length, + observedAxioms: observed, + artifactSHA256, + }, + } +} + +if (mode === "protocol") console.log(JSON.stringify({ protocol: await protocol(first) }, null, 2)) +else if (mode === "submission") console.log(JSON.stringify(await submission(first, second!), null, 2)) +else throw new Error(`Unknown mode ${mode}`) diff --git a/backend/cli/src/command/template/learn.txt b/backend/cli/src/command/template/learn.txt index c4fb927a..c02f7509 100644 --- a/backend/cli/src/command/template/learn.txt +++ b/backend/cli/src/command/template/learn.txt @@ -8,7 +8,7 @@ Analyze the following aspects: 4. **Key Parameters**: What specific configurations, thresholds, or settings were important? 5. **Reproducibility**: What would someone need to know to reproduce this workflow on a similar problem? -Then call the `learn` tool with: +Then call the `learn` tool to create an inactive, content-addressed proposal with: - `name`: A descriptive kebab-case identifier (e.g., "debug-cuda-memory-leak", "deploy-vercel-prebuilt") - `description`: A one-line summary of what this skill teaches - `content`: A complete SKILL.md with this structure: diff --git a/backend/cli/src/server/routes/harness.ts b/backend/cli/src/server/routes/harness.ts new file mode 100644 index 00000000..80015344 --- /dev/null +++ b/backend/cli/src/server/routes/harness.ts @@ -0,0 +1,1275 @@ +import { Hono } from "hono" +import { describeRoute, resolver, validator } from "hono-openapi" +import z from "zod" +import { lazy } from "@/util/lazy" +import { HarnessAblation } from "@/session/harness/ablation" +import { HarnessAdapter } from "@/session/harness/adapter" +import { HarnessAudit } from "@/session/harness/audit" +import { HarnessAutonomy } from "@/session/harness/autonomy" +import { HarnessBlueprint } from "@/session/harness/blueprint" +import { HarnessConfirmation } from "@/session/harness/confirmation" +import { HarnessContract } from "@/session/harness/contract" +import { HarnessEvaluation } from "@/session/harness/evaluation" +import { HarnessEvolution } from "@/session/harness/evolution" +import { HarnessFailure } from "@/session/harness/failure" +import { HarnessFormal } from "@/session/harness/formal" +import { HarnessJudge } from "@/session/harness/judge" +import { HarnessMeta } from "@/session/harness/meta" +import { HarnessIntegrity } from "@/session/harness/integrity" +import { HarnessIntervention } from "@/session/harness/intervention" +import { HarnessOrchestrator } from "@/session/harness/orchestrator" +import { HarnessReport } from "@/session/harness/report" +import { HarnessReplication } from "@/session/harness/replication" +import { HarnessSimulation } from "@/session/harness/simulation" +import { HarnessSemantic } from "@/session/harness/semantic" +import { HarnessSkill } from "@/session/harness/skill" +import { HarnessSynthesis } from "@/session/harness/synthesis" +import { HarnessWorld } from "@/session/harness/world" +import { errors } from "../error" + +const SessionID = z.object({ sessionID: z.string().min(1) }) +const Compare = z + .object({ + sessionIDs: z.array(z.string().min(1)).min(2).max(100), + baselineRunID: z.string().min(1), + }) + .strict() + +export const HarnessRoutes = lazy(() => + new Hono() + .post( + "/audits", + describeRoute({ + summary: "Initialize an evaluator-owned active audit", + description: + "Commits an opaque probe pool and binds uncertainty-aware selection to the evaluator capability and audited artifact.", + operationId: "harness.audit.initialize", + responses: { + 200: { + description: "Active audit state", + content: { "application/json": { schema: resolver(HarnessAudit.State) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessAudit.Initialize), + async (c) => c.json(await HarnessAudit.initialize(c.req.valid("json"))), + ) + .post( + "/audits/:auditID/status", + describeRoute({ + summary: "Read a capability-protected active audit", + operationId: "harness.audit.status", + responses: { + 200: { + description: "Active audit state", + content: { "application/json": { schema: resolver(HarnessAudit.State) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ auditID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessAudit.Access), + async (c) => c.json(await HarnessAudit.status(c.req.valid("param").auditID, c.req.valid("json"))), + ) + .post( + "/audits/:auditID/selection", + describeRoute({ + summary: "Select the next opaque active-audit probe", + description: + "Combines weighted integral-variance reduction, failure UCB, failure-region diversity, and stratum coverage.", + operationId: "harness.audit.select", + responses: { 200: { description: "Selected opaque probe commitment" }, ...errors(400, 403, 404, 409) }, + }), + validator("param", z.object({ auditID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessAudit.Access), + async (c) => c.json(await HarnessAudit.select(c.req.valid("param").auditID, c.req.valid("json"))), + ) + .post( + "/audits/:auditID/observations", + describeRoute({ + summary: "Record an evaluator-authenticated probe outcome", + description: + "Updates the GP posterior and stopping rule without promoting the audit estimate into benchmark evidence.", + operationId: "harness.audit.observe", + responses: { + 200: { + description: "Updated active audit state", + content: { "application/json": { schema: resolver(HarnessAudit.State) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ auditID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessAudit.Observe), + async (c) => c.json(await HarnessAudit.observe(c.req.valid("param").auditID, c.req.valid("json"))), + ) + .post( + "/audits/:auditID/receipt", + describeRoute({ + summary: "Seal a terminal active-audit receipt", + description: + "Content-addresses the completed audit, exact subject artifact, committed pool, derived estimate, transfer qualification, and terminal revision for optional promotion gating.", + operationId: "harness.audit.seal", + responses: { + 200: { + description: "Immutable active-audit receipt", + content: { "application/json": { schema: resolver(HarnessAudit.Receipt) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ auditID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessAudit.Access), + async (c) => c.json(await HarnessAudit.seal(c.req.valid("param").auditID, c.req.valid("json"))), + ) + .post( + "/failure-streams", + describeRoute({ + summary: "Initialize a topic-aware adversarial failure stream", + description: + "Binds deterministic UCB1 topic allocation and server-derived failure anchors to a terminal active-audit receipt without adding generated cases to the population estimate.", + operationId: "harness.failure.initialize", + responses: { + 200: { + description: "Topic-aware failure discovery state", + content: { "application/json": { schema: resolver(HarnessFailure.State) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("json", HarnessFailure.Initialize), + async (c) => c.json(await HarnessFailure.initialize(c.req.valid("json"))), + ) + .post( + "/failure-streams/:streamID/status", + describeRoute({ + summary: "Read a capability-protected failure discovery stream", + operationId: "harness.failure.status", + responses: { + 200: { + description: "Topic-aware failure discovery state", + content: { "application/json": { schema: resolver(HarnessFailure.State) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ streamID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessFailure.Access), + async (c) => c.json(await HarnessFailure.status(c.req.valid("param").streamID, c.req.valid("json"))), + ) + .post( + "/failure-streams/:streamID/selection", + describeRoute({ + summary: "Select the next topic and authenticated failure anchors", + description: + "Forces every frozen topic once, then derives UCB1 from the immutable attempt journal with deterministic tie-breaking.", + operationId: "harness.failure.select", + responses: { + 200: { + description: "Server-selected topic, anchors, and allocation evidence", + content: { "application/json": { schema: resolver(HarnessFailure.Selection) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ streamID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessFailure.Access), + async (c) => c.json(await HarnessFailure.next(c.req.valid("param").streamID, c.req.valid("json"))), + ) + .post( + "/failure-streams/:streamID/attempts", + describeRoute({ + summary: "Record a validated adversarial generation attempt", + description: + "Consumes one attempt budget and derives admissibility and reward from the frozen correctness, topic, and novelty validators plus the target outcome.", + operationId: "harness.failure.observe", + responses: { + 200: { + description: "Updated topic-aware failure discovery state", + content: { "application/json": { schema: resolver(HarnessFailure.State) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ streamID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessFailure.Observe), + async (c) => c.json(await HarnessFailure.observe(c.req.valid("param").streamID, c.req.valid("json"))), + ) + .post( + "/failure-streams/:streamID/receipt", + describeRoute({ + summary: "Seal a terminal failure discovery receipt", + description: + "Content-addresses the exact audit source, subject, topic contract, attempt journal, replayed UCB statistics, failure yield, and diversity evidence.", + operationId: "harness.failure.seal", + responses: { + 200: { + description: "Immutable topic-aware failure discovery receipt", + content: { "application/json": { schema: resolver(HarnessFailure.Receipt) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ streamID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessFailure.Access), + async (c) => c.json(await HarnessFailure.seal(c.req.valid("param").streamID, c.req.valid("json"))), + ) + .post( + "/ablations", + describeRoute({ + summary: "Freeze a matched scientific ablation plan", + description: + "Binds at least three evaluator-authenticated seed pairs before evaluation and permits exactly one declared contract factor to differ.", + operationId: "harness.ablation.initialize", + responses: { + 200: { + description: "Immutable matched ablation plan", + content: { "application/json": { schema: resolver(HarnessAblation.State) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessAblation.Initialize), + async (c) => c.json(await HarnessAblation.initialize(c.req.valid("json"))), + ) + .post( + "/ablations/:planID/assessment", + describeRoute({ + summary: "Assess a frozen matched ablation", + description: + "Authenticates every paired run, verifies immutable contracts and final evaluations, then derives paired effects and a 95% interval.", + operationId: "harness.ablation.assess", + responses: { + 200: { + description: "Immutable matched ablation assessment", + content: { "application/json": { schema: resolver(HarnessAblation.State) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ planID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessAblation.Assess), + async (c) => c.json(await HarnessAblation.assess(c.req.valid("param").planID, c.req.valid("json"))), + ) + .post( + "/interventions", + describeRoute({ + summary: "Freeze an evaluator-owned controlled replay study", + description: + "Binds a candidate and exact evolution receipt to predeclared replay, retuning, ablation, repair, or transfer pairs before the candidate's final evaluation.", + operationId: "harness.intervention.initialize", + responses: { + 200: { + description: "Immutable controlled intervention plan", + content: { "application/json": { schema: resolver(HarnessIntervention.State) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessIntervention.Initialize), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessIntervention.initialize(input, contract)) + }, + ) + .post( + "/interventions/:candidateID/observations", + describeRoute({ + summary: "Record an evaluator-authenticated intervention outcome", + description: + "Binds one numeric outcome to an exact frozen pair target without adding it to candidate fitness or the benchmark evaluation journal.", + operationId: "harness.intervention.observe", + responses: { + 200: { + description: "Immutable intervention outcome", + content: { "application/json": { schema: resolver(HarnessIntervention.Outcome) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ candidateID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessIntervention.Observe), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessIntervention.observe(c.req.valid("param").candidateID, input, contract)) + }, + ) + .post( + "/interventions/:candidateID/assessment", + describeRoute({ + summary: "Assess a complete controlled replay study", + description: + "Recomputes direction-aware paired effects, confidence intervals, stability, tuning gap, component dependence, and transfer robustness from every frozen outcome.", + operationId: "harness.intervention.assess", + responses: { + 200: { + description: "Immutable controlled intervention receipt", + content: { "application/json": { schema: resolver(HarnessIntervention.Receipt) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", z.object({ candidateID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessIntervention.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessIntervention.assess(input.sessionID, c.req.valid("param").candidateID, contract)) + }, + ) + .post( + "/interventions/:candidateID/status", + describeRoute({ + summary: "Read a capability-protected controlled replay study", + operationId: "harness.intervention.status", + responses: { + 200: { + description: "Controlled intervention state", + content: { "application/json": { schema: resolver(HarnessIntervention.State.nullable()) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ candidateID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessIntervention.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessIntervention.status(input.sessionID, c.req.valid("param").candidateID, contract)) + }, + ) + .post( + "/evaluators/qualifications", + describeRoute({ + summary: "Qualify a bound benchmark evaluator", + description: + "Uses an independent auditor capability and a committed hidden fault suite to recompute evaluator discrimination and calibration metrics.", + operationId: "harness.judge.record", + responses: { + 200: { + description: "Immutable evaluator audit receipt", + content: { "application/json": { schema: resolver(HarnessJudge.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessJudge.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeAuditor(input.sessionID, input.auditorToken) + return c.json(await HarnessJudge.record(input, contract)) + }, + ) + .post( + "/evaluators/qualifications/:receiptID", + describeRoute({ + summary: "Read a capability-protected evaluator qualification", + operationId: "harness.judge.receipt", + responses: { + 200: { + description: "Evaluator audit receipt", + content: { "application/json": { schema: resolver(HarnessJudge.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessJudge.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeAuditor(input.sessionID, input.auditorToken) + return c.json( + await HarnessJudge.assert({ + contract, + receiptID: c.req.valid("param").receiptID, + recordedAt: Date.now(), + requirePassed: false, + }), + ) + }, + ) + .post( + "/replications/receipts", + describeRoute({ + summary: "Record an evaluator-authenticated replicated evaluation", + description: + "Requires the complete frozen stratum-by-cluster grid, then recomputes a robust estimate, uncertainty interval, and conservative promotion verdict.", + operationId: "harness.replication.record", + responses: { + 200: { + description: "Immutable replicated evaluation receipt", + content: { "application/json": { schema: resolver(HarnessReplication.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessReplication.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessReplication.record(input, contract)) + }, + ) + .post( + "/replications/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected replicated evaluation receipt", + operationId: "harness.replication.receipt", + responses: { + 200: { + description: "Replicated evaluation receipt", + content: { "application/json": { schema: resolver(HarnessReplication.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessReplication.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + const receipt = await HarnessReplication.read(c.req.valid("param").receiptID) + if (!receipt) throw new Error(`Unknown or corrupt replicated evaluation receipt`) + return c.json( + await HarnessReplication.assert({ + contract, + receiptID: receipt.receiptID, + subject: receipt.subject, + score: receipt.statistics.estimate, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + requirePassed: false, + }), + ) + }, + ) + .post( + "/meta/selection", + describeRoute({ + summary: "Resolve the terminal meta-harness qualification subject", + description: + "Returns the backend-selected verified winner after search termination, isolated behind the independent meta-harness qualifier capability.", + operationId: "harness.meta.selection", + responses: { + 200: { + description: "Content-addressed terminal meta-harness selection", + content: { "application/json": { schema: resolver(HarnessMeta.Selection) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessMeta.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeMeta(input.sessionID, input.metaToken) + return c.json(await HarnessMeta.select(contract)) + }, + ) + .post( + "/meta/receipts", + describeRoute({ + summary: "Record a one-shot continual-harness qualification", + description: + "Freezes the complete refinement lineage, full trace archive, cross-model held-out matrix, activation/adherence diagnostics, and backend-derived promotion verdict.", + operationId: "harness.meta.record", + responses: { + 200: { + description: "Immutable meta-harness qualification receipt", + content: { "application/json": { schema: resolver(HarnessMeta.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessMeta.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeMeta(input.sessionID, input.metaToken) + return c.json(await HarnessMeta.record(input, contract)) + }, + ) + .post( + "/meta/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected meta-harness receipt", + operationId: "harness.meta.receipt", + responses: { + 200: { + description: "Canonical meta-harness qualification receipt", + content: { "application/json": { schema: resolver(HarnessMeta.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessMeta.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeMeta(input.sessionID, input.metaToken) + return c.json(await HarnessMeta.assert(contract, c.req.valid("param").receiptID)) + }, + ) + .post( + "/confirmations/selection", + describeRoute({ + summary: "Resolve the sealed post-search confirmation subject", + description: + "Returns exactly one backend-selected verified winner only after adaptive search is terminal. The endpoint is isolated behind the claim evaluator capability.", + operationId: "harness.confirmation.selection", + responses: { + 200: { + description: "Immutable terminal winner selection", + content: { "application/json": { schema: resolver(HarnessConfirmation.Selection) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessConfirmation.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeConfirmation(input.sessionID, input.confirmationToken) + return c.json(await HarnessConfirmation.select(contract)) + }, + ) + .post( + "/confirmations/receipts", + describeRoute({ + summary: "Record a one-shot sealed claim evaluation", + description: + "Freezes the claim result for the server-selected terminal winner without feeding the result into search, adaptive control, hindsight memory, or skill learning.", + operationId: "harness.confirmation.record", + responses: { + 200: { + description: "Immutable sealed confirmation receipt", + content: { "application/json": { schema: resolver(HarnessConfirmation.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessConfirmation.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeConfirmation(input.sessionID, input.confirmationToken) + return c.json(await HarnessConfirmation.record(input, contract)) + }, + ) + .post( + "/confirmations/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected sealed confirmation receipt", + operationId: "harness.confirmation.receipt", + responses: { + 200: { + description: "Canonical sealed confirmation receipt", + content: { "application/json": { schema: resolver(HarnessConfirmation.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessConfirmation.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeConfirmation(input.sessionID, input.confirmationToken) + return c.json(await HarnessConfirmation.assert(contract, c.req.valid("param").receiptID)) + }, + ) + .post( + "/semantics/receipts", + describeRoute({ + summary: "Record an independent semantic audit", + description: + "Derives whether one bound result is meaningful, merely technically valid, ambiguous, or incorrect from independent evidence-backed reviews of frozen intent, shortcuts, and literature-relative novelty.", + operationId: "harness.semantic.record", + responses: { + 200: { + description: "Immutable semantic audit receipt", + content: { "application/json": { schema: resolver(HarnessSemantic.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessSemantic.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeSemantic(input.sessionID, input.reviewerToken) + return c.json(await HarnessSemantic.record(input, contract)) + }, + ) + .post( + "/semantics/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected semantic audit receipt", + operationId: "harness.semantic.receipt", + responses: { + 200: { + description: "Semantic audit receipt", + content: { "application/json": { schema: resolver(HarnessSemantic.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessSemantic.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorizeSemantic(input.sessionID, input.reviewerToken) + const receipt = await HarnessSemantic.read(c.req.valid("param").receiptID) + if (!receipt) throw new Error(`Unknown or corrupt semantic audit receipt`) + return c.json( + await HarnessSemantic.assert({ + contract, + receiptID: receipt.receiptID, + subject: receipt.subject, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + requirePassed: false, + }), + ) + }, + ) + .post( + "/syntheses/receipts", + describeRoute({ + summary: "Record an evaluator-authenticated clean-room synthesis", + description: + "Binds a complete retrieval trace and hidden atomic-fact manifest, rejects clean-room policy drift, and derives factual precision, recall, contradiction penalty, and F1 without trusting caller-authored metrics.", + operationId: "harness.synthesis.record", + responses: { + 200: { + description: "Immutable scientific synthesis receipt", + content: { "application/json": { schema: resolver(HarnessSynthesis.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessSynthesis.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessSynthesis.record(input, contract)) + }, + ) + .post( + "/syntheses/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected scientific synthesis receipt", + operationId: "harness.synthesis.receipt", + responses: { + 200: { + description: "Canonical scientific synthesis receipt", + content: { "application/json": { schema: resolver(HarnessSynthesis.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessSynthesis.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessSynthesis.read(c.req.valid("param").receiptID, contract)) + }, + ) + .post( + "/autonomy/receipts", + describeRoute({ + summary: "Record an evaluator-authenticated human-AI autonomy trace", + description: + "Binds a complete interaction log to the exact run or candidate artifact and derives the Aletheia-inspired essentially-autonomous, collaborative, or primarily-human contribution level without trusting the caller's claim.", + operationId: "harness.autonomy.record", + responses: { + 200: { + description: "Immutable backend-derived human-AI autonomy receipt", + content: { "application/json": { schema: resolver(HarnessAutonomy.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessAutonomy.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessAutonomy.record(input, contract)) + }, + ) + .post( + "/autonomy/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected human-AI autonomy receipt", + operationId: "harness.autonomy.receipt", + responses: { + 200: { + description: "Canonical human-AI autonomy receipt", + content: { "application/json": { schema: resolver(HarnessAutonomy.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessAutonomy.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessAutonomy.read(c.req.valid("param").receiptID, contract)) + }, + ) + .post( + "/proofs/blueprints", + describeRoute({ + summary: "Initialize an evaluator-grounded formal proof blueprint", + description: + "Creates the content-addressed root of a bounded LEAP-inspired AND/OR proof graph without granting the graph final proof authority.", + operationId: "harness.blueprint.initialize", + responses: { + 200: { + description: "Canonical proof blueprint view", + content: { "application/json": { schema: resolver(HarnessBlueprint.View) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessBlueprint.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessBlueprint.initialize(contract)) + }, + ) + .post( + "/proofs/blueprints/status", + describeRoute({ + summary: "Read an evaluator-grounded formal proof blueprint", + operationId: "harness.blueprint.status", + responses: { + 200: { + description: "Backend-derived goal, decomposition, attempt, and lease state", + content: { "application/json": { schema: resolver(HarnessBlueprint.View) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("json", HarnessBlueprint.Access), + async (c) => { + const input = c.req.valid("json") + await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessBlueprint.read(input.sessionID)) + }, + ) + .post( + "/proofs/blueprints/leases", + describeRoute({ + summary: "Lease bounded ready goals from a formal proof blueprint", + description: + "Atomically expires stale work and leases distinct deepest-ready goals up to the frozen parallelism limit.", + operationId: "harness.blueprint.lease", + responses: { + 200: { + description: "Issued leases and updated proof blueprint", + content: { + "application/json": { + schema: resolver(z.object({ leases: z.array(HarnessBlueprint.Lease), state: HarnessBlueprint.View })), + }, + }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessBlueprint.LeaseRequest), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessBlueprint.lease(contract, input.count)) + }, + ) + .post( + "/proofs/blueprints/attempts", + describeRoute({ + summary: "Record an evaluator-authenticated proof or decomposition attempt", + description: + "Consumes one active goal lease, retains failed verifier or reviewer outcomes, and admits only exact compiler-checked sketches into the monotone acyclic graph.", + operationId: "harness.blueprint.record", + responses: { + 200: { + description: "Recorded attempt and updated proof blueprint", + content: { + "application/json": { + schema: resolver( + z.object({ + attemptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + decompositionID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + state: HarnessBlueprint.View, + }), + ), + }, + }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessBlueprint.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessBlueprint.record(input, contract)) + }, + ) + .post( + "/proofs/receipts", + describeRoute({ + summary: "Record an evaluator-authenticated formal proof verification", + description: + "Binds a trusted Lean challenge, exact proof artifact, frozen environment, transitive axiom audit, and the contract's kernel, fresh-recheck, or external-crosscheck trust tier.", + operationId: "harness.formal.record", + responses: { + 200: { + description: "Immutable backend-derived formal proof receipt", + content: { "application/json": { schema: resolver(HarnessFormal.Receipt) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessFormal.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessFormal.record(input, contract)) + }, + ) + .post( + "/proofs/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected formal proof receipt", + operationId: "harness.formal.receipt", + responses: { + 200: { + description: "Canonical formal proof receipt", + content: { "application/json": { schema: resolver(HarnessFormal.Receipt) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessFormal.Access), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessFormal.read(c.req.valid("param").receiptID, contract)) + }, + ) + .post( + "/integrity/receipts", + describeRoute({ + summary: "Record evaluator-authenticated runtime integrity", + description: + "Derives trace-completeness, model-identity, contamination, external-model, benchmark-lookup, and hidden-canary gates against an immutable protocol.", + operationId: "harness.integrity.record", + responses: { + 200: { + description: "Immutable runtime integrity receipt", + content: { "application/json": { schema: resolver(HarnessIntegrity.Info) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessIntegrity.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessIntegrity.record(input, contract)) + }, + ) + .post( + "/integrity/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected runtime integrity receipt", + operationId: "harness.integrity.receipt", + responses: { + 200: { + description: "Runtime integrity receipt", + content: { "application/json": { schema: resolver(HarnessIntegrity.Info.nullable()) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessIntegrity.Access), + async (c) => { + const input = c.req.valid("json") + await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessIntegrity.read(input.sessionID, c.req.valid("param").receiptID)) + }, + ) + .post( + "/evolution/receipts", + describeRoute({ + summary: "Record evaluator-authenticated evolutionary provenance", + description: + "Binds a candidate snapshot and every parent delta to immutable search lineage, then derives replay and ancestral line-reintroduction diagnostics without changing fitness.", + operationId: "harness.evolution.record", + responses: { + 200: { + description: "Immutable evolution trace receipt", + content: { + "application/json": { + schema: resolver(HarnessEvolution.Info as z.ZodType>), + }, + }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessEvolution.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json((await HarnessEvolution.record(input, contract)) as Record) + }, + ) + .post( + "/evolution/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected evolution trace receipt", + operationId: "harness.evolution.receipt", + responses: { + 200: { + description: "Evolution trace receipt", + content: { + "application/json": { + schema: resolver(HarnessEvolution.Info.nullable() as z.ZodType | null>), + }, + }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessEvolution.Access), + async (c) => { + const input = c.req.valid("json") + await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json( + (await HarnessEvolution.read(input.sessionID, c.req.valid("param").receiptID)) as Record< + string, + unknown + > | null, + ) + }, + ) + .post( + "/simulations/receipts", + describeRoute({ + summary: "Record an evaluator-authenticated simulator validation", + description: + "Recomputes convergence, residual, invariant, and stress-test gates against the immutable simulator protocol and exact subject artifact.", + operationId: "harness.simulation.record", + responses: { + 200: { + description: "Immutable simulator validation receipt", + content: { "application/json": { schema: resolver(HarnessSimulation.Info) } }, + }, + ...errors(400, 403, 409), + }, + }), + validator("json", HarnessSimulation.Submit), + async (c) => { + const input = c.req.valid("json") + const contract = await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessSimulation.record(input, contract)) + }, + ) + .post( + "/simulations/receipts/:receiptID", + describeRoute({ + summary: "Read a capability-protected simulator validation receipt", + operationId: "harness.simulation.receipt", + responses: { + 200: { + description: "Simulator validation receipt", + content: { "application/json": { schema: resolver(HarnessSimulation.Info.nullable()) } }, + }, + ...errors(400, 403, 404), + }, + }), + validator("param", z.object({ receiptID: z.string().regex(/^[a-f0-9]{64}$/) })), + validator("json", HarnessSimulation.Access), + async (c) => { + const input = c.req.valid("json") + await HarnessAdapter.authorize(input.sessionID, input.evaluatorToken) + return c.json(await HarnessSimulation.read(input.sessionID, c.req.valid("param").receiptID)) + }, + ) + .post( + "/runs/:sessionID/orchestration", + describeRoute({ + summary: "Initialize contract-bound scientific orchestration", + description: + "Selects a bounded topology from immutable contract traits and creates a restart-safe provisional work DAG.", + operationId: "harness.orchestration.start", + responses: { + 200: { + description: "Scientific orchestration state", + content: { "application/json": { schema: resolver(HarnessOrchestrator.State) } }, + }, + ...errors(400, 404, 409), + }, + }), + validator("param", SessionID), + async (c) => c.json(await HarnessOrchestrator.initialize(c.req.valid("param").sessionID)), + ) + .get( + "/runs/:sessionID/orchestration", + describeRoute({ + summary: "Read scientific orchestration state", + operationId: "harness.orchestration.status", + responses: { + 200: { + description: "Scientific orchestration state", + content: { "application/json": { schema: resolver(HarnessOrchestrator.State) } }, + }, + ...errors(400, 404), + }, + }), + validator("param", SessionID), + async (c) => c.json(await HarnessOrchestrator.read(c.req.valid("param").sessionID)), + ) + .post( + "/runs/:sessionID/orchestration/checkpoints", + describeRoute({ + summary: "Record an evaluator-authenticated orchestration utility checkpoint", + description: + "Gates the next evolution round and stops low-utility search without allowing worker self-scores to control budget.", + operationId: "harness.orchestration.checkpoint", + responses: { + 200: { + description: "Scientific orchestration state", + content: { "application/json": { schema: resolver(HarnessOrchestrator.State) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", SessionID), + validator("json", HarnessOrchestrator.CheckpointSubmit), + async (c) => { + const input = c.req.valid("json") + const sessionID = c.req.valid("param").sessionID + const contract = await HarnessAdapter.authorize(sessionID, input.evaluatorToken) + return c.json( + await HarnessOrchestrator.checkpoint( + { + sessionID, + round: input.round, + utility: input.utility, + uncertainty: input.uncertainty, + evidenceRefs: input.evidenceRefs, + evaluatedAt: input.evaluatedAt, + }, + contract, + ), + ) + }, + ) + .get( + "/runs/:sessionID/world", + describeRoute({ + summary: "Read the session-local continual world model", + description: + "Returns confidence-graded mutable working state, event boundaries, context epoch, refinement trigger, and rollback revisions without exposing evaluator capabilities.", + operationId: "harness.world.status", + responses: { + 200: { + description: "Continual world-model state", + content: { "application/json": { schema: resolver(HarnessWorld.State) } }, + }, + ...errors(400, 404), + }, + }), + validator("param", SessionID), + async (c) => c.json(await HarnessWorld.read(c.req.valid("param").sessionID)), + ) + .post( + "/runs/:sessionID/world/refinements", + describeRoute({ + summary: "Apply an evaluator-authenticated world-model refinement", + description: + "Applies a small revision-checked patch. Evaluator evidence may raise confidence beyond the agent's self-report ceiling while the immutable base prompt remains unchanged.", + operationId: "harness.world.refine", + responses: { + 200: { + description: "Refined continual world-model state", + content: { "application/json": { schema: resolver(HarnessWorld.State) } }, + }, + ...errors(400, 403, 404, 409), + }, + }), + validator("param", SessionID), + validator("json", HarnessWorld.EvaluatorRefine), + async (c) => { + const input = c.req.valid("json") + const sessionID = c.req.valid("param").sessionID + await HarnessAdapter.authorize(sessionID, input.evaluatorToken) + return c.json( + await HarnessWorld.evaluatorRefine({ + sessionID, + expectedRevision: input.expectedRevision, + reason: input.reason, + patches: input.patches, + }), + ) + }, + ) + .post( + "/runs", + describeRoute({ + summary: "Bind an immutable scientific evaluation run", + description: + "Called by a local or external evaluator before agent execution. The evaluator capability is hashed and never returned.", + operationId: "harness.bind", + responses: { + 200: { + description: "Bound harness contract", + content: { "application/json": { schema: resolver(HarnessContract.Info) } }, + }, + ...errors(400, 409), + }, + }), + validator("json", HarnessAdapter.Task), + async (c) => c.json(await HarnessAdapter.bind(c.req.valid("json"))), + ) + .post( + "/evaluations", + describeRoute({ + summary: "Ingest an evaluator-authenticated result", + description: + "Records an immutable subject result, promotes a verified search candidate, and captures task-scoped hindsight.", + operationId: "harness.evaluate", + responses: { 200: { description: "Recorded external evaluation" }, ...errors(400, 403, 409) }, + }), + validator("json", HarnessAdapter.Evaluation), + async (c) => c.json(await HarnessAdapter.ingest(c.req.valid("json"))), + ) + .post( + "/compare", + describeRoute({ + summary: "Compare compatible scientific evaluation runs", + description: "Reports direction-aware deltas and the quality-cost Pareto frontier.", + operationId: "harness.compare", + responses: { 200: { description: "Comparable run deltas" }, ...errors(400) }, + }), + validator("json", Compare), + async (c) => { + const input = c.req.valid("json") + const reports = await Promise.all(input.sessionIDs.map((sessionID) => HarnessReport.build(sessionID))) + return c.json(HarnessReport.compare(reports, input.baselineRunID)) + }, + ) + .get( + "/skills", + describeRoute({ + summary: "List quarantined learned skill proposals", + operationId: "harness.skills", + responses: { + 200: { + description: "Learned skill qualification manifests", + content: { "application/json": { schema: resolver(z.array(HarnessSkill.Manifest)) } }, + }, + }, + }), + async (c) => c.json(await HarnessSkill.list()), + ) + .post( + "/skills", + describeRoute({ + summary: "Create an inactive learned skill proposal", + operationId: "harness.skill.propose", + responses: { + 200: { + description: "Quarantined proposal", + content: { "application/json": { schema: resolver(HarnessSkill.Manifest.nullable()) } }, + }, + ...errors(400, 409), + }, + }), + validator("json", HarnessSkill.ProposalInput), + async (c) => c.json(await HarnessSkill.propose(c.req.valid("json"))), + ) + .post( + "/skills/evidence", + describeRoute({ + summary: "Attach paired held-out skill evidence", + description: + "Requires both evaluator capabilities and accepts only otherwise-identical candidate/control contracts.", + operationId: "harness.skill.attest", + responses: { 200: { description: "Updated qualification state" }, ...errors(400, 403, 409) }, + }), + validator("json", HarnessSkill.Attestation), + async (c) => c.json(await HarnessSkill.attest(c.req.valid("json"))), + ) + .post( + "/skills/:name/promotion", + describeRoute({ + summary: "Promote a qualified learned skill", + description: "Copies only an unchanged proposal that has met every held-out qualification criterion.", + operationId: "harness.skill.promote", + responses: { 200: { description: "Promoted skill" }, ...errors(400, 409) }, + }), + validator("param", z.object({ name: z.string().min(1) })), + async (c) => c.json(await HarnessSkill.promote(c.req.valid("param").name)), + ) + .get( + "/runs/:sessionID/contract", + describeRoute({ + summary: "Read a bound harness contract", + operationId: "harness.contract", + responses: { + 200: { + description: "Harness contract", + content: { "application/json": { schema: resolver(HarnessContract.Info.nullable()) } }, + }, + ...errors(400), + }, + }), + validator("param", SessionID), + async (c) => c.json(await HarnessContract.read(c.req.valid("param").sessionID)), + ) + .get( + "/runs/:sessionID/evaluations", + describeRoute({ + summary: "List immutable harness evaluations", + operationId: "harness.evaluations", + responses: { + 200: { + description: "Evaluation journal", + content: { "application/json": { schema: resolver(z.array(HarnessEvaluation.Info)) } }, + }, + ...errors(400), + }, + }), + validator("param", SessionID), + async (c) => c.json(await HarnessEvaluation.list(c.req.valid("param").sessionID)), + ) + .get( + "/runs/:sessionID/report", + describeRoute({ + summary: "Build an evaluation quality-cost report", + operationId: "harness.report", + responses: { + 200: { + description: "Quality-cost report", + content: { "application/json": { schema: resolver(HarnessReport.Info) } }, + }, + ...errors(400, 404), + }, + }), + validator("param", SessionID), + async (c) => c.json(await HarnessReport.build(c.req.valid("param").sessionID)), + ), +) diff --git a/backend/cli/src/server/server.ts b/backend/cli/src/server/server.ts index 48596fff..01f21dd3 100644 --- a/backend/cli/src/server/server.ts +++ b/backend/cli/src/server/server.ts @@ -27,6 +27,7 @@ import { Command } from "../command" import { Global } from "../global" import { ProjectRoutes } from "./routes/project" import { SessionRoutes } from "./routes/session" +import { HarnessRoutes } from "./routes/harness" import { PtyRoutes } from "./routes/pty" import { McpRoutes } from "./routes/mcp" import { FileRoutes } from "./routes/file" @@ -312,6 +313,7 @@ export namespace Server { .route("/config", ConfigRoutes()) .route("/experimental", ExperimentalRoutes()) .route("/session", SessionRoutes()) + .route("/harness", HarnessRoutes()) .route("/search", SearchRoutes()) .route("/permission", PermissionRoutes()) .route("/question", QuestionRoutes()) diff --git a/backend/cli/src/session/harness/ablation.ts b/backend/cli/src/session/harness/ablation.ts new file mode 100644 index 00000000..d2c0e705 --- /dev/null +++ b/backend/cli/src/session/harness/ablation.ts @@ -0,0 +1,532 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessAdapter } from "./adapter" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" +import { HarnessSearch } from "./search" + +export namespace HarnessAblation { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + export const Factor = z + .object({ + kind: z.enum([ + "profile", + "orchestration", + "search", + "audit", + "simulation", + "evaluator_audit", + "semantic_audit", + "synthesis", + "autonomy", + "formal_proof", + "replication", + "fidelities", + "skill", + "tool", + ]), + name: z.string().min(1).max(200).optional(), + }) + .strict() + .superRefine((value, ctx) => { + const named = value.kind === "skill" || value.kind === "tool" + if (named && !value.name) { + ctx.addIssue({ code: "custom", path: ["name"], message: `${value.kind} ablations require a name` }) + } + if (!named && value.name) { + ctx.addIssue({ code: "custom", path: ["name"], message: `${value.kind} ablations cannot name a subfactor` }) + } + }) + export type Factor = z.infer + + const Ref = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + + export const Initialize = z + .object({ + schemaVersion: z.literal(1), + studyID: z.string().min(1).max(240), + factor: Factor, + minEffect: z.number().finite().nonnegative(), + maxPairRegression: z.number().finite().nonnegative().default(0), + pairs: z + .array(z.object({ baseline: Ref, arm: Ref }).strict()) + .min(3) + .max(32), + }) + .strict() + export type Initialize = z.input + + const Run = z + .object({ + sessionID: z.string().min(1).max(240), + runID: z.string().min(1).max(240), + contractFingerprint: Hash, + }) + .strict() + + export const Plan = z + .object({ + schemaVersion: z.literal(1), + planID: Hash, + studyID: z.string().min(1).max(240), + factor: Factor, + baselineValueSHA256: Hash, + armValueSHA256: Hash, + contextSHA256: Hash, + benchmark: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + taskID: z.string().min(1), + split: z.enum(["held_out", "release"]), + evaluator: z.string().min(1), + evaluatorVersion: z.string().min(1), + metric: z.string().min(1), + direction: z.enum(["maximize", "minimize"]), + }) + .strict(), + minEffect: z.number().finite().nonnegative(), + maxPairRegression: z.number().finite().nonnegative(), + pairs: z + .array( + z + .object({ + seed: z.number().int(), + baseline: Run, + arm: Run, + }) + .strict(), + ) + .min(3) + .max(32), + createdAt: z.number().int().positive(), + }) + .strict() + export type Plan = z.infer + + export const Assess = z + .object({ + runs: z + .array(Ref) + .min(6) + .max(64) + .refine( + (items) => new Set(items.map((item) => item.sessionID)).size === items.length, + "Run credentials must be unique", + ), + }) + .strict() + export type Assess = z.input + + const Outcome = z + .object({ + sessionID: z.string().min(1), + runID: z.string().min(1), + status: HarnessEvaluation.Status, + score: z.number().finite().optional(), + evaluationSHA256: Hash, + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + + export const Receipt = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + planID: Hash, + studyID: z.string().min(1), + factor: Factor, + pairs: z + .array( + z + .object({ + seed: z.number().int(), + baseline: Outcome, + arm: Outcome, + effect: z.number().finite().optional(), + }) + .strict(), + ) + .min(3) + .max(32), + statistics: z + .object({ + pairs: z.number().int().min(3), + validPairs: z.number().int().nonnegative(), + meanEffect: z.number().finite().optional(), + standardDeviation: z.number().finite().nonnegative().optional(), + standardError: z.number().finite().nonnegative().optional(), + confidence95: z.tuple([z.number().finite(), z.number().finite()]).optional(), + regressions: z.number().int().nonnegative(), + minEffect: z.number().finite().nonnegative(), + maxPairRegression: z.number().finite().nonnegative(), + }) + .strict(), + verdict: z.enum(["supported", "rejected", "inconclusive"]), + assessedAt: z.number().int().positive(), + }) + .strict() + export type Receipt = z.infer + + export const State = z + .object({ + schemaVersion: z.literal(1), + plan: Plan, + receipt: Receipt.optional(), + }) + .strict() + .superRefine((value, ctx) => { + const payload = structuredClone(value.plan) as Record + delete payload.planID + if (digest(payload) !== value.plan.planID) { + ctx.addIssue({ code: "custom", path: ["plan", "planID"], message: "Ablation plan content hash is invalid" }) + } + if (!value.receipt) return + if (value.receipt.planID !== value.plan.planID) { + ctx.addIssue({ code: "custom", path: ["receipt", "planID"], message: "Ablation receipt targets another plan" }) + } + const receipt = structuredClone(value.receipt) as Record + delete receipt.receiptID + delete receipt.assessedAt + if (digest(receipt) !== value.receipt.receiptID) { + ctx.addIssue({ + code: "custom", + path: ["receipt", "receiptID"], + message: "Ablation receipt content hash is invalid", + }) + } + }) + export type State = z.infer + + const root = path.join(Global.Path.data, "harness", "ablations") + const file = (planID: string) => path.join(root, `${planID}.json`) + + function value(contract: HarnessContract.Info, factor: Factor) { + if (factor.kind === "profile") return contract.profile + if (factor.kind === "orchestration") return contract.orchestration ?? null + if (factor.kind === "search") return contract.search ?? null + if (factor.kind === "audit") return contract.audit ?? null + if (factor.kind === "simulation") return contract.simulation ?? null + if (factor.kind === "evaluator_audit") return contract.evaluatorAudit ?? null + if (factor.kind === "semantic_audit") return contract.semanticAudit ?? null + if (factor.kind === "synthesis") return contract.synthesis ?? null + if (factor.kind === "autonomy") return contract.autonomy ?? null + if (factor.kind === "formal_proof") return contract.formalProof ?? null + if (factor.kind === "replication") return contract.replication ?? null + if (factor.kind === "fidelities") return contract.benchmark.fidelities ?? null + if (factor.kind === "skill") return contract.skills.find((item) => item.name === factor.name) ?? null + return contract.tools.includes(factor.name!) + } + + function context(contract: HarnessContract.Info, factor: Factor, seed: boolean) { + const benchmark = { + name: contract.benchmark.name, + version: contract.benchmark.version, + taskID: contract.benchmark.taskID, + split: contract.benchmark.split, + evaluator: contract.benchmark.evaluator, + evaluatorVersion: contract.benchmark.evaluatorVersion, + evaluatorSource: contract.benchmark.evaluatorSource, + ...(factor.kind === "fidelities" ? {} : { fidelities: contract.benchmark.fidelities }), + metric: contract.benchmark.metric, + direction: contract.benchmark.direction, + target: contract.benchmark.target, + } + return { + objective: contract.objective, + benchmark, + ...(factor.kind === "profile" ? {} : { profile: contract.profile }), + ...(factor.kind === "orchestration" ? {} : { orchestration: contract.orchestration }), + ...(factor.kind === "search" ? {} : { search: contract.search }), + ...(factor.kind === "audit" ? {} : { audit: contract.audit }), + integrity: contract.integrity, + evolution: contract.evolution, + metaHarness: contract.metaHarness, + interventions: contract.interventions, + ...(factor.kind === "simulation" ? {} : { simulation: contract.simulation }), + ...(factor.kind === "evaluator_audit" ? {} : { evaluatorAudit: contract.evaluatorAudit }), + ...(factor.kind === "semantic_audit" ? {} : { semanticAudit: contract.semanticAudit }), + ...(factor.kind === "synthesis" ? {} : { synthesis: contract.synthesis }), + ...(factor.kind === "autonomy" ? {} : { autonomy: contract.autonomy }), + ...(factor.kind === "formal_proof" ? {} : { formalProof: contract.formalProof }), + ...(factor.kind === "replication" ? {} : { replication: contract.replication }), + confirmation: contract.confirmation, + packs: (contract.packs ?? []).filter((item) => factor.kind !== "formal_proof" || item !== "formal").toSorted(), + model: contract.model, + tools: contract.tools.filter((item) => factor.kind !== "tool" || item !== factor.name).toSorted(), + skills: contract.skills + .filter((item) => factor.kind !== "skill" || item.name !== factor.name) + .toSorted((left, right) => left.name.localeCompare(right.name)), + budget: contract.budget, + ...(seed ? { seed: contract.seed } : {}), + intervention: contract.intervention, + contamination: contract.contamination, + } + } + + function validate(left: HarnessContract.Info, right: HarnessContract.Info, factor: Factor) { + if (left.sessionID === right.sessionID || left.runID === right.runID) { + throw new Error(`Ablation baseline and arm must be separate runs`) + } + if (left.seed !== right.seed) throw new Error(`Ablation pairs must use the same seed`) + if (!same(context(left, factor, true), context(right, factor, true))) { + throw new Error(`Ablation pair differs outside the declared factor`) + } + const baseline = value(left, factor) + const arm = value(right, factor) + if (same(baseline, arm)) throw new Error(`Ablation factor does not change between baseline and arm`) + return { baseline: digest(baseline), arm: digest(arm) } + } + + export async function initialize(input: Initialize) { + const value = Initialize.parse(input) + const frozenAt = Date.now() + const pairs = await Promise.all( + value.pairs.map(async (pair) => { + const [baseline, arm] = await Promise.all([ + HarnessAdapter.authorize(pair.baseline.sessionID, pair.baseline.evaluatorToken), + HarnessAdapter.authorize(pair.arm.sessionID, pair.arm.evaluatorToken), + ]) + const factor = validate(baseline, arm, value.factor) + return { baseline, arm, factor } + }), + ) + const seeds = pairs.map((pair) => pair.baseline.seed) + if (new Set(seeds).size !== seeds.length) throw new Error(`Ablation pairs require distinct seeds`) + const sessions = pairs.flatMap((pair) => [pair.baseline.sessionID, pair.arm.sessionID]) + if (new Set(sessions).size !== sessions.length) throw new Error(`Ablation runs cannot be reused across pairs`) + const histories = await Promise.all(sessions.map((sessionID) => HarnessEvaluation.list(sessionID))) + if (histories.some((items) => items.length)) { + throw new Error(`Ablation plans must be frozen before any paired evaluation is recorded`) + } + const first = pairs[0]! + if (first.baseline.benchmark.split !== "held_out" && first.baseline.benchmark.split !== "release") { + throw new Error(`Ablation evidence requires a held-out or release split`) + } + if (first.baseline.benchmark.direction === "pass" || !first.baseline.benchmark.direction) { + throw new Error(`Ablation evidence requires a numeric benchmark direction`) + } + if (!first.baseline.benchmark.metric || !first.baseline.benchmark.evaluatorVersion) { + throw new Error(`Ablation evidence requires a bound metric and evaluator version`) + } + if (pairs.some((pair) => pair.factor.baseline !== first.factor.baseline || pair.factor.arm !== first.factor.arm)) { + throw new Error(`Every ablation pair must use the same baseline and arm factor values`) + } + const contexts = pairs.map((pair) => digest(context(pair.baseline, value.factor, false))) + if (new Set(contexts).size !== 1) throw new Error(`Ablation pairs differ outside their seeds`) + if (pairs.some((pair) => frozenAt < pair.baseline.createdAt || frozenAt < pair.arm.createdAt)) { + throw new Error(`Ablation plan predates one of its bound run contracts`) + } + const payload = { + schemaVersion: 1 as const, + studyID: value.studyID, + factor: value.factor, + baselineValueSHA256: first.factor.baseline, + armValueSHA256: first.factor.arm, + contextSHA256: contexts[0]!, + benchmark: { + name: first.baseline.benchmark.name, + version: first.baseline.benchmark.version, + taskID: first.baseline.benchmark.taskID, + split: first.baseline.benchmark.split as "held_out" | "release", + evaluator: first.baseline.benchmark.evaluator, + evaluatorVersion: first.baseline.benchmark.evaluatorVersion, + metric: first.baseline.benchmark.metric, + direction: first.baseline.benchmark.direction, + }, + minEffect: value.minEffect, + maxPairRegression: value.maxPairRegression, + pairs: pairs.map((pair) => ({ + seed: pair.baseline.seed, + baseline: { + sessionID: pair.baseline.sessionID, + runID: pair.baseline.runID, + contractFingerprint: HarnessContract.fingerprint(pair.baseline), + }, + arm: { + sessionID: pair.arm.sessionID, + runID: pair.arm.runID, + contractFingerprint: HarnessContract.fingerprint(pair.arm), + }, + })), + createdAt: frozenAt, + } + const plan = Plan.parse({ ...payload, planID: digest(payload) }) + const expected = State.parse({ schemaVersion: 1, plan }) + await JsonStore.update(file(plan.planID), (data) => { + if (!Object.keys(data).length) return expected + const current = State.parse(data) + if (current.plan.planID === plan.planID) return current + throw new Error(`Ablation plan is immutable once initialized`) + }) + return read(plan.planID) + } + + export async function read(planID: string) { + const data = await JsonStore.read(file(Hash.parse(planID))) + const parsed = State.safeParse(data) + return parsed.success ? parsed.data : null + } + + async function outcome(sessionID: string) { + const evaluations = await HarnessEvaluation.list(sessionID) + const search = await HarnessSearch.read(sessionID).catch(() => undefined) + const candidate = search?.bestID + ? evaluations.findLast( + (item) => + item.subject?.type === "candidate" && item.subject.id === search.bestID && HarnessEvaluation.final(item), + ) + : undefined + return ( + candidate ?? + evaluations.findLast((item) => !item.subject && HarnessEvaluation.final(item)) ?? + evaluations.findLast(HarnessEvaluation.final) + ) + } + + function critical(pairs: number) { + const values = [ + 0, 0, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, + 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, + ] + return values[Math.min(31, pairs - 1)]! + } + + export async function assess(planID: string, input: Assess) { + const value = Assess.parse(input) + const id = Hash.parse(planID) + const state = await read(id) + if (!state) throw new Error(`Unknown ablation plan ${id}`) + const expected = state.plan.pairs.flatMap((pair) => [pair.baseline.sessionID, pair.arm.sessionID]).toSorted() + const supplied = value.runs.map((run) => run.sessionID).toSorted() + if (!same(expected, supplied)) throw new Error(`Assessment credentials do not match every planned run`) + const tokens = new Map(value.runs.map((run) => [run.sessionID, run.evaluatorToken])) + const contracts = new Map( + await Promise.all( + expected.map(async (sessionID) => { + const contract = await HarnessAdapter.authorize(sessionID, tokens.get(sessionID)!) + return [sessionID, contract] as const + }), + ), + ) + for (const pair of state.plan.pairs) { + for (const run of [pair.baseline, pair.arm]) { + const contract = contracts.get(run.sessionID)! + if (HarnessContract.fingerprint(contract) !== run.contractFingerprint) { + throw new Error(`Ablation run contract changed after planning`) + } + } + } + const pairs = await Promise.all( + state.plan.pairs.map(async (pair) => { + const [baseline, arm] = await Promise.all([outcome(pair.baseline.sessionID), outcome(pair.arm.sessionID)]) + if (!baseline || !arm) throw new Error(`Every planned ablation run requires a final external evaluation`) + if (!baseline.recordedAt || !arm.recordedAt) { + throw new Error(`Ablation evidence requires server-timestamped evaluation receipts`) + } + if (baseline.recordedAt < state.plan.createdAt || arm.recordedAt < state.plan.createdAt) { + throw new Error(`Ablation evaluation was received before its plan was frozen`) + } + const valid = + HarnessEvaluation.verified(baseline) && + HarnessEvaluation.verified(arm) && + baseline.score !== undefined && + arm.score !== undefined + const effect = valid + ? state.plan.benchmark.direction === "maximize" + ? arm.score! - baseline.score! + : baseline.score! - arm.score! + : undefined + return { + seed: pair.seed, + baseline: { + sessionID: pair.baseline.sessionID, + runID: pair.baseline.runID, + status: baseline.status, + score: baseline.score, + evaluationSHA256: HarnessEvaluation.fingerprint(baseline), + evaluatedAt: baseline.evaluatedAt, + recordedAt: baseline.recordedAt, + }, + arm: { + sessionID: pair.arm.sessionID, + runID: pair.arm.runID, + status: arm.status, + score: arm.score, + evaluationSHA256: HarnessEvaluation.fingerprint(arm), + evaluatedAt: arm.evaluatedAt, + recordedAt: arm.recordedAt, + }, + effect, + } + }), + ) + const effects = pairs.flatMap((pair) => (pair.effect === undefined ? [] : [pair.effect])) + const mean = effects.length ? effects.reduce((sum, item) => sum + item, 0) / effects.length : undefined + const variance = + mean === undefined || effects.length < 2 + ? undefined + : effects.reduce((sum, item) => sum + (item - mean) ** 2, 0) / (effects.length - 1) + const deviation = variance === undefined ? undefined : Math.sqrt(variance) + const error = deviation === undefined ? undefined : deviation / Math.sqrt(effects.length) + const interval = + mean === undefined || error === undefined + ? undefined + : ([mean - critical(effects.length) * error, mean + critical(effects.length) * error] as const) + const regressions = effects.filter((effect) => effect < -state.plan.maxPairRegression).length + const complete = effects.length === pairs.length + const assessedAt = Date.now() + if (assessedAt < Math.max(...pairs.flatMap((pair) => [pair.baseline.recordedAt, pair.arm.recordedAt]))) { + throw new Error(`Ablation assessment predates one of its paired evaluations`) + } + const verdict = (() => { + if (!complete || regressions) return "rejected" as const + if (mean! > state.plan.minEffect && interval![0] > state.plan.minEffect) return "supported" as const + if (interval![1] <= state.plan.minEffect) return "rejected" as const + return "inconclusive" as const + })() + const stable = { + schemaVersion: 1 as const, + planID: state.plan.planID, + studyID: state.plan.studyID, + factor: state.plan.factor, + pairs, + statistics: { + pairs: pairs.length, + validPairs: effects.length, + meanEffect: mean, + standardDeviation: deviation, + standardError: error, + confidence95: interval, + regressions, + minEffect: state.plan.minEffect, + maxPairRegression: state.plan.maxPairRegression, + }, + verdict, + } + const receipt = Receipt.parse({ + ...stable, + receiptID: digest(stable), + assessedAt, + }) + await JsonStore.update(file(state.plan.planID), (data) => { + const current = State.parse(data) + if (!current.receipt) return State.parse({ ...current, receipt }) + if (current.receipt.receiptID === receipt.receiptID) return current + throw new Error(`Ablation assessment is immutable once recorded`) + }) + return read(state.plan.planID) + } +} diff --git a/backend/cli/src/session/harness/adaptation.ts b/backend/cli/src/session/harness/adaptation.ts new file mode 100644 index 00000000..aba65897 --- /dev/null +++ b/backend/cli/src/session/harness/adaptation.ts @@ -0,0 +1,194 @@ +import z from "zod" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" + +export namespace HarnessAdaptation { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + + export const Event = z + .object({ + candidateID: Hash, + island: z.number().int().nonnegative(), + revision: z.number().int().nonnegative(), + status: HarnessEvaluation.Status, + score: z.number().finite().optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.status !== "passed" || value.score !== undefined) return + ctx.addIssue({ code: "custom", path: ["score"], message: "A passing adaptation event requires a score" }) + }) + export type Event = z.infer + + export const Signal = z + .object({ + island: z.number().int().nonnegative(), + visits: z.number().int().nonnegative(), + decayedVisits: z.number().finite().nonnegative(), + improvements: z.number().int().nonnegative(), + accumulatedImprovement: z.number().finite().nonnegative(), + decayedReward: z.number().finite().nonnegative(), + rewardMean: z.number().finite().nonnegative(), + intensity: z.number().finite().min(0).max(1), + ucb: z.number().finite().nonnegative(), + bestID: Hash.optional(), + bestFitness: z.number().finite().optional(), + }) + .strict() + export type Signal = z.infer + + export const Summary = z + .object({ + protocolVersion: z.literal("adaptive-search-v1"), + policySHA256: Hash, + events: z.number().int().nonnegative(), + stalled: z.number().int().nonnegative(), + selectedIsland: z.number().int().nonnegative().optional(), + globalStagnation: z.boolean(), + islands: z.array(Signal).min(1).max(4), + }) + .strict() + export type Summary = z.infer + + export const Control = z + .object({ + protocolVersion: z.literal("adaptive-search-v1"), + policySHA256: Hash, + eventCount: z.number().int().nonnegative(), + stalled: z.number().int().nonnegative(), + selectedIsland: z.number().int().nonnegative().optional(), + targetIsland: z.number().int().nonnegative(), + visits: z.number().int().nonnegative(), + accumulatedImprovement: z.number().finite().nonnegative(), + rewardMean: z.number().finite().nonnegative(), + intensity: z.number().finite().min(0).max(1), + draw: z.number().finite().min(0).max(1), + explore: z.boolean(), + globalStagnation: z.boolean(), + }) + .strict() + export type Control = z.infer + + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const fingerprint = (policy: HarnessContract.Search) => digest(HarnessContract.Search.parse(policy)) + + export function derive(input: { + policy: HarnessContract.Search + direction: "maximize" | "minimize" + islands: number + events: Event[] + }): Summary { + const policy = HarnessContract.Search.parse(input.policy) + const count = z.number().int().min(1).max(4).parse(input.islands) + const events = input.events + .map((event) => Event.parse(event)) + .toSorted((a, b) => a.revision - b.revision || a.candidateID.localeCompare(b.candidateID)) + if (new Set(events.map((event) => event.candidateID)).size !== events.length) { + throw new Error(`Adaptive search may consume at most one final event per candidate`) + } + if (events.some((event) => event.island >= count)) { + throw new Error(`Adaptive search event references an unknown island`) + } + const state = Array.from({ length: count }, (_, island) => ({ + island, + visits: 0, + decayedVisits: 0, + improvements: 0, + accumulatedImprovement: 0, + decayedReward: 0, + bestID: undefined as string | undefined, + bestFitness: undefined as number | undefined, + })) + const global = { best: undefined as number | undefined, stalled: 0 } + for (const event of events) { + const item = state[event.island]! + const fitness = + event.status === "passed" ? (input.direction === "maximize" ? event.score! : -event.score!) : undefined + const local = item.bestFitness + const gain = fitness === undefined || local === undefined ? 0 : Math.max(fitness - local, 0) + const delta = local === undefined ? 0 : Math.min(gain / Math.max(Math.abs(local), policy.signal.epsilon), 1) + const globalGain = global.best === undefined ? 0 : gain / Math.max(Math.abs(global.best), policy.signal.epsilon) + const reward = Math.min(Math.max(globalGain, 0), 1) + item.visits += 1 + item.decayedVisits = policy.signal.decay * item.decayedVisits + 1 + item.accumulatedImprovement = + policy.signal.decay * item.accumulatedImprovement + (1 - policy.signal.decay) * delta ** 2 + item.decayedReward = policy.signal.decay * item.decayedReward + reward + if (fitness !== undefined && (local === undefined || fitness > local)) { + item.bestFitness = fitness + item.bestID = event.candidateID + if (local !== undefined) item.improvements += 1 + } + const improved = fitness !== undefined && (global.best === undefined || fitness > global.best) + global.stalled = improved ? 0 : global.stalled + 1 + if (improved) global.best = fitness + } + const total = Math.max( + 1, + state.reduce((sum, item) => sum + item.visits, 0), + ) + const islands = state.map((item) => { + const rewardMean = item.decayedVisits ? item.decayedReward / item.decayedVisits : 0 + const intensity = + policy.local.minIntensity + + (policy.local.maxIntensity - policy.local.minIntensity) / + (1 + Math.sqrt(item.accumulatedImprovement + policy.signal.epsilon)) + const bonus = item.visits ? policy.global.exploration * Math.sqrt(Math.log(total + 1) / item.visits) : 0 + return Signal.parse({ ...item, rewardMean, intensity, ucb: rewardMean + bonus }) + }) + const active = islands.filter((item) => item.visits) + const cold = active.filter((item) => item.visits < policy.global.minVisits) + const selected = (cold.length ? cold : active).toSorted( + (a, b) => + (cold.length ? a.visits - b.visits : b.ucb - a.ucb) || b.rewardMean - a.rewardMean || a.island - b.island, + )[0] + return Summary.parse({ + protocolVersion: policy.protocolVersion, + policySHA256: fingerprint(policy), + events: events.length, + stalled: global.stalled, + selectedIsland: selected?.island, + globalStagnation: + active.length > 0 && + global.stalled >= policy.stagnation.patience && + active.every((item) => item.accumulatedImprovement <= policy.stagnation.maxSignal), + islands, + }) + } + + export function control(input: { + policy: HarnessContract.Search + direction: "maximize" | "minimize" + islands: number + events: Event[] + targetIsland: number + key: string + }): Control { + const summary = derive(input) + const signal = summary.islands[input.targetIsland] + if (!signal) throw new Error(`Adaptive search target references an unknown island`) + const token = digest({ + key: input.key, + policySHA256: summary.policySHA256, + island: signal.island, + events: summary.events, + }) + const draw = Number.parseInt(token.slice(0, 13), 16) / 0xfffffffffffff + return Control.parse({ + protocolVersion: summary.protocolVersion, + policySHA256: summary.policySHA256, + eventCount: summary.events, + stalled: summary.stalled, + selectedIsland: summary.selectedIsland, + targetIsland: signal.island, + visits: signal.visits, + accumulatedImprovement: signal.accumulatedImprovement, + rewardMean: signal.rewardMean, + intensity: signal.intensity, + draw, + explore: draw < signal.intensity, + globalStagnation: summary.globalStagnation, + }) + } +} diff --git a/backend/cli/src/session/harness/adapter.ts b/backend/cli/src/session/harness/adapter.ts new file mode 100644 index 00000000..7f79300e --- /dev/null +++ b/backend/cli/src/session/harness/adapter.ts @@ -0,0 +1,730 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { timingSafeEqual } from "@/util/timing-safe" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" +import { HarnessDomain } from "./domain" +import { HarnessMemory } from "./memory" +import { HarnessPack } from "./pack" +import { HarnessSearch } from "./search" +import { HarnessWorld } from "./world" + +export namespace HarnessAdapter { + const Token = z.string().min(32).max(1_024) + + export const Task = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + benchmark: z.string().min(1).max(120), + title: z.string().min(1).max(240).optional(), + family: HarnessContract.Family.default("custom"), + task: z.string().min(1).max(4_000).optional(), + version: z.string().min(1).max(120), + taskID: z.string().min(1).max(500), + split: HarnessContract.Split, + evaluator: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "external"]), + token: Token, + }) + .strict(), + objective: z.string().min(1).max(4_000), + profile: HarnessContract.Profile.optional(), + search: z.enum(["adaptive", "static"]).optional(), + orchestration: HarnessContract.Orchestration.optional(), + audit: HarnessContract.Audit.optional(), + failureDiscovery: HarnessContract.FailureDiscovery.optional(), + integrity: HarnessContract.Integrity.optional(), + evolution: HarnessContract.Evolution.optional(), + metaHarness: z + .object({ + protocol: HarnessContract.MetaHarness, + token: Token, + }) + .strict() + .optional(), + interventions: HarnessContract.Interventions.optional(), + simulation: HarnessContract.Simulation.optional(), + evaluatorAudit: z + .object({ + protocol: HarnessContract.EvaluatorAudit, + token: Token, + }) + .strict() + .optional(), + semanticAudit: z + .object({ + protocol: HarnessContract.SemanticAudit, + token: Token, + }) + .strict() + .optional(), + synthesis: HarnessContract.ScientificSynthesis.optional(), + autonomy: HarnessContract.HumanAIAutonomy.optional(), + formalProof: HarnessContract.FormalProof.optional(), + replication: HarnessContract.Replication.optional(), + confirmation: z + .object({ + protocol: HarnessContract.Confirmation, + token: Token, + }) + .strict() + .optional(), + packs: z + .array(HarnessPack.Id) + .max(HarnessPack.Id.options.length) + .refine((items) => new Set(items).size === items.length, "Harness packs must be unique") + .optional(), + metric: z + .object({ + name: z.string().min(1).max(200).optional(), + direction: z.enum(["maximize", "minimize", "pass"]), + target: z.number().finite().optional(), + }) + .strict() + .default({ direction: "pass" }), + objectives: HarnessContract.Objectives.optional(), + objectiveAudit: HarnessContract.ObjectiveAudit.optional(), + fidelities: HarnessContract.FidelityPlan.optional(), + model: z + .object({ + provider: z.string().min(1), + name: z.string().min(1), + effort: z.string().min(1).optional(), + }) + .strict(), + tools: z.array(z.string().min(1)).max(256).default([]), + skills: z + .array( + z + .object({ + name: z.string().min(1), + version: z.string().min(1).optional(), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }) + .strict(), + ) + .max(256) + .default([]), + budget: z + .object({ + wallTimeMs: z.number().int().positive().optional(), + steps: z.number().int().positive().optional(), + candidates: z.number().int().positive().optional(), + tokens: z.number().int().positive().optional(), + costUSD: z.number().nonnegative().optional(), + cpuHours: z.number().nonnegative().optional(), + gpuHours: z.number().nonnegative().optional(), + }) + .strict(), + seed: z.number().int(), + intervention: z.enum(["autonomous", "human_reprompted"]), + contamination: z + .object({ + policy: z.string().min(1).max(2_000), + hiddenTestsAccessible: z.literal(false), + publicDataCutoff: z.string().min(1).max(120).optional(), + }) + .strict(), + createdAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.evaluatorAudit && value.evaluatorAudit.token === value.evaluator.token) { + ctx.addIssue({ + code: "custom", + path: ["evaluatorAudit", "token"], + message: "Evaluator and independent auditor capabilities must differ", + }) + } + if (value.semanticAudit && value.semanticAudit.token === value.evaluator.token) { + ctx.addIssue({ + code: "custom", + path: ["semanticAudit", "token"], + message: "Evaluator and semantic reviewer capabilities must differ", + }) + } + if (value.semanticAudit && value.evaluatorAudit && value.semanticAudit.token === value.evaluatorAudit.token) { + ctx.addIssue({ + code: "custom", + path: ["semanticAudit", "token"], + message: "Evaluator auditor and semantic reviewer capabilities must differ", + }) + } + if (value.confirmation && value.confirmation.token === value.evaluator.token) { + ctx.addIssue({ + code: "custom", + path: ["confirmation", "token"], + message: "Optimization and claim evaluator capabilities must differ", + }) + } + if (value.metaHarness && value.metaHarness.token === value.evaluator.token) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness", "token"], + message: "Optimization evaluator and meta-harness qualifier capabilities must differ", + }) + } + if (value.metaHarness && value.evaluatorAudit && value.metaHarness.token === value.evaluatorAudit.token) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness", "token"], + message: "Evaluator auditor and meta-harness qualifier capabilities must differ", + }) + } + if (value.metaHarness && value.semanticAudit && value.metaHarness.token === value.semanticAudit.token) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness", "token"], + message: "Semantic reviewer and meta-harness qualifier capabilities must differ", + }) + } + if (value.metaHarness && value.confirmation && value.metaHarness.token === value.confirmation.token) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness", "token"], + message: "Claim evaluator and meta-harness qualifier capabilities must differ", + }) + } + if (value.confirmation && value.evaluatorAudit && value.confirmation.token === value.evaluatorAudit.token) { + ctx.addIssue({ + code: "custom", + path: ["confirmation", "token"], + message: "Claim evaluator and independent auditor capabilities must differ", + }) + } + if (value.confirmation && value.semanticAudit && value.confirmation.token === value.semanticAudit.token) { + ctx.addIssue({ + code: "custom", + path: ["confirmation", "token"], + message: "Claim evaluator and semantic reviewer capabilities must differ", + }) + } + if (Boolean(value.objectives?.length) !== Boolean(value.objectiveAudit)) { + ctx.addIssue({ + code: "custom", + path: ["objectiveAudit"], + message: "Secondary objectives require exactly one preflighted objective audit", + }) + } + }) + + export type Task = z.input + + export const Evaluation = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + candidateID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + stage: z.string().min(1).max(100).optional(), + simulationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + integrityReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evolutionReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + interventionReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evaluatorAuditReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + semanticReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + replicationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + auditReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + failureDiscoveryReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + synthesisReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + autonomyReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + proofReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + status: HarnessEvaluation.Status, + score: z.number().finite().optional(), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "An evaluation may contain at most 128 metrics") + .default({}), + checks: z.array(HarnessEvaluation.Check).max(128), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + usage: HarnessEvaluation.Usage.optional(), + evaluatedAt: z.number().int().positive(), + notes: z.string().max(8_000).optional(), + }) + .strict() + export type Evaluation = z.input + + const Binding = z + .object({ + schemaVersion: z.literal(1), + sessionID: z.string().min(1), + runID: z.string().min(1), + contractFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + tokenSHA256: z.string().regex(/^[a-f0-9]{64}$/), + evaluator: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + source: z.enum(["benchmark", "gate", "external"]), + }) + .strict(), + auditor: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + source: z.enum(["benchmark", "gate", "human", "external"]), + tokenSHA256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() + .optional(), + semantic: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + source: z.enum(["gate", "human", "external"]), + tokenSHA256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() + .optional(), + meta: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + promptSHA256: z.string().regex(/^[a-f0-9]{64}$/), + configSHA256: z.string().regex(/^[a-f0-9]{64}$/), + tokenSHA256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() + .optional(), + confirmation: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + source: z.enum(["benchmark", "gate", "external"]), + tokenSHA256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() + .optional(), + createdAt: z.number().int().positive(), + }) + .strict() + type Binding = z.infer + + const root = path.join(Global.Path.data, "harness", "bindings") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const digest = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + + async function credential(sessionID: string): Promise { + const data = await JsonStore.read(file(sessionID)) + const parsed = Binding.safeParse(data) + if (!parsed.success) throw new Error(`No evaluator capability is bound to session ${sessionID}`) + return parsed.data + } + + export async function authorize(sessionID: string, token: string) { + const [contract, binding] = await Promise.all([HarnessContract.read(sessionID), credential(sessionID)]) + if (!contract) throw new Error(`No harness contract is bound to session ${sessionID}`) + if (binding.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Evaluator capability does not match the bound harness contract`) + } + if (!timingSafeEqual(binding.tokenSHA256, digest(Token.parse(token)))) { + throw new Error(`Evaluator capability was rejected`) + } + return contract + } + + export async function authorizeAuditor(sessionID: string, token: string) { + const [contract, binding] = await Promise.all([HarnessContract.read(sessionID), credential(sessionID)]) + if (!contract || !contract.evaluatorAudit || !binding.auditor) { + throw new Error(`No independent evaluator auditor is bound to session ${sessionID}`) + } + if (binding.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Evaluator auditor capability does not match the bound harness contract`) + } + if (!timingSafeEqual(binding.auditor.tokenSHA256, digest(Token.parse(token)))) { + throw new Error(`Evaluator auditor capability was rejected`) + } + const auditor = contract.evaluatorAudit.auditor + if ( + binding.auditor.name !== auditor.name || + binding.auditor.version !== auditor.version || + binding.auditor.source !== auditor.source + ) { + throw new Error(`Evaluator auditor identity does not match the bound harness contract`) + } + return contract + } + + export async function authorizeSemantic(sessionID: string, token: string) { + const [contract, binding] = await Promise.all([HarnessContract.read(sessionID), credential(sessionID)]) + if (!contract || !contract.semanticAudit || !binding.semantic) { + throw new Error(`No independent semantic reviewer is bound to session ${sessionID}`) + } + if (binding.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Semantic reviewer capability does not match the bound harness contract`) + } + if (!timingSafeEqual(binding.semantic.tokenSHA256, digest(Token.parse(token)))) { + throw new Error(`Semantic reviewer capability was rejected`) + } + if ( + JSON.stringify(binding.semantic) !== + JSON.stringify({ + ...contract.semanticAudit.reviewer, + tokenSHA256: binding.semantic.tokenSHA256, + }) + ) { + throw new Error(`Semantic reviewer identity does not match the bound harness contract`) + } + return contract + } + + export async function authorizeConfirmation(sessionID: string, token: string) { + const [contract, binding] = await Promise.all([HarnessContract.read(sessionID), credential(sessionID)]) + if (!contract?.confirmation || !binding.confirmation) { + throw new Error(`No sealed claim evaluator is bound to session ${sessionID}`) + } + if (binding.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Claim evaluator capability does not match the bound harness contract`) + } + if (!timingSafeEqual(binding.confirmation.tokenSHA256, digest(Token.parse(token)))) { + throw new Error(`Claim evaluator capability was rejected`) + } + if ( + JSON.stringify(binding.confirmation) !== + JSON.stringify({ + ...contract.confirmation.claim.evaluator, + tokenSHA256: binding.confirmation.tokenSHA256, + }) + ) { + throw new Error(`Claim evaluator identity does not match the bound harness contract`) + } + return contract + } + + export async function authorizeMeta(sessionID: string, token: string) { + const [contract, binding] = await Promise.all([HarnessContract.read(sessionID), credential(sessionID)]) + if (!contract?.metaHarness || !binding.meta) { + throw new Error(`No meta-harness qualifier is bound to session ${sessionID}`) + } + if (binding.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Meta-harness qualifier capability does not match the bound harness contract`) + } + if (!timingSafeEqual(binding.meta.tokenSHA256, digest(Token.parse(token)))) { + throw new Error(`Meta-harness qualifier capability was rejected`) + } + if ( + JSON.stringify(binding.meta) !== + JSON.stringify({ + ...contract.metaHarness.judge, + tokenSHA256: binding.meta.tokenSHA256, + }) + ) { + throw new Error(`Meta-harness qualifier identity does not match the bound harness contract`) + } + return contract + } + + export async function bind(input: Task) { + const task = Task.parse(input) + const search = + task.search === "adaptive" || + task.evolution !== undefined || + task.metaHarness !== undefined || + task.interventions !== undefined || + task.confirmation !== undefined || + (task.budget.candidates !== undefined && task.metric.direction !== "pass") + const profile = task.profile ?? (search ? "optimize" : task.simulation ? "numerical" : "react") + if (task.metric.direction !== "pass" && !task.metric.name) { + throw new Error(`A ${task.metric.direction} evaluation must declare its metric name`) + } + if (profile === "optimize" && task.budget.candidates === undefined) { + throw new Error(`An optimize run must declare a candidate budget`) + } + if (task.search && profile !== "optimize") { + throw new Error(`A search policy selection requires the optimize profile`) + } + const family: Record = { + data: ["statistics"], + biology: ["statistics", "biology"], + physics: ["physics"], + chemistry: ["chemistry"], + ml: ["ml"], + generalist: [], + custom: [], + } + const agent = + task.family === "biology" + ? "biology" + : task.family === "physics" + ? "physics" + : task.family === "ml" + ? "ml" + : "research" + const recommended = HarnessDomain.recommend({ + agent, + profile, + text: `${task.benchmark} ${task.title ?? ""} ${task.task ?? ""} ${task.objective}`, + }) + const packs = [ + ...(task.packs ?? family[task.family]), + ...recommended, + ...(task.formalProof ? (["formal"] as const) : []), + ].filter((pack, index, items) => items.indexOf(pack) === index) + if (task.simulation && !packs.some((pack) => ["physics", "pde", "chemistry"].includes(pack))) { + throw new Error(`A simulator validation contract requires a physics, PDE, or chemistry verification pack`) + } + const contract = HarnessContract.Info.parse({ + schemaVersion: 1, + runID: task.runID, + sessionID: task.sessionID, + objective: task.objective, + benchmark: { + name: task.benchmark, + title: task.title ?? task.benchmark, + family: task.family, + task: task.task ?? task.objective, + version: task.version, + taskID: task.taskID, + split: task.split, + evaluator: task.evaluator.name, + evaluatorVersion: task.evaluator.version, + evaluatorSource: task.evaluator.source, + fidelities: task.fidelities, + metric: task.metric.name, + direction: task.metric.direction, + target: task.metric.target, + objectives: task.objectives, + objectiveAudit: task.objectiveAudit, + }, + profile, + orchestration: task.orchestration, + search: + profile === "optimize" && task.metric.name && task.metric.direction !== "pass" && task.search !== "static" + ? HarnessContract.adaptiveSearch + : undefined, + audit: task.audit, + failureDiscovery: task.failureDiscovery, + integrity: task.integrity, + evolution: task.evolution, + metaHarness: task.metaHarness?.protocol, + interventions: task.interventions, + simulation: task.simulation, + evaluatorAudit: task.evaluatorAudit?.protocol, + semanticAudit: task.semanticAudit?.protocol, + synthesis: task.synthesis, + autonomy: task.autonomy, + formalProof: task.formalProof, + replication: task.replication, + confirmation: task.confirmation?.protocol, + packs, + model: task.model, + tools: task.tools, + skills: task.skills, + budget: task.budget, + seed: task.seed, + intervention: task.intervention, + contamination: task.contamination, + createdAt: task.createdAt, + }) + await HarnessContract.bind(contract) + const binding = Binding.parse({ + schemaVersion: 1, + sessionID: task.sessionID, + runID: task.runID, + contractFingerprint: HarnessContract.fingerprint(contract), + tokenSHA256: digest(task.evaluator.token), + evaluator: { + name: task.evaluator.name, + version: task.evaluator.version, + source: task.evaluator.source, + }, + auditor: task.evaluatorAudit + ? { + ...task.evaluatorAudit.protocol.auditor, + tokenSHA256: digest(task.evaluatorAudit.token), + } + : undefined, + semantic: task.semanticAudit + ? { + ...task.semanticAudit.protocol.reviewer, + tokenSHA256: digest(task.semanticAudit.token), + } + : undefined, + meta: task.metaHarness + ? { + ...task.metaHarness.protocol.judge, + tokenSHA256: digest(task.metaHarness.token), + } + : undefined, + confirmation: task.confirmation + ? { + ...task.confirmation.protocol.claim.evaluator, + tokenSHA256: digest(task.confirmation.token), + } + : undefined, + createdAt: task.createdAt, + }) + await JsonStore.update(file(task.sessionID), (data) => { + if (!Object.keys(data).length) return binding + const current = Binding.parse(data) + if (JSON.stringify(current) === JSON.stringify(binding)) return current + throw new Error(`Evaluator capability for session ${task.sessionID} is immutable once bound`) + }) + return contract + } + + export async function ingest(input: Evaluation) { + const value = Evaluation.parse(input) + const [contract, binding] = await Promise.all([ + authorize(value.sessionID, value.evaluatorToken), + credential(value.sessionID), + ]) + if (contract.runID !== value.runID || binding.runID !== value.runID) { + throw new Error(`Evaluation does not match the bound harness run`) + } + if (value.evaluatedAt < contract.createdAt) { + throw new Error(`Evaluation predates the bound harness contract`) + } + const fidelity = (() => { + if (!contract.benchmark.fidelities && value.stage === undefined) return undefined + if (!contract.benchmark.fidelities) throw new Error(`Evaluation stage is not declared by the contract`) + if (!value.stage) throw new Error(`Evaluation must name a fidelity stage`) + const stage = contract.benchmark.fidelities.find((item) => item.id === value.stage) + if (!stage) throw new Error(`Evaluation fidelity stage is not in the bound contract`) + return { stage: stage.id, final: stage.final } + })() + const stage = value.stage ? contract.benchmark.fidelities?.find((item) => item.id === value.stage) : undefined + if (stage?.maxWallTimeMs !== undefined && value.usage?.wallTimeMs === undefined) { + throw new Error(`Evaluation stage ${stage.id} must report wall-time usage`) + } + if (stage?.maxCostUSD !== undefined && value.usage?.costUSD === undefined) { + throw new Error(`Evaluation stage ${stage.id} must report cost usage`) + } + const wall = value.usage?.wallTimeMs + const cost = value.usage?.costUSD + if (stage?.maxWallTimeMs !== undefined && wall !== undefined && wall > stage.maxWallTimeMs) { + throw new Error(`Evaluation stage ${stage.id} exceeded its wall-time budget`) + } + if (stage?.maxCostUSD !== undefined && cost !== undefined && cost > stage.maxCostUSD) { + throw new Error(`Evaluation stage ${stage.id} exceeded its cost budget`) + } + const metric = contract.benchmark.metric + if ( + value.status === "passed" && + contract.benchmark.direction !== "pass" && + (value.score === undefined || metric === undefined || value.metrics[metric] === undefined) + ) { + throw new Error(`A passing numeric benchmark evaluation must report its bound score and metric`) + } + if (metric !== undefined && value.score !== undefined && value.metrics[metric] !== value.score) { + throw new Error(`Evaluation score does not match the bound ${metric} metric`) + } + if (value.status === "passed" && fidelity?.final !== false) { + const missing = contract.benchmark.objectives?.find((item) => value.metrics[item.metric] === undefined) + if (missing) throw new Error(`Passing evaluation is missing declared objective metric ${missing.metric}`) + } + if (value.candidateID) { + const search = await HarnessSearch.read(value.sessionID) + if (search.runID !== value.runID || !search.candidates[value.candidateID]) { + throw new Error(`Evaluation candidate does not exist in the bound search`) + } + } + const evaluation = HarnessEvaluation.Info.parse({ + schemaVersion: 1, + runID: value.runID, + sessionID: value.sessionID, + subject: value.candidateID ? { type: "candidate", id: value.candidateID } : undefined, + fidelity, + simulationReceiptID: value.simulationReceiptID, + integrityReceiptID: value.integrityReceiptID, + evolutionReceiptID: value.evolutionReceiptID, + interventionReceiptID: value.interventionReceiptID, + evaluatorAuditReceiptID: value.evaluatorAuditReceiptID, + semanticReceiptID: value.semanticReceiptID, + replicationReceiptID: value.replicationReceiptID, + auditReceiptID: value.auditReceiptID, + failureDiscoveryReceiptID: value.failureDiscoveryReceiptID, + synthesisReceiptID: value.synthesisReceiptID, + autonomyReceiptID: value.autonomyReceiptID, + proofReceiptID: value.proofReceiptID, + evaluator: binding.evaluator, + status: value.status, + score: value.score, + metrics: value.metrics, + checks: value.checks, + evidence: value.evidence, + usage: value.usage, + evaluatedAt: value.evaluatedAt, + notes: value.notes, + }) + const recorded = await HarnessEvaluation.record(evaluation) + await HarnessWorld.event({ + sessionID: value.sessionID, + type: + value.status === "failed" + ? "failure" + : value.status === "passed" && HarnessEvaluation.final(recorded) + ? "milestone" + : "evaluation", + summary: `${value.status} evaluation recorded for ${value.candidateID ?? value.runID}`, + evidenceRefs: value.evidence, + changed: true, + }).catch(() => undefined) + if (!value.candidateID) return { evaluation: recorded } + if (fidelity?.final === false) { + const search = await HarnessSearch.screen({ + sessionID: value.sessionID, + candidateID: value.candidateID, + evaluation: recorded, + }) + return { evaluation: recorded, search } + } + const search = await HarnessSearch.verify({ sessionID: value.sessionID, candidateID: value.candidateID }) + const memory = await HarnessMemory.capture({ + sessionID: value.sessionID, + candidateID: value.candidateID, + stage: "evaluation", + }) + return { evaluation: recorded, search, memory } + } +} diff --git a/backend/cli/src/session/harness/audit.ts b/backend/cli/src/session/harness/audit.ts new file mode 100644 index 00000000..dd001872 --- /dev/null +++ b/backend/cli/src/session/harness/audit.ts @@ -0,0 +1,926 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessAdapter } from "./adapter" +import { HarnessContract } from "./contract" +import { HarnessSearch } from "./search" + +export namespace HarnessAudit { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(500), + artifactSHA256: Hash, + }) + .strict() + export type Subject = z.infer + + export const Probe = z + .object({ + id: z.string().min(1).max(240), + commitment: Hash, + features: z.array(z.number().finite()).min(1).max(32), + stratum: z.string().min(1).max(120), + weight: z.number().positive().max(1_000).default(1), + priorLoss: z.number().min(0).max(1).default(0.5), + }) + .strict() + export type Probe = z.infer + + export const TransferProbe = z + .object({ + id: z.string().min(1).max(240), + commitment: Hash, + sourceLosses: z.array(z.number().min(0).max(1)).min(3).max(64), + stratum: z.string().min(1).max(120), + weight: z.number().positive().max(1_000).default(1), + }) + .strict() + export type TransferProbe = z.infer + + const InputProbe = z.union([Probe, TransferProbe]) + + export const Selection = z + .object({ + round: z.number().int().positive(), + selectedAt: z.number().int().positive(), + phase: z.enum(["calibration", "adaptive", "fallback"]).optional(), + acquisition: z + .object({ + posteriorLoss: z.number().min(0).max(1), + posteriorStd: z.number().nonnegative(), + failureUCB: z.number().min(0).max(1), + varianceReduction: z.number().nonnegative(), + diversity: z.number().min(0).max(1), + coverage: z.number().min(0).max(1), + score: z.number().finite(), + }) + .strict(), + }) + .strict() + export type Selection = z.infer + + export const Observation = z + .object({ + loss: z.number().min(0).max(1), + failure: z.boolean(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + note: z.string().max(4_000).optional(), + evaluatedAt: z.number().int().positive(), + }) + .strict() + export type Observation = z.infer + + export const Entry = Probe.extend({ + features: z.array(z.number().finite()).min(1).max(64), + sourceLosses: z.array(z.number().min(0).max(1)).min(3).max(64).optional(), + selection: Selection.optional(), + observation: Observation.optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.observation && !value.selection) { + ctx.addIssue({ code: "custom", path: ["observation"], message: "An observation requires a selection" }) + } + }) + export type Entry = z.infer + + export const Transfer = z + .object({ + status: z.enum(["not_configured", "calibrating", "accepted", "rejected"]), + observed: z.number().int().nonnegative(), + required: z.number().int().nonnegative(), + meanAbsoluteError: z.number().min(0).max(1).optional(), + threshold: z.number().positive().max(1).optional(), + }) + .strict() + export type Transfer = z.infer + + export const Estimate = z + .object({ + observed: z.number().int().nonnegative(), + failures: z.number().int().nonnegative(), + meanLoss: z.number().min(0).max(1), + standardDeviation: z.number().nonnegative(), + lower95: z.number().min(0).max(1), + upper95: z.number().min(0).max(1), + abstain: z.boolean(), + effectivePoolSize: z.number().positive(), + stratumCoverage: z.number().min(0).max(1), + transfer: Transfer.default({ status: "not_configured", observed: 0, required: 0 }), + }) + .strict() + export type Estimate = z.infer + + export const Stop = z.enum(["budget_exhausted", "precision_reached", "failure_target_reached", "pool_exhausted"]) + export type Stop = z.infer + + export const State = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.enum(["active-audit-v1", "proactive-audit-v2"]), + auditID: Hash, + runID: z.string().min(1), + sessionID: z.string().min(1), + evaluator: z.string().min(1), + contractFingerprint: Hash, + poolFingerprint: Hash, + subject: Subject, + config: HarnessContract.Audit, + status: z.enum(["active", "completed"]), + stopReason: Stop.optional(), + pool: z.record(z.string(), Entry), + order: z.array(z.string().min(1)), + estimate: Estimate, + revision: z.number().int().nonnegative(), + createdAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.order).size !== value.order.length || value.order.length !== Object.keys(value.pool).length) { + ctx.addIssue({ code: "custom", path: ["order"], message: "Audit order must exactly cover a unique probe pool" }) + } + const dimensions = new Set(value.order.map((id) => value.pool[id]?.features.length)) + if (dimensions.size !== 1) { + ctx.addIssue({ code: "custom", path: ["pool"], message: "Audit probe features must share one dimension" }) + } + const commitments = value.order.flatMap((id) => (value.pool[id] ? [value.pool[id]!.commitment] : [])) + if (new Set(commitments).size !== commitments.length) { + ctx.addIssue({ code: "custom", path: ["pool"], message: "Audit probe commitments must be unique" }) + } + const selections = value.order.flatMap((id) => { + const selection = value.pool[id]?.selection + return selection ? [selection.round] : [] + }) + if (new Set(selections).size !== selections.length) { + ctx.addIssue({ code: "custom", path: ["pool"], message: "Audit selection rounds must be unique" }) + } + const pending = value.order.filter((id) => value.pool[id]?.selection && !value.pool[id]?.observation) + if (pending.length > 1) { + ctx.addIssue({ code: "custom", path: ["pool"], message: "Only one audit probe may be pending" }) + } + for (const id of value.order) { + if (value.pool[id]) continue + ctx.addIssue({ code: "custom", path: ["pool", id], message: "Audit probe is missing" }) + } + const expected = value.config.transfer ? "proactive-audit-v2" : "active-audit-v1" + if (value.protocolVersion !== expected) { + ctx.addIssue({ + code: "custom", + path: ["protocolVersion"], + message: "Audit protocol does not match its contract", + }) + } + for (const id of value.order) { + const entry = value.pool[id] + if (!entry) continue + if (!value.config.transfer && !entry.sourceLosses) continue + if (!value.config.transfer || !entry.sourceLosses) { + ctx.addIssue({ code: "custom", path: ["pool", id], message: "Audit transfer probe shape is inconsistent" }) + continue + } + if (entry.sourceLosses.length !== value.config.transfer.sourceModels.length) { + ctx.addIssue({ + code: "custom", + path: ["pool", id, "sourceLosses"], + message: "Audit source-loss dimension does not match the frozen source models", + }) + continue + } + const priorLoss = entry.sourceLosses.reduce((sum, loss) => sum + loss, 0) / entry.sourceLosses.length + const features = entry.sourceLosses.map( + (loss) => (loss - priorLoss) / Math.sqrt(Math.max(entry.sourceLosses!.length - 1, 1)), + ) + if (entry.priorLoss !== priorLoss || JSON.stringify(entry.features) !== JSON.stringify(features)) { + ctx.addIssue({ + code: "custom", + path: ["pool", id], + message: "Audit score-history prior was not derived by the backend", + }) + } + } + }) + export type State = z.infer + + const ReceiptBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("proactive-audit-receipt-v1"), + receiptID: Hash, + auditID: Hash, + runID: z.string().min(1), + sessionID: z.string().min(1), + contractFingerprint: Hash, + poolFingerprint: Hash, + subject: Subject, + config: HarnessContract.Audit, + stopReason: Stop, + estimate: Estimate, + revision: z.number().int().positive(), + qualified: z.boolean(), + completedAt: z.number().int().positive(), + sealedAt: z.number().int().positive(), + }) + .strict() + + export const Receipt = ReceiptBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + if (digest(stable) !== value.receiptID) { + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Active audit receipt content hash is invalid" }) + } + const qualified = + value.config.mode !== "failure" && value.estimate.transfer.status === "accepted" && !value.estimate.abstain + if (value.qualified !== qualified) { + ctx.addIssue({ code: "custom", path: ["qualified"], message: "Active audit receipt qualification drifted" }) + } + if (value.completedAt > value.sealedAt) { + ctx.addIssue({ code: "custom", path: ["sealedAt"], message: "Active audit receipt predates completion" }) + } + const auditID = digest({ + contractFingerprint: value.contractFingerprint, + subject: value.subject, + poolFingerprint: value.poolFingerprint, + config: value.config, + }) + if (value.auditID !== auditID) { + ctx.addIssue({ code: "custom", path: ["auditID"], message: "Active audit receipt identity is invalid" }) + } + }) + export type Receipt = z.infer + + export const Initialize = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: z.string().min(32).max(1_024), + subject: Subject, + probes: z.array(InputProbe).min(2).max(2_000), + }) + .strict() + export type Initialize = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: z.string().min(32).max(1_024), + }) + .strict() + export type Access = z.infer + + export const Observe = Access.extend({ + probeID: z.string().min(1).max(240), + loss: z.number().min(0).max(1), + failure: z.boolean(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + note: z.string().max(4_000).optional(), + }).strict() + export type Observe = z.infer + + const root = path.join(Global.Path.data, "harness", "audits") + const receipts = path.join(Global.Path.data, "harness", "audit-receipts") + const bases = new Map() + const file = (sessionID: string, auditID: string) => + path.join(root, encodeURIComponent(sessionID), `${encodeURIComponent(auditID)}.json`) + const receiptFile = (receiptID: string) => path.join(receipts, `${receiptID}.json`) + const clamp = (value: number) => Math.max(0, Math.min(1, value)) + const dot = (left: number[], right: number[]) => left.reduce((sum, value, index) => sum + value * right[index]!, 0) + const kernel = (left: number[], right: number[], lengthscale: number) => + Math.exp( + -left.reduce((sum, value, index) => sum + (value - right[index]!) ** 2, 0) / (2 * lengthscale * lengthscale), + ) + + function derive(probe: TransferProbe): Entry { + const priorLoss = probe.sourceLosses.reduce((sum, loss) => sum + loss, 0) / probe.sourceLosses.length + const features = probe.sourceLosses.map( + (loss) => (loss - priorLoss) / Math.sqrt(Math.max(probe.sourceLosses.length - 1, 1)), + ) + return Entry.parse({ ...probe, features, priorLoss }) + } + + function committed(entry: Entry, transfer: boolean) { + if (transfer) { + return TransferProbe.parse({ + id: entry.id, + commitment: entry.commitment, + sourceLosses: entry.sourceLosses, + stratum: entry.stratum, + weight: entry.weight, + }) + } + return Probe.parse({ + id: entry.id, + commitment: entry.commitment, + features: entry.features, + stratum: entry.stratum, + weight: entry.weight, + priorLoss: entry.priorLoss, + }) + } + + function drift(state: State) { + const probes = state.order.map((id) => committed(state.pool[id]!, Boolean(state.config.transfer))) + if (digest(probes) !== state.poolFingerprint) return `Active audit probe pool failed its commitment` + const auditID = digest({ + contractFingerprint: state.contractFingerprint, + subject: state.subject, + poolFingerprint: state.poolFingerprint, + config: state.config, + }) + if (auditID !== state.auditID) return `Active audit identity failed its commitment` + if (JSON.stringify(state.estimate) !== JSON.stringify(summarize(state))) { + return `Active audit estimate does not match its backend-derived posterior` + } + } + + function parse(input: Record) { + const state = State.parse(input) + const error = drift(state) + if (error) throw new Error(error) + return state + } + + function match(state: State, contract: HarnessContract.Info) { + if (state.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Active audit does not match the bound harness contract`) + } + if (state.evaluator !== contract.benchmark.evaluator) { + throw new Error(`Active audit evaluator does not match the bound harness contract`) + } + } + + function cholesky(matrix: number[][]) { + const output = matrix.map((row) => row.map(() => 0)) + for (const i of matrix.keys()) { + for (const j of matrix.keys()) { + if (j > i) break + const sum = output[i]!.slice(0, j).reduce((total, value, index) => total + value * output[j]![index]!, 0) + const value = + i === j ? Math.sqrt(Math.max(matrix[i]![i]! - sum, 1e-12)) : (matrix[i]![j]! - sum) / output[j]![j]! + if (!Number.isFinite(value)) throw new Error(`Audit kernel is not numerically stable`) + output[i]![j] = value + } + } + return output + } + + function forward(matrix: number[][], target: number[]) { + const output: number[] = [] + for (const i of target.keys()) { + const sum = matrix[i]!.slice(0, i).reduce((total, value, index) => total + value * output[index]!, 0) + output.push((target[i]! - sum) / matrix[i]![i]!) + } + return output + } + + function solve(matrix: number[][], target: number[]) { + const lower = forward(matrix, target) + const output = target.map(() => 0) + for (const i of [...target.keys()].toReversed()) { + const sum = target + .slice(i + 1) + .reduce((total, _, offset) => total + matrix[i + offset + 1]![i]! * output[i + offset + 1]!, 0) + output[i] = (lower[i]! - sum) / matrix[i]![i]! + } + return output + } + + function scale(entries: Entry[]) { + const dimensions = entries[0]!.features.map((_, index) => { + const values = entries.map((entry) => entry.features[index]!) + const mean = values.reduce((sum, value) => sum + value, 0) / values.length + const variance = values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / values.length + return { mean, standardDeviation: Math.sqrt(variance) || 1 } + }) + return entries.map((entry) => + entry.features.map((value, index) => (value - dimensions[index]!.mean) / dimensions[index]!.standardDeviation), + ) + } + + function basis(state: State, entries: Entry[]) { + const key = `${state.protocolVersion}:${state.poolFingerprint}:${state.config.lengthscale}` + const cached = bases.get(key) + if (cached) return cached + const features = state.config.transfer ? entries.map((entry) => entry.features) : scale(entries) + const total = entries.reduce((sum, entry) => sum + entry.weight, 0) + const weights = entries.map((entry) => entry.weight / total) + const matrix = features.map((left) => + features.map((right) => + state.config.transfer ? dot(left, right) : kernel(left, right, state.config.lengthscale), + ), + ) + const meanKernel = matrix.map((row) => row.reduce((sum, value, index) => sum + weights[index]! * value, 0)) + const value = { matrix, features, weights, meanKernel } + if (bases.size >= 4) { + const oldest = bases.keys().next().value + if (oldest) bases.delete(oldest) + } + bases.set(key, value) + return value + } + + function posterior(state: State) { + const entries = state.order.map((id) => state.pool[id]!) + const base = basis(state, entries) + const observed = entries.flatMap((entry, index) => (entry.observation ? [index] : [])) + const matrix = observed.map((left, i) => + observed.map((right, j) => base.matrix[left]![right]! + (i === j ? state.config.noiseVariance : 0)), + ) + const factor = matrix.length ? cholesky(matrix) : [] + const residual = observed.map((index) => entries[index]!.observation!.loss - entries[index]!.priorLoss) + const alpha = factor.length ? solve(factor, residual) : [] + const weightedObserved = observed.map((index) => base.meanKernel[index]!) + const weightedProjection = factor.length ? forward(factor, weightedObserved) : [] + const probes = entries.map((entry, index) => { + const cross = observed.map((peer) => base.matrix[index]![peer]!) + const projection = factor.length ? forward(factor, cross) : [] + const mean = clamp(entry.priorLoss + dot(cross, alpha)) + const variance = Math.max(base.matrix[index]![index]! - dot(projection, projection), 1e-12) + const covariance = base.meanKernel[index]! - dot(projection, weightedProjection) + return { + mean, + variance, + varianceReduction: Math.max((covariance * covariance) / (variance + state.config.noiseVariance), 0), + } + }) + const priorVariance = base.weights.reduce((sum, weight, index) => sum + weight * base.meanKernel[index]!, 0) + const integralVariance = Math.max(priorVariance - dot(weightedProjection, weightedProjection), 0) + return { entries, weights: base.weights, features: base.features, observed, probes, integralVariance } + } + + function transfer(state: State): Transfer { + const protocol = state.config.transfer + if (!protocol) return { status: "not_configured", observed: 0, required: 0 } + const samples = state.order + .map((id) => state.pool[id]!) + .filter((entry) => entry.selection && entry.selection.round <= protocol.calibrationSamples && entry.observation) + const meanAbsoluteError = samples.length + ? samples.reduce((sum, entry) => sum + Math.abs(entry.observation!.loss - entry.priorLoss), 0) / samples.length + : undefined + const status = + samples.length < protocol.calibrationSamples + ? ("calibrating" as const) + : meanAbsoluteError! <= protocol.maxCalibrationMAE + ? ("accepted" as const) + : ("rejected" as const) + return Transfer.parse({ + status, + observed: samples.length, + required: protocol.calibrationSamples, + meanAbsoluteError, + threshold: protocol.maxCalibrationMAE, + }) + } + + function summarize(state: State): Estimate { + const model = posterior(state) + const qualification = transfer(state) + const meanLoss = clamp(model.probes.reduce((sum, probe, index) => sum + model.weights[index]! * probe.mean, 0)) + const standardDeviation = Math.sqrt(model.integralVariance) + const strata = new Set(model.entries.map((entry) => entry.stratum)) + const covered = new Set(model.entries.filter((entry) => entry.observation).map((entry) => entry.stratum)) + const denominator = model.weights.reduce((sum, weight) => sum + weight * weight, 0) + return Estimate.parse({ + observed: model.observed.length, + failures: model.entries.filter((entry) => entry.observation?.failure).length, + meanLoss, + standardDeviation, + lower95: clamp(meanLoss - 1.96 * standardDeviation), + upper95: clamp(meanLoss + 1.96 * standardDeviation), + abstain: + model.observed.length < state.config.minSamples || + standardDeviation > state.config.maxUncertainty || + (Boolean(state.config.transfer) && qualification.status !== "accepted"), + effectivePoolSize: 1 / denominator, + stratumCoverage: covered.size / strata.size, + transfer: qualification, + }) + } + + function distance(index: number, failures: number[], features: number[][]) { + if (!failures.length) return 1 + const nearest = Math.min( + ...failures.map((failure) => + Math.sqrt( + features[index]!.reduce((sum, value, dimension) => sum + (value - features[failure]![dimension]!) ** 2, 0) / + features[index]!.length, + ), + ), + ) + return clamp(nearest) + } + + function choose(state: State) { + const model = posterior(state) + const qualification = transfer(state) + const candidates = model.entries.flatMap((entry, index) => (entry.selection ? [] : [{ entry, index }])) + if (!candidates.length) throw new Error(`Audit probe pool is exhausted`) + const reductions = candidates.map((candidate) => model.probes[candidate.index]!.varianceReduction) + const low = Math.min(...reductions) + const high = Math.max(...reductions) + const maximumVariance = Math.max(...candidates.map((candidate) => model.probes[candidate.index]!.variance)) + const failures = model.entries.flatMap((entry, index) => (entry.observation?.failure ? [index] : [])) + const counts: Record = {} + for (const entry of model.entries) { + if (!entry.observation) continue + counts[entry.stratum] = (counts[entry.stratum] ?? 0) + 1 + } + const weighted = candidates.map((candidate) => { + const probe = model.probes[candidate.index]! + const posteriorStd = Math.sqrt(probe.variance) + const failureUCB = clamp(probe.mean + state.config.beta * posteriorStd) + const eligible = failureUCB >= state.config.failureThreshold + const potential = state.config.transfer + ? eligible + ? probe.variance / Math.max(maximumVariance, 1e-12) + : 0 + : clamp((failureUCB - state.config.failureThreshold) / Math.max(1 - state.config.failureThreshold, 1e-9)) + const diversity = distance(candidate.index, failures, model.features) + const coverage = 1 / Math.sqrt((counts[candidate.entry.stratum] ?? 0) + 1) + const variance = high === low ? 1 : (probe.varianceReduction - low) / (high - low) + const failure = + state.config.transfer && !eligible + ? 0 + : (1 - state.config.diversityWeight - state.config.coverageWeight) * potential + + state.config.diversityWeight * diversity + + state.config.coverageWeight * coverage + const score = + state.config.mode === "performance" + ? variance + : state.config.mode === "failure" + ? failure + : state.config.estimationWeight * variance + (1 - state.config.estimationWeight) * failure + return { + id: candidate.entry.id, + acquisition: { + posteriorLoss: probe.mean, + posteriorStd, + failureUCB, + varianceReduction: probe.varianceReduction, + diversity, + coverage, + score, + }, + } + }) + if (state.config.transfer && qualification.status !== "accepted") { + const phase = qualification.status === "calibrating" ? ("calibration" as const) : ("fallback" as const) + const selected = weighted.toSorted((left, right) => { + const leftEntry = state.pool[left.id]! + const rightEntry = state.pool[right.id]! + const leftKey = digest([state.config.transfer!.selectionSHA256, leftEntry.commitment, left.id]) + const rightKey = digest([state.config.transfer!.selectionSHA256, rightEntry.commitment, right.id]) + return leftKey.localeCompare(rightKey) || left.id.localeCompare(right.id) + })[0]! + return { ...selected, phase } + } + const selected = weighted.toSorted( + (left, right) => right.acquisition.score - left.acquisition.score || left.id.localeCompare(right.id), + )[0]! + return { ...selected, phase: "adaptive" as const } + } + + function finish(state: State, now: number): State { + const estimate = summarize(state) + const reason = (() => { + if (state.config.targetFailures !== undefined && estimate.failures >= state.config.targetFailures) { + return "failure_target_reached" as const + } + if (estimate.observed >= state.config.budget) return "budget_exhausted" as const + if (estimate.observed >= state.order.length) return "pool_exhausted" as const + if ( + state.config.mode !== "failure" && + estimate.observed >= state.config.minSamples && + !estimate.abstain && + estimate.standardDeviation <= state.config.tolerance + ) { + return "precision_reached" as const + } + })() + return State.parse({ + ...state, + estimate, + status: reason ? "completed" : "active", + stopReason: reason, + updatedAt: now, + }) + } + + async function verify(contract: HarnessContract.Info, subject: Subject) { + if (subject.type === "run") { + if (subject.id !== contract.runID) throw new Error(`Audit run subject does not match the bound contract`) + return + } + const search = await HarnessSearch.read(contract.sessionID) + const candidate = search.candidates[subject.id] + if (!candidate) throw new Error(`Audit candidate does not exist in the bound search`) + if (candidate.artifact.sha256 !== subject.artifactSHA256) { + throw new Error(`Audit artifact commitment does not match the candidate`) + } + } + + export async function initialize(input: Initialize) { + const parsed = Initialize.parse(input) + const contract = await HarnessAdapter.authorize(parsed.sessionID, parsed.evaluatorToken) + if (!contract.audit) throw new Error(`The bound harness contract does not declare an active audit`) + if (parsed.probes.length < contract.audit.budget) throw new Error(`Audit pool is smaller than the contract budget`) + await verify(contract, parsed.subject) + const source = parsed.probes.toSorted((left, right) => left.id.localeCompare(right.id)) + const probes = source.map((probe) => (contract.audit!.transfer ? TransferProbe.parse(probe) : Probe.parse(probe))) + if (new Set(probes.map((probe) => probe.id)).size !== probes.length) + throw new Error(`Audit probe ids must be unique`) + if (contract.audit.transfer) { + const invalid = probes.find( + (probe) => TransferProbe.parse(probe).sourceLosses.length !== contract.audit!.transfer!.sourceModels.length, + ) + if (invalid) throw new Error(`Audit source-loss dimension does not match the frozen source models`) + } + const poolFingerprint = digest(probes) + if (contract.audit.transfer && poolFingerprint !== contract.audit.transfer.poolSHA256) { + throw new Error(`Audit probe pool does not match the frozen transfer-pool commitment`) + } + if (contract.audit.transfer) { + const sourceManifestSHA256 = digest({ + sourceModels: contract.audit.transfer.sourceModels, + scores: probes.map((probe) => { + const transfer = TransferProbe.parse(probe) + return { id: transfer.id, sourceLosses: transfer.sourceLosses } + }), + }) + if (sourceManifestSHA256 !== contract.audit.transfer.sourceManifestSHA256) { + throw new Error(`Audit source scores do not match the frozen source manifest`) + } + } + const auditID = digest({ + contractFingerprint: HarnessContract.fingerprint(contract), + subject: parsed.subject, + poolFingerprint, + config: contract.audit, + }) + const entries = probes.map((probe) => + contract.audit!.transfer ? derive(TransferProbe.parse(probe)) : Entry.parse(Probe.parse(probe)), + ) + const pool = Object.fromEntries(entries.map((probe) => [probe.id, probe])) + const now = Date.now() + const base = { + schemaVersion: 1 as const, + protocolVersion: contract.audit.transfer ? ("proactive-audit-v2" as const) : ("active-audit-v1" as const), + auditID, + runID: contract.runID, + sessionID: contract.sessionID, + evaluator: contract.benchmark.evaluator, + contractFingerprint: HarnessContract.fingerprint(contract), + poolFingerprint, + subject: parsed.subject, + config: contract.audit, + status: "active" as const, + pool, + order: entries.map((probe) => probe.id), + revision: 0, + createdAt: now, + updatedAt: now, + } + const initial = State.parse({ + ...base, + estimate: summarize(State.parse({ ...base, estimate: seed(entries, contract.audit) })), + }) + await JsonStore.update(file(parsed.sessionID, auditID), (data) => { + if (!Object.keys(data).length) return initial + const current = parse(data) + if (current.poolFingerprint === poolFingerprint && current.contractFingerprint === base.contractFingerprint) { + return current + } + throw new Error(`Active audit already exists with different immutable inputs`) + }) + return read(parsed.sessionID, auditID) + } + + function seed(probes: Entry[], config: HarnessContract.Audit): Estimate { + const total = probes.reduce((sum, probe) => sum + probe.weight, 0) + const weights = probes.map((probe) => probe.weight / total) + const meanLoss = clamp(probes.reduce((sum, probe, index) => sum + weights[index]! * probe.priorLoss, 0)) + const denominator = weights.reduce((sum, weight) => sum + weight * weight, 0) + return Estimate.parse({ + observed: 0, + failures: 0, + meanLoss, + standardDeviation: 1, + lower95: 0, + upper95: 1, + abstain: true, + effectivePoolSize: 1 / denominator, + stratumCoverage: 0, + transfer: config.transfer + ? { + status: "calibrating", + observed: 0, + required: config.transfer.calibrationSamples, + threshold: config.transfer.maxCalibrationMAE, + } + : { status: "not_configured", observed: 0, required: 0 }, + }) + } + + export async function read(sessionID: string, auditID: string) { + return parse(await JsonStore.read(file(sessionID, auditID))) + } + + export async function status(auditID: string, input: Access) { + const access = Access.parse(input) + const contract = await HarnessAdapter.authorize(access.sessionID, access.evaluatorToken) + const state = await read(access.sessionID, auditID) + match(state, contract) + return state + } + + export async function select(auditID: string, input: Access) { + const access = Access.parse(input) + const contract = await HarnessAdapter.authorize(access.sessionID, access.evaluatorToken) + await JsonStore.update(file(access.sessionID, auditID), (data) => { + const state = parse(data) + match(state, contract) + if (state.status !== "active") throw new Error(`Active audit is already completed`) + const pending = state.order.find((id) => state.pool[id]!.selection && !state.pool[id]!.observation) + if (pending) return state + const selected = choose(state) + const now = Date.now() + return State.parse({ + ...state, + pool: { + ...state.pool, + [selected.id]: { + ...state.pool[selected.id]!, + selection: { + round: state.estimate.observed + 1, + selectedAt: now, + phase: selected.phase, + acquisition: selected.acquisition, + }, + }, + }, + revision: state.revision + 1, + updatedAt: now, + }) + }) + const state = await read(access.sessionID, auditID) + const selected = state.order.find((id) => state.pool[id]!.selection && !state.pool[id]!.observation)! + const entry = state.pool[selected]! + return { + auditID: state.auditID, + probeID: entry.id, + commitment: entry.commitment, + round: entry.selection!.round, + phase: entry.selection!.phase ?? "adaptive", + acquisition: entry.selection!.acquisition, + revision: state.revision, + } + } + + export async function observe(auditID: string, input: Observe) { + const observation = Observe.parse(input) + const contract = await HarnessAdapter.authorize(observation.sessionID, observation.evaluatorToken) + await JsonStore.update(file(observation.sessionID, auditID), (data) => { + const state = parse(data) + match(state, contract) + const entry = state.pool[observation.probeID] + if (!entry) throw new Error(`Unknown audit probe ${observation.probeID}`) + if (!entry.selection) throw new Error(`Audit probe must be selected before observation`) + if (observation.failure !== observation.loss >= state.config.failureThreshold) { + throw new Error(`Audit failure label does not match the contract threshold`) + } + const result = Observation.parse({ + loss: observation.loss, + failure: observation.failure, + evidence: observation.evidence, + note: observation.note, + evaluatedAt: Date.now(), + }) + if (entry.observation) { + const previous = { ...entry.observation, evaluatedAt: result.evaluatedAt } + if (JSON.stringify(previous) === JSON.stringify(result)) return state + throw new Error(`Audit probe observation is immutable`) + } + const now = result.evaluatedAt + const next = State.parse({ + ...state, + pool: { ...state.pool, [entry.id]: { ...entry, observation: result } }, + revision: state.revision + 1, + updatedAt: now, + }) + return finish(next, now) + }) + return read(observation.sessionID, auditID) + } + + export async function seal(auditID: string, input: Access) { + const access = Access.parse(input) + const contract = await HarnessAdapter.authorize(access.sessionID, access.evaluatorToken) + const state = await read(access.sessionID, auditID) + match(state, contract) + if (state.status !== "completed" || !state.stopReason) { + throw new Error(`Active audit must reach a terminal state before sealing`) + } + const stable = { + schemaVersion: 1 as const, + protocolVersion: "proactive-audit-receipt-v1" as const, + auditID: state.auditID, + runID: state.runID, + sessionID: state.sessionID, + contractFingerprint: state.contractFingerprint, + poolFingerprint: state.poolFingerprint, + subject: state.subject, + config: state.config, + stopReason: state.stopReason, + estimate: state.estimate, + revision: state.revision, + qualified: + state.config.mode !== "failure" && state.estimate.transfer.status === "accepted" && !state.estimate.abstain, + completedAt: state.updatedAt, + sealedAt: state.updatedAt, + } + const receipt = Receipt.parse({ ...stable, receiptID: digest(stable) }) + await JsonStore.update(receiptFile(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Active audit receipt is immutable once recorded`) + }) + const saved = await readReceipt(receipt.receiptID) + if (!saved) throw new Error(`Active audit receipt was not durable after recording`) + return saved + } + + export async function readReceipt(receiptID: string) { + const id = Hash.parse(receiptID) + const parsed = Receipt.safeParse(await JsonStore.read(receiptFile(id))) + if (!parsed.success || parsed.data.receiptID !== id) return null + const data = await JsonStore.read(file(parsed.data.sessionID, parsed.data.auditID)) + const parsedState = State.safeParse(data) + if (!parsedState.success || drift(parsedState.data)) return null + const state = parsedState.data + const snapshot = { + auditID: state.auditID, + runID: state.runID, + sessionID: state.sessionID, + contractFingerprint: state.contractFingerprint, + poolFingerprint: state.poolFingerprint, + subject: state.subject, + config: state.config, + stopReason: state.stopReason, + estimate: state.estimate, + revision: state.revision, + completedAt: state.updatedAt, + } + const receipt = { + auditID: parsed.data.auditID, + runID: parsed.data.runID, + sessionID: parsed.data.sessionID, + contractFingerprint: parsed.data.contractFingerprint, + poolFingerprint: parsed.data.poolFingerprint, + subject: parsed.data.subject, + config: parsed.data.config, + stopReason: parsed.data.stopReason, + estimate: parsed.data.estimate, + revision: parsed.data.revision, + completedAt: parsed.data.completedAt, + } + return state.status === "completed" && JSON.stringify(snapshot) === JSON.stringify(receipt) ? parsed.data : null + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: { type: "run" | "candidate"; id: string } + evaluatedAt: number + recordedAt: number + requireQualified: boolean + }) { + const receipt = await readReceipt(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt active audit receipt ${input.receiptID}`) + if (!input.contract.audit) + throw new Error(`Evaluation cites an audit receipt without a bound active audit protocol`) + if (receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract)) { + throw new Error(`Active audit receipt does not match the bound harness contract`) + } + if (receipt.sessionID !== input.contract.sessionID || receipt.runID !== input.contract.runID) { + throw new Error(`Active audit receipt belongs to a different harness run`) + } + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Active audit receipt belongs to a different evaluation subject`) + } + if (receipt.completedAt < input.contract.createdAt) { + throw new Error(`Active audit receipt predates the bound harness contract`) + } + if (receipt.completedAt > input.evaluatedAt || receipt.sealedAt > input.recordedAt) { + throw new Error(`Evaluation predates its active audit receipt`) + } + if (input.requireQualified && !receipt.qualified) { + throw new Error(`A passing final evaluation requires a qualified non-abstaining active audit receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/autonomy.ts b/backend/cli/src/session/harness/autonomy.ts new file mode 100644 index 00000000..fe24f108 --- /dev/null +++ b/backend/cli/src/session/harness/autonomy.ts @@ -0,0 +1,492 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessAutonomy { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(240), + }) + .strict() + export type Subject = z.infer + + export const Actor = z.enum(["benchmark", "human", "agent"]) + export type Actor = z.infer + + export const Contribution = z.enum(["problem", "auxiliary", "essential", "core", "unclear"]) + export type Contribution = z.infer + + export const Kind = z.enum([ + "problem_statement", + "clarification", + "resource_provision", + "strategy", + "technical_correction", + "artifact_edit", + "candidate_selection", + "evaluation_feedback", + "exposition", + "other", + ]) + export type Kind = z.infer + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + export type Access = z.infer + + const EventInput = z + .object({ + sequence: z.number().int().positive(), + at: z.number().int().positive(), + actor: Actor, + kind: Kind, + contribution: Contribution, + contentSHA256: Hash, + artifactBeforeSHA256: Hash.optional(), + artifactAfterSHA256: Hash.optional(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + export const Event = EventInput.extend({ + priorEventID: Hash.nullable(), + eventID: Hash, + }).strict() + export type Event = z.infer + + export const Submit = Access.extend({ + subject: Subject, + artifactSHA256: Hash, + trace: z + .object({ + owner: z.literal("evaluator_runtime"), + complete: z.literal(true), + recorderArtifactSHA256: Hash, + schemaSHA256: Hash, + classificationPolicySHA256: Hash, + rawLogSHA256: Hash, + startedAt: z.number().int().positive(), + endedAt: z.number().int().positive(), + events: z.array(EventInput).min(2).max(10_000), + }) + .strict(), + }).strict() + export type Submit = z.infer + + const Counts = z.record(Actor, z.record(Contribution, z.number().int().nonnegative())) + + export const Metrics = z + .object({ + events: z.number().int().positive(), + counts: Counts, + problemEvents: z.number().int().nonnegative(), + humanSubstantiveEvents: z.number().int().nonnegative(), + agentSubstantiveEvents: z.number().int().nonnegative(), + unclearEvents: z.number().int().nonnegative(), + linkedArtifactEvents: z.number().int().nonnegative(), + artifactTransitions: z.number().int().nonnegative(), + finalArtifactLinked: z.boolean(), + }) + .strict() + export type Metrics = z.infer + + export const Receipt = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("human-ai-autonomy-receipt-v1"), + receiptID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + protocolSHA256: Hash, + subject: Subject, + artifactSHA256: Hash, + traceSHA256: Hash, + recorderArtifactSHA256: Hash, + rawLogSHA256: Hash, + startedAt: z.number().int().positive(), + endedAt: z.number().int().positive(), + events: z.array(Event).min(2).max(10_000), + claimedLevel: HarnessContract.AutonomyLevel, + derivedLevel: HarnessContract.AutonomyLevel.optional(), + metrics: Metrics, + status: z.enum(["passed", "failed", "inconclusive"]), + failures: z.array(z.string().min(1).max(500)).max(32), + recordedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + delete stable.recordedAt + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Human-AI autonomy receipt hash is invalid" }) + }) + export type Receipt = z.infer + + export function prompt(contract: HarnessContract.Info) { + const protocol = contract.autonomy + if (!protocol) return "" + return [ + '', + `This run predeclares the ${protocol.claimedLevel} human-AI contribution level under human-ai-autonomy-v1.`, + "Every benchmark, human, and agent interaction is captured by the evaluator runtime; do not ask for unrecorded side-channel help.", + "Problem statements are distinct from auxiliary exposition, while strategy, technical corrections, artifact edits, selection, and feedback may be classified as essential or core.", + "The backend derives the final level from the complete trace. Unclear contribution labels are inconclusive, and an essential human contribution prevents an essentially-autonomous pass.", + "Raw interaction content is retained outside the candidate context; receipts contain hashes and evidence references, not hidden prompts or evaluator capabilities.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + return contract ? prompt(contract) : "" + } + + const root = path.join(Global.Path.data, "harness", "autonomy") + const receiptFile = (receiptID: string) => path.join(root, "receipts", `${receiptID}.json`) + const subjectFile = (sessionID: string, subject: Subject) => + path.join( + root, + "subjects", + encodeURIComponent(sessionID), + `${encodeURIComponent(`${subject.type}:${subject.id}`)}.json`, + ) + + async function target(contract: HarnessContract.Info, subject: Subject) { + if (subject.type === "run") { + if (subject.id !== contract.runID) throw new Error(`Human-AI autonomy run subject does not match its contract`) + return { createdAt: contract.createdAt } + } + const state = await import("./search") + .then((module) => module.HarnessSearch.read(contract.sessionID)) + .catch(() => null) + const candidate = state?.runID === contract.runID ? state.candidates[subject.id] : undefined + if (!candidate) throw new Error(`Human-AI autonomy candidate does not exist in the bound search`) + return { createdAt: candidate.createdAt, artifactSHA256: candidate.artifact.sha256 } + } + + function replay(input: z.infer[], startedAt: number, endedAt: number) { + const state = input.reduce<{ events: Event[]; artifact?: string }>( + (state, item, index) => { + if (item.sequence !== index + 1) throw new Error(`Human-AI interaction trace must be contiguous from one`) + if (item.at < startedAt || item.at > endedAt) { + throw new Error(`Human-AI interaction event ${item.sequence} falls outside the trace interval`) + } + if (state.events.length && item.at < state.events.at(-1)!.at) { + throw new Error(`Human-AI interaction trace time must be monotonic`) + } + if (state.artifact && item.artifactAfterSHA256 && item.artifactBeforeSHA256 !== state.artifact) { + throw new Error(`Human-AI interaction artifact transitions must form one continuous chain`) + } + if (item.kind === "problem_statement" && item.contribution !== "problem") { + throw new Error(`A problem statement must use the problem contribution class`) + } + if ((!index && item.kind !== "problem_statement") || (index > 0 && item.kind === "problem_statement")) { + throw new Error(`Human-AI interaction trace requires exactly one initial problem statement`) + } + if (item.contribution === "problem" && item.kind !== "problem_statement") { + throw new Error(`Only a problem statement may use the problem contribution class`) + } + if (item.contribution === "problem" && item.actor === "agent") { + throw new Error(`An agent-authored event cannot pose the frozen benchmark problem`) + } + if (item.kind === "exposition" && !["auxiliary", "unclear"].includes(item.contribution)) { + throw new Error(`Exposition cannot be classified as essential or core scientific content`) + } + const priorEventID = state.events.at(-1)?.eventID ?? null + const stable = { ...item, priorEventID } + state.events.push(Event.parse({ ...stable, eventID: digest(stable) })) + return { events: state.events, artifact: item.artifactAfterSHA256 ?? state.artifact } + }, + { events: [] }, + ) + return state.events + } + + function assess(input: { protocol: HarnessContract.HumanAIAutonomy; events: Event[]; artifactSHA256: string }) { + const counts = Counts.parse( + Object.fromEntries( + Actor.options.map((actor) => [ + actor, + Object.fromEntries( + Contribution.options.map((contribution) => [ + contribution, + input.events.filter((item) => item.actor === actor && item.contribution === contribution).length, + ]), + ), + ]), + ), + ) + const humanSubstantiveEvents = counts.human.essential + counts.human.core + const agentSubstantiveEvents = counts.agent.essential + counts.agent.core + const unclearEvents = counts.benchmark.unclear + counts.human.unclear + counts.agent.unclear + const problemEvents = input.events.filter((item) => item.kind === "problem_statement").length + const linkedArtifactEvents = input.events.filter((item) => item.artifactAfterSHA256 === input.artifactSHA256).length + const transitions = input.events.filter((item) => item.artifactAfterSHA256) + const artifactTransitions = transitions.length + const finalArtifactLinked = transitions.at(-1)?.artifactAfterSHA256 === input.artifactSHA256 + const derivedLevel = unclearEvents + ? undefined + : humanSubstantiveEvents && agentSubstantiveEvents + ? ("human_ai_collaboration" as const) + : humanSubstantiveEvents + ? ("primarily_human" as const) + : agentSubstantiveEvents + ? ("essentially_autonomous" as const) + : undefined + const metrics = Metrics.parse({ + events: input.events.length, + counts, + problemEvents, + humanSubstantiveEvents, + agentSubstantiveEvents, + unclearEvents, + linkedArtifactEvents, + artifactTransitions, + finalArtifactLinked, + }) + const structural = [ + ...(!problemEvents ? ["interaction trace has no frozen problem statement"] : []), + ...(!input.events.some((item) => item.actor === "agent") ? ["interaction trace has no agent event"] : []), + ...(!finalArtifactLinked ? ["last interaction artifact transition does not bind the final artifact"] : []), + ...(!derivedLevel && !unclearEvents + ? ["interaction trace has no essential or core scientific contribution"] + : []), + ] + const uncertain = unclearEvents ? [`${unclearEvents} interaction contribution classifications are unclear`] : [] + const mismatch = + derivedLevel && derivedLevel !== input.protocol.claimedLevel + ? [`derived autonomy level ${derivedLevel} does not match claimed level ${input.protocol.claimedLevel}`] + : [] + const failures = [...structural, ...uncertain, ...mismatch] + const status = uncertain.length + ? ("inconclusive" as const) + : failures.length + ? ("failed" as const) + : ("passed" as const) + return { metrics, derivedLevel, status, failures } + } + + function verify(receipt: Receipt, protocol: HarnessContract.HumanAIAutonomy) { + if ( + receipt.recorderArtifactSHA256 !== protocol.recorder.artifactSHA256 || + receipt.claimedLevel !== protocol.claimedLevel + ) { + throw new Error(`Human-AI autonomy receipt changed its frozen protocol identity`) + } + if (receipt.events.length > protocol.maxEvents) { + throw new Error(`Human-AI autonomy receipt exceeds its frozen event budget`) + } + const events = replay( + receipt.events.map((item) => + EventInput.parse( + Object.fromEntries(Object.entries(item).filter(([key]) => key !== "eventID" && key !== "priorEventID")), + ), + ), + receipt.startedAt, + receipt.endedAt, + ) + const trace = { + recorderArtifactSHA256: receipt.recorderArtifactSHA256, + rawLogSHA256: receipt.rawLogSHA256, + startedAt: receipt.startedAt, + endedAt: receipt.endedAt, + events, + } + if (!same(events, receipt.events) || digest(trace) !== receipt.traceSHA256) { + throw new Error(`Human-AI autonomy receipt does not match its backend-replayed interaction trace`) + } + const result = assess({ protocol, events, artifactSHA256: receipt.artifactSHA256 }) + if ( + !same(result.metrics, receipt.metrics) || + result.derivedLevel !== receipt.derivedLevel || + result.status !== receipt.status || + !same(result.failures, receipt.failures) + ) { + throw new Error(`Human-AI autonomy receipt does not match its backend-derived classification`) + } + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + if (value.sessionID !== contract.sessionID) { + throw new Error(`Human-AI autonomy session does not match its bound harness contract`) + } + const protocol = contract.autonomy + if (!protocol) throw new Error(`Harness contract does not require human-AI autonomy tracing`) + if (value.trace.recorderArtifactSHA256 !== protocol.recorder.artifactSHA256) { + throw new Error(`Human-AI interaction recorder does not match the bound protocol`) + } + if (value.trace.schemaSHA256 !== protocol.traceSchemaSHA256) { + throw new Error(`Human-AI interaction trace schema does not match the bound protocol`) + } + if (value.trace.classificationPolicySHA256 !== protocol.classificationPolicySHA256) { + throw new Error(`Human-AI contribution policy does not match the bound protocol`) + } + if (value.trace.events.length > protocol.maxEvents) { + throw new Error(`Human-AI interaction trace exceeds its frozen event budget`) + } + if (value.trace.endedAt < value.trace.startedAt) { + throw new Error(`Human-AI interaction trace ends before it starts`) + } + const recordedAt = Date.now() + const subject = await target(contract, value.subject) + if (value.trace.startedAt !== contract.createdAt || value.trace.endedAt > recordedAt) { + throw new Error(`Human-AI interaction trace falls outside its bound run interval`) + } + if ( + value.subject.type === "candidate" && + (value.trace.startedAt > subject.createdAt || value.trace.endedAt < subject.createdAt) + ) { + throw new Error(`Human-AI interaction trace does not enclose candidate creation`) + } + if (subject.artifactSHA256 && subject.artifactSHA256 !== value.artifactSHA256) { + throw new Error(`Human-AI autonomy receipt changed the candidate artifact`) + } + const events = replay(value.trace.events, value.trace.startedAt, value.trace.endedAt) + const result = assess({ protocol, events, artifactSHA256: value.artifactSHA256 }) + const trace = { + recorderArtifactSHA256: value.trace.recorderArtifactSHA256, + rawLogSHA256: value.trace.rawLogSHA256, + startedAt: value.trace.startedAt, + endedAt: value.trace.endedAt, + events, + } + const stable = { + schemaVersion: 1 as const, + protocolVersion: "human-ai-autonomy-receipt-v1" as const, + runID: contract.runID, + sessionID: contract.sessionID, + contractFingerprint: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + subject: value.subject, + artifactSHA256: value.artifactSHA256, + traceSHA256: digest(trace), + recorderArtifactSHA256: value.trace.recorderArtifactSHA256, + rawLogSHA256: value.trace.rawLogSHA256, + startedAt: value.trace.startedAt, + endedAt: value.trace.endedAt, + events, + claimedLevel: protocol.claimedLevel, + derivedLevel: result.derivedLevel, + metrics: result.metrics, + status: result.status, + failures: result.failures, + } + const receipt = Receipt.parse({ ...stable, receiptID: digest(stable), recordedAt }) + const claimed = await JsonStore.read(subjectFile(receipt.sessionID, receipt.subject)) + if (Object.keys(claimed).length) { + const current = Receipt.parse(claimed) + if (current.receiptID !== receipt.receiptID) { + throw new Error(`Human-AI autonomy subject already has a canonical receipt`) + } + } + await JsonStore.update(receiptFile(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Human-AI autonomy receipt is immutable once recorded`) + }) + await JsonStore.update(subjectFile(receipt.sessionID, receipt.subject), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Human-AI autonomy subject already has a canonical receipt`) + }) + const saved = await readReceipt(receipt.receiptID) + if (!saved) throw new Error(`Human-AI autonomy receipt was not durable after recording`) + return saved + } + + export async function readReceipt(receiptID: string) { + const id = Hash.parse(receiptID) + const parsed = Receipt.safeParse(await JsonStore.read(receiptFile(id))) + if (!parsed.success || parsed.data.receiptID !== id) return null + const canonical = Receipt.safeParse(await JsonStore.read(subjectFile(parsed.data.sessionID, parsed.data.subject))) + if (!canonical.success || canonical.data.receiptID !== id || !same(canonical.data, parsed.data)) return null + return parsed.data + } + + export async function read(receiptID: string, contract: HarnessContract.Info) { + const receipt = await readReceipt(receiptID) + if (!receipt || receipt.sessionID !== contract.sessionID) { + throw new Error(`Unknown human-AI autonomy receipt ${receiptID}`) + } + const protocol = contract.autonomy + if (!protocol || receipt.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Human-AI autonomy receipt belongs to a different harness run`) + } + verify(receipt, protocol) + const subject = await target(contract, receipt.subject) + if ( + receipt.startedAt !== contract.createdAt || + receipt.endedAt > receipt.recordedAt || + (receipt.subject.type === "candidate" && + (receipt.startedAt > subject.createdAt || receipt.endedAt < subject.createdAt)) || + (subject.artifactSHA256 && subject.artifactSHA256 !== receipt.artifactSHA256) + ) { + throw new Error(`Human-AI autonomy receipt changed its bound run, candidate, or artifact interval`) + } + return receipt + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: Subject + evaluatedAt: number + recordedAt: number + requirePassed: boolean + }) { + const receipt = await readReceipt(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt human-AI autonomy receipt ${input.receiptID}`) + const protocol = input.contract.autonomy + if (!protocol) throw new Error(`Evaluation cites an autonomy receipt without a bound protocol`) + if ( + receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract) || + receipt.protocolSHA256 !== digest(protocol) || + receipt.sessionID !== input.contract.sessionID || + receipt.runID !== input.contract.runID + ) { + throw new Error(`Human-AI autonomy receipt belongs to a different harness run`) + } + verify(receipt, protocol) + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Human-AI autonomy receipt belongs to a different evaluation subject`) + } + const subject = await target(input.contract, input.subject) + if ( + receipt.startedAt !== input.contract.createdAt || + receipt.endedAt > receipt.recordedAt || + (input.subject.type === "candidate" && + (receipt.startedAt > subject.createdAt || receipt.endedAt < subject.createdAt)) || + (subject.artifactSHA256 && subject.artifactSHA256 !== receipt.artifactSHA256) + ) { + throw new Error(`Human-AI autonomy receipt belongs to a different candidate artifact or interval`) + } + if ( + receipt.endedAt > input.evaluatedAt || + receipt.recordedAt > input.evaluatedAt || + receipt.recordedAt > input.recordedAt + ) { + throw new Error(`Evaluation predates its human-AI autonomy receipt`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A passing final evaluation requires a passing human-AI autonomy receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/blueprint.ts b/backend/cli/src/session/harness/blueprint.ts new file mode 100644 index 00000000..193191c3 --- /dev/null +++ b/backend/cli/src/session/harness/blueprint.ts @@ -0,0 +1,910 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessBlueprint { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + export type Access = z.infer + + export const GoalSpec = z + .object({ + statementSHA256: Hash, + declaration: z.string().min(1).max(500), + module: z.string().min(1).max(500), + }) + .strict() + export type GoalSpec = z.infer + + const Goal = GoalSpec.extend({ + id: Hash, + createdAt: z.number().int().positive(), + }).strict() + export type Goal = z.infer + + export const LeaseRequest = Access.extend({ count: z.number().int().min(1).max(32) }).strict() + + export const Lease = z + .object({ + id: Hash, + goalID: Hash, + revision: z.number().int().nonnegative(), + ordinal: z.number().int().nonnegative(), + status: z.enum(["open", "consumed", "expired"]), + issuedAt: z.number().int().positive(), + expiresAt: z.number().int().positive(), + consumedAt: z.number().int().positive().optional(), + }) + .strict() + export type Lease = z.infer + + const Verification = z + .object({ + compilerArtifactSHA256: Hash, + statementMatched: z.boolean(), + exitCode: z.number().int(), + warnings: z.number().int().nonnegative(), + transcriptSHA256: Hash, + feedbackSHA256: Hash, + startedAt: z.number().int().positive(), + endedAt: z.number().int().positive(), + }) + .strict() + + export const DirectSubmit = Access.extend({ + kind: z.literal("direct"), + leaseID: Hash, + artifactSHA256: Hash, + claim: z.enum(["proof", "refutation", "failure"]), + verification: Verification, + }).strict() + export type DirectSubmit = z.infer + + const Sketch = Verification.extend({ + validatorArtifactSHA256: Hash, + placeholderDeclarations: z.array(z.string().min(1).max(500)).min(1).max(16), + validatorTranscriptSHA256: Hash, + }).strict() + + const Review = z + .object({ + reviewerArtifactSHA256: Hash, + promptSHA256: Hash, + relevant: z.boolean(), + easier: z.boolean(), + plausible: z.boolean(), + transcriptSHA256: Hash, + }) + .strict() + + export const DecompositionSubmit = Access.extend({ + kind: z.literal("decomposition"), + leaseID: Hash, + informalPlanSHA256: Hash, + artifactSHA256: Hash, + children: z.array(GoalSpec).min(1).max(16), + verification: Sketch, + review: Review, + }).strict() + export type DecompositionSubmit = z.infer + + export const Submit = z.discriminatedUnion("kind", [DirectSubmit, DecompositionSubmit]) + export type Submit = z.infer + + export const Attempt = z + .object({ + id: Hash, + ordinal: z.number().int().nonnegative(), + goalID: Hash, + leaseID: Hash, + kind: z.enum(["direct", "decomposition"]), + artifactSHA256: Hash, + result: z.enum(["proved", "refuted", "failed", "accepted", "rejected"]), + claim: z.enum(["proof", "refutation", "failure"]).optional(), + decompositionID: Hash.optional(), + transcriptSHA256: Hash, + feedbackSHA256: Hash, + failures: z.array(z.string().min(1).max(500)).max(16), + startedAt: z.number().int().positive(), + endedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + const direct = value.kind === "direct" + const result = direct + ? (["proved", "refuted", "failed"] as const).includes(value.result as "proved") + : (["accepted", "rejected"] as const).includes(value.result as "accepted") + if (!result) { + ctx.addIssue({ code: "custom", path: ["result"], message: "Attempt result does not match its operation" }) + } + if (direct !== Boolean(value.claim)) { + ctx.addIssue({ code: "custom", path: ["claim"], message: "Only direct attempts carry a proof claim" }) + } + if (Boolean(value.decompositionID) !== (value.kind === "decomposition" && value.result === "accepted")) { + ctx.addIssue({ + code: "custom", + path: ["decompositionID"], + message: "Only accepted decomposition attempts bind a graph node", + }) + } + const successful = value.result === "proved" || value.result === "refuted" || value.result === "accepted" + if (successful === !value.failures.length) return + ctx.addIssue({ + code: "custom", + path: ["failures"], + message: "Successful attempts cannot contain failures and rejected attempts must explain one", + }) + }) + export type Attempt = z.infer + + export const Decomposition = z + .object({ + id: Hash, + parentID: Hash, + childIDs: z.array(Hash).min(1).max(16), + informalPlanSHA256: Hash, + sketchArtifactSHA256: Hash, + sketchTranscriptSHA256: Hash, + reviewerTranscriptSHA256: Hash, + attemptID: Hash, + createdAt: z.number().int().positive(), + }) + .strict() + export type Decomposition = z.infer + + export const State = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("proof-blueprint-state-v1"), + blueprintID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + protocolSHA256: Hash, + rootGoalID: Hash, + goals: z.record(z.string(), Goal), + decompositions: z.record(z.string(), Decomposition), + attempts: z.record(z.string(), Attempt), + leases: z.record(z.string(), Lease), + revision: z.number().int().nonnegative(), + createdAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + }) + .strict() + export type State = z.infer + + export const GoalStatus = z.enum(["open", "proved", "refuted", "exhausted"]) + export type GoalStatus = z.infer + export const DecompositionStatus = z.enum(["open", "closed", "blocked"]) + export type DecompositionStatus = z.infer + + export const Summary = z + .object({ + blueprintID: Hash, + status: GoalStatus, + goals: z.number().int().nonnegative(), + proved: z.number().int().nonnegative(), + refuted: z.number().int().nonnegative(), + exhausted: z.number().int().nonnegative(), + decompositions: z.number().int().nonnegative(), + attempts: z.number().int().nonnegative(), + rejected: z.number().int().nonnegative(), + refinements: z.number().int().nonnegative(), + openLeases: z.number().int().nonnegative(), + revision: z.number().int().nonnegative(), + }) + .strict() + export type Summary = z.infer + + export const View = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("proof-blueprint-view-v1"), + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + rootGoalID: Hash, + summary: Summary, + goals: z.array(Goal.extend({ status: GoalStatus, ready: z.boolean() }).strict()), + decompositions: z.array(Decomposition.extend({ status: DecompositionStatus }).strict()), + attempts: z.array(Attempt), + leases: z.array(Lease), + }) + .strict() + export type View = z.infer + + const root = path.join(Global.Path.data, "harness", "blueprints") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const goalID = (spec: GoalSpec) => + digest({ + protocol: "proof-blueprint-goal-v1", + statementSHA256: spec.statementSHA256, + declaration: spec.declaration, + module: spec.module, + }) + const blueprintID = (contract: HarnessContract.Info, protocol: HarnessContract.ProofBlueprint) => + digest({ + protocol: "proof-blueprint-state-v1", + runID: contract.runID, + sessionID: contract.sessionID, + contractFingerprint: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + }) + + const leaseID = (state: State, input: Pick) => + digest({ + blueprintID: state.blueprintID, + goalID: input.goalID, + revision: input.revision, + ordinal: input.ordinal, + issuedAt: input.issuedAt, + expiresAt: input.expiresAt, + }) + + const attemptID = (state: State, input: Attempt) => + digest({ + blueprintID: state.blueprintID, + ordinal: input.ordinal, + goalID: input.goalID, + leaseID: input.leaseID, + kind: input.kind, + artifactSHA256: input.artifactSHA256, + result: input.result, + claim: input.claim, + transcriptSHA256: input.transcriptSHA256, + feedbackSHA256: input.feedbackSHA256, + failures: input.failures, + startedAt: input.startedAt, + endedAt: input.endedAt, + recordedAt: input.recordedAt, + }) + + const decompositionID = (state: State, input: Decomposition) => + digest({ + blueprintID: state.blueprintID, + parentID: input.parentID, + childIDs: input.childIDs, + informalPlanSHA256: input.informalPlanSHA256, + sketchArtifactSHA256: input.sketchArtifactSHA256, + sketchTranscriptSHA256: input.sketchTranscriptSHA256, + reviewerTranscriptSHA256: input.reviewerTranscriptSHA256, + createdAt: input.createdAt, + }) + + const items = (record: Record) => Object.values(record) + const direct = (state: State, id: string) => + items(state.attempts).filter((item) => item.goalID === id && item.kind === "direct") + const sketches = (state: State, id: string) => + items(state.attempts).filter((item) => item.goalID === id && item.kind === "decomposition") + const branches = (state: State, id: string) => items(state.decompositions).filter((item) => item.parentID === id) + + function statuses(state: State, protocol: HarnessContract.ProofBlueprint) { + const goal = new Map() + const branch = new Map() + const visit = (id: string): GoalStatus => { + const cached = goal.get(id) + if (cached) return cached + const attempts = direct(state, id) + const proved = attempts.some((item) => item.result === "proved") + const refuted = attempts.some((item) => item.result === "refuted") + if (proved && refuted) throw new Error(`Proof blueprint goal cannot be both proved and refuted`) + if (proved) { + goal.set(id, "proved") + return "proved" + } + if (refuted) { + goal.set(id, "refuted") + return "refuted" + } + const paths = branches(state, id) + for (const item of paths) { + const children = item.childIDs.map(visit) + const status = children.every((value) => value === "proved") + ? "closed" + : children.some((value) => value === "refuted" || value === "exhausted") + ? "blocked" + : "open" + branch.set(item.id, status) + } + if (paths.some((item) => branch.get(item.id) === "closed")) { + goal.set(id, "proved") + return "proved" + } + const attemptsFull = attempts.length >= protocol.maxAttemptsPerGoal + const pathsFull = sketches(state, id).length >= protocol.maxRefinementsPerGoal + 1 + const allBlocked = paths.length > 0 && paths.every((item) => branch.get(item.id) === "blocked") + const status = attemptsFull && pathsFull && (!paths.length || allBlocked) ? "exhausted" : "open" + goal.set(id, status) + return status + } + visit(state.rootGoalID) + for (const id of Object.keys(state.goals)) visit(id) + return { goal, branch } + } + + function depths(state: State) { + const values = new Map([[state.rootGoalID, 0]]) + const queue = [state.rootGoalID] + while (queue.length) { + const id = queue.shift()! + const depth = values.get(id)! + for (const child of branches(state, id).flatMap((item) => item.childIDs)) { + const next = Math.max(values.get(child) ?? 0, depth + 1) + if (next === values.get(child)) continue + values.set(child, next) + queue.push(child) + } + } + return values + } + + function graphIssue(state: State, parentID: string, childIDs: string[], protocol: HarnessContract.ProofBlueprint) { + const ids = [...new Set([...Object.keys(state.goals), ...childIDs])] + if (ids.length > protocol.maxNodes) return "decomposition exceeds the frozen graph node budget" + const adjacency = Object.fromEntries(ids.map((id) => [id, new Set()])) + for (const item of items(state.decompositions)) { + for (const child of item.childIDs) adjacency[item.parentID]!.add(child) + } + for (const child of childIDs) adjacency[parentID]!.add(child) + const indegree = Object.fromEntries(ids.map((id) => [id, 0])) + for (const children of Object.values(adjacency)) { + for (const child of children) indegree[child] = indegree[child]! + 1 + } + const queue = ids.filter((id) => indegree[id] === 0) + const order: string[] = [] + while (queue.length) { + const id = queue.shift()! + order.push(id) + for (const child of adjacency[id]!) { + indegree[child] = indegree[child]! - 1 + if (indegree[child] === 0) queue.push(child) + } + } + if (order.length !== ids.length) return "decomposition would make the proof blueprint cyclic" + const depth = new Map([[state.rootGoalID, 0]]) + for (const id of order) { + for (const child of adjacency[id]!) { + depth.set(child, Math.max(depth.get(child) ?? 0, (depth.get(id) ?? 0) + 1)) + } + } + if (Math.max(...depth.values()) > protocol.maxDepth) { + return "decomposition exceeds the frozen graph depth budget" + } + return undefined + } + + function ready(state: State, protocol: HarnessContract.ProofBlueprint, now = Date.now()) { + const status = statuses(state, protocol) + const held = new Set( + items(state.leases) + .filter((item) => item.status === "open" && item.expiresAt > now) + .map((item) => item.goalID), + ) + const depth = depths(state) + return items(state.goals) + .filter((item) => { + if (status.goal.get(item.id) !== "open" || held.has(item.id)) return false + if (branches(state, item.id).some((path) => status.branch.get(path.id) === "open")) return false + const attempts = direct(state, item.id).length + const paths = sketches(state, item.id).length + return attempts < protocol.maxAttemptsPerGoal || (attempts > 0 && paths < protocol.maxRefinementsPerGoal + 1) + }) + .toSorted( + (a, b) => + (depth.get(b.id) ?? 0) - (depth.get(a.id) ?? 0) || + direct(state, a.id).length - direct(state, b.id).length || + a.id.localeCompare(b.id), + ) + } + + function integrity(state: State, contract: HarnessContract.Info, protocol: HarnessContract.ProofBlueprint) { + if ( + state.blueprintID !== blueprintID(contract, protocol) || + state.runID !== contract.runID || + state.sessionID !== contract.sessionID || + state.contractFingerprint !== HarnessContract.fingerprint(contract) || + state.protocolSHA256 !== digest(protocol) + ) { + throw new Error(`Proof blueprint belongs to a different harness contract`) + } + const formal = contract.formalProof! + const root = state.goals[state.rootGoalID] + if ( + !root || + root.statementSHA256 !== formal.statementSHA256 || + root.declaration !== formal.declaration || + root.module !== formal.module + ) { + throw new Error(`Proof blueprint root does not match the frozen formal statement`) + } + if (items(state.goals).length > protocol.maxNodes) throw new Error(`Proof blueprint exceeds its node budget`) + for (const [id, goal] of Object.entries(state.goals)) { + if (id !== goal.id || goal.id !== goalID(goal)) throw new Error(`Proof blueprint goal identity is invalid`) + } + const adjacency = Object.fromEntries(Object.keys(state.goals).map((id) => [id, new Set()])) + for (const item of items(state.decompositions)) { + for (const child of item.childIDs) adjacency[item.parentID]?.add(child) + } + const seen = new Set([state.rootGoalID]) + const visit = [state.rootGoalID] + while (visit.length) { + const id = visit.shift()! + for (const child of adjacency[id] ?? []) { + if (seen.has(child)) continue + seen.add(child) + visit.push(child) + } + } + if (seen.size !== items(state.goals).length) throw new Error(`Proof blueprint contains unreachable goals`) + const indegree = Object.fromEntries(Object.keys(state.goals).map((id) => [id, 0])) + for (const children of Object.values(adjacency)) { + for (const child of children) indegree[child] = (indegree[child] ?? 0) + 1 + } + const queue = Object.keys(indegree).filter((id) => indegree[id] === 0) + const order: string[] = [] + while (queue.length) { + const id = queue.shift()! + order.push(id) + for (const child of adjacency[id] ?? []) { + indegree[child] = indegree[child]! - 1 + if (indegree[child] === 0) queue.push(child) + } + } + if (order.length !== items(state.goals).length) throw new Error(`Proof blueprint graph must be acyclic`) + const depth = new Map([[state.rootGoalID, 0]]) + for (const id of order) { + for (const child of adjacency[id] ?? []) { + depth.set(child, Math.max(depth.get(child) ?? 0, (depth.get(id) ?? 0) + 1)) + } + } + if (Math.max(...depth.values()) > protocol.maxDepth) throw new Error(`Proof blueprint exceeds its depth budget`) + for (const [id, item] of Object.entries(state.decompositions)) { + if (id !== item.id || item.id !== decompositionID(state, item)) { + throw new Error(`Proof blueprint decomposition identity is invalid`) + } + if (!state.goals[item.parentID] || item.childIDs.some((child) => !state.goals[child])) { + throw new Error(`Proof blueprint decomposition references an unknown goal`) + } + if ( + new Set(item.childIDs).size !== item.childIDs.length || + item.childIDs.some((child) => child === item.parentID) || + item.childIDs.some((child, index) => Boolean(index) && item.childIDs[index - 1]!.localeCompare(child) >= 0) + ) { + throw new Error(`Proof blueprint decomposition children must be unique, canonical, and non-recursive`) + } + const attempt = state.attempts[item.attemptID] + if (!attempt || attempt.result !== "accepted" || attempt.decompositionID !== item.id) { + throw new Error(`Proof blueprint decomposition lacks its accepted verifier attempt`) + } + } + for (const [id, item] of Object.entries(state.leases)) { + if (id !== item.id || item.id !== leaseID(state, item)) + throw new Error(`Proof blueprint lease identity is invalid`) + if (!state.goals[item.goalID] || item.expiresAt <= item.issuedAt) { + throw new Error(`Proof blueprint lease references an invalid goal or interval`) + } + if ((item.status === "consumed") !== Boolean(item.consumedAt)) { + throw new Error(`Proof blueprint consumed lease provenance is incomplete`) + } + if (item.consumedAt && (item.consumedAt < item.issuedAt || item.consumedAt > item.expiresAt)) { + throw new Error(`Proof blueprint lease consumption falls outside its interval`) + } + } + const leases = items(state.leases).toSorted((a, b) => a.ordinal - b.ordinal) + if (leases.some((item, index) => item.ordinal !== index)) { + throw new Error(`Proof blueprint lease history must be complete and contiguous`) + } + const open = leases.filter((item) => item.status === "open") + if (open.length > protocol.maxParallel || new Set(open.map((item) => item.goalID)).size !== open.length) { + throw new Error(`Proof blueprint open leases violate frozen parallelism or goal exclusivity`) + } + const attempts = items(state.attempts).toSorted((a, b) => a.ordinal - b.ordinal) + if (attempts.some((item, index) => item.ordinal !== index)) { + throw new Error(`Proof blueprint attempt history must be complete and contiguous`) + } + for (const [id, item] of Object.entries(state.attempts)) { + if (id !== item.id || item.id !== attemptID(state, item)) { + throw new Error(`Proof blueprint attempt identity is invalid`) + } + const lease = state.leases[item.leaseID] + if (!lease || lease.status !== "consumed" || lease.goalID !== item.goalID) { + throw new Error(`Proof blueprint attempt lacks its consumed goal lease`) + } + if ( + item.startedAt < lease.issuedAt || + item.endedAt < item.startedAt || + item.endedAt > lease.expiresAt || + item.recordedAt < item.endedAt || + item.recordedAt !== lease.consumedAt + ) { + throw new Error(`Proof blueprint attempt timing does not match its consumed lease`) + } + } + const consumed = leases.filter((item) => item.status === "consumed") + if ( + consumed.some( + (lease) => + attempts.filter((attempt) => attempt.leaseID === lease.id && attempt.goalID === lease.goalID).length !== 1, + ) + ) { + throw new Error(`Every consumed proof blueprint lease must bind exactly one retained attempt`) + } + for (const goal of items(state.goals)) { + if (direct(state, goal.id).length > protocol.maxAttemptsPerGoal) { + throw new Error(`Proof blueprint goal exceeds its direct-attempt budget`) + } + if (sketches(state, goal.id).length > protocol.maxRefinementsPerGoal + 1) { + throw new Error(`Proof blueprint goal exceeds its refinement budget`) + } + } + statuses(state, protocol) + return state + } + + function parse(data: Record, contract: HarnessContract.Info) { + const protocol = contract.formalProof?.blueprint + if (!protocol) throw new Error(`Harness contract does not enable a proof blueprint`) + return integrity(State.parse(data), contract, protocol) + } + + function expire(state: State, now: number): State { + const leases = Object.fromEntries( + items(state.leases).map((item) => [ + item.id, + item.status === "open" && item.expiresAt <= now ? { ...item, status: "expired" as const } : item, + ]), + ) + if (JSON.stringify(leases) === JSON.stringify(state.leases)) return state + return { ...state, leases, revision: state.revision + 1, updatedAt: now } + } + + export function summarize(state: State, protocol: HarnessContract.ProofBlueprint): Summary { + const parsed = State.parse(state) + const status = statuses(parsed, protocol).goal + const goals = items(parsed.goals) + const attempts = items(parsed.attempts) + const decomposed = new Set(attempts.filter((item) => item.kind === "decomposition").map((item) => item.goalID)).size + const refinements = attempts.filter((item) => item.kind === "decomposition").length - decomposed + return Summary.parse({ + blueprintID: parsed.blueprintID, + status: status.get(parsed.rootGoalID), + goals: goals.length, + proved: goals.filter((item) => status.get(item.id) === "proved").length, + refuted: goals.filter((item) => status.get(item.id) === "refuted").length, + exhausted: goals.filter((item) => status.get(item.id) === "exhausted").length, + decompositions: items(parsed.decompositions).length, + attempts: attempts.length, + rejected: attempts.filter((item) => item.result === "rejected" || item.result === "failed").length, + refinements: Math.max(0, refinements), + openLeases: items(parsed.leases).filter((item) => item.status === "open" && item.expiresAt > Date.now()).length, + revision: parsed.revision, + }) + } + + export function bind(contract: HarnessContract.Info, summary: Summary) { + const protocol = contract.formalProof?.blueprint + if (!protocol) throw new Error(`A proof blueprint summary requires a bound blueprint protocol`) + const parsed = Summary.parse(summary) + if (parsed.blueprintID !== blueprintID(contract, protocol)) { + throw new Error(`Proof blueprint summary belongs to a different harness contract`) + } + return parsed + } + + function render(state: State, protocol: HarnessContract.ProofBlueprint): View { + const status = statuses(state, protocol) + const available = new Set(ready(state, protocol).map((item) => item.id)) + return View.parse({ + schemaVersion: 1, + protocolVersion: "proof-blueprint-view-v1", + runID: state.runID, + sessionID: state.sessionID, + rootGoalID: state.rootGoalID, + summary: summarize(state, protocol), + goals: items(state.goals) + .toSorted((a, b) => a.id.localeCompare(b.id)) + .map((item) => ({ ...item, status: status.goal.get(item.id), ready: available.has(item.id) })), + decompositions: items(state.decompositions) + .toSorted((a, b) => a.createdAt - b.createdAt || a.id.localeCompare(b.id)) + .map((item) => ({ ...item, status: status.branch.get(item.id) })), + attempts: items(state.attempts).toSorted((a, b) => a.ordinal - b.ordinal), + leases: items(state.leases).toSorted((a, b) => a.ordinal - b.ordinal), + }) + } + + export async function initialize(contract: HarnessContract.Info) { + const parsed = HarnessContract.Info.parse(contract) + const protocol = parsed.formalProof?.blueprint + if (!protocol || !parsed.formalProof) throw new Error(`Harness contract does not enable a proof blueprint`) + const now = Date.now() + const spec = GoalSpec.parse({ + statementSHA256: parsed.formalProof.statementSHA256, + declaration: parsed.formalProof.declaration, + module: parsed.formalProof.module, + }) + const id = goalID(spec) + const expected = State.parse({ + schemaVersion: 1, + protocolVersion: "proof-blueprint-state-v1", + blueprintID: blueprintID(parsed, protocol), + runID: parsed.runID, + sessionID: parsed.sessionID, + contractFingerprint: HarnessContract.fingerprint(parsed), + protocolSHA256: digest(protocol), + rootGoalID: id, + goals: { [id]: { id, ...spec, createdAt: now } }, + decompositions: {}, + attempts: {}, + leases: {}, + revision: 0, + createdAt: now, + updatedAt: now, + }) + await JsonStore.update(file(parsed.sessionID), (data) => { + if (!Object.keys(data).length) return expected + return parse(data, parsed) + }) + return read(parsed.sessionID) + } + + export async function read(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${sessionID}`) + const protocol = contract.formalProof?.blueprint + if (!protocol) throw new Error(`Harness contract does not enable a proof blueprint`) + return render(parse(await JsonStore.read(file(sessionID)), contract), protocol) + } + + export async function state(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${sessionID}`) + return parse(await JsonStore.read(file(sessionID)), contract) + } + + export async function lease(contract: HarnessContract.Info, count: number) { + const protocol = contract.formalProof?.blueprint + if (!protocol) throw new Error(`Harness contract does not enable a proof blueprint`) + const size = z.number().int().min(1).max(protocol.maxParallel).parse(count) + const issued: Lease[] = [] + await JsonStore.update(file(contract.sessionID), (data) => { + const current = expire(parse(data, contract), Date.now()) + const now = Date.now() + const active = items(current.leases).filter((item) => item.status === "open" && item.expiresAt > now).length + const choices = ready(current, protocol, now).slice(0, Math.min(size, protocol.maxParallel - active)) + if (!choices.length) return current + const start = items(current.leases).length + const additions = choices.map((goal, index) => { + const draft = { + goalID: goal.id, + revision: current.revision, + ordinal: start + index, + status: "open" as const, + issuedAt: now, + expiresAt: now + protocol.leaseDurationMs, + } + return Lease.parse({ id: leaseID(current, draft), ...draft }) + }) + issued.push(...additions) + return { + ...current, + leases: { ...current.leases, ...Object.fromEntries(additions.map((item) => [item.id, item])) }, + revision: current.revision + 1, + updatedAt: now, + } + }) + return { leases: issued, state: await read(contract.sessionID) } + } + + function timing(value: Submit, lease: Lease, now: number) { + if ( + value.verification.startedAt < lease.issuedAt || + value.verification.endedAt < value.verification.startedAt || + value.verification.endedAt > now || + value.verification.endedAt > lease.expiresAt + ) { + throw new Error(`Proof blueprint verification falls outside its active lease interval`) + } + } + + export async function record(value: Submit, contract: HarnessContract.Info) { + const input = Submit.parse(value) + if (input.sessionID !== contract.sessionID) throw new Error(`Proof blueprint submission belongs to another session`) + const protocol = contract.formalProof?.blueprint + if (!protocol) throw new Error(`Harness contract does not enable a proof blueprint`) + const out: { attemptID?: string; decompositionID?: string } = {} + await JsonStore.update(file(contract.sessionID), (data) => { + const state = parse(data, contract) + const now = Date.now() + const lease = state.leases[input.leaseID] + if (!lease || lease.status !== "open") throw new Error(`Unknown, stale, or consumed proof blueprint lease`) + if (lease.expiresAt <= now) throw new Error(`Proof blueprint lease has expired`) + if (input.verification.compilerArtifactSHA256 !== protocol.compilerArtifactSHA256) { + throw new Error(`Proof blueprint attempt changed its frozen Lean compiler`) + } + timing(input, lease, now) + const status = statuses(state, protocol).goal.get(lease.goalID) + if (status !== "open") throw new Error(`Proof blueprint goal is no longer open`) + const history = direct(state, lease.goalID) + const paths = sketches(state, lease.goalID) + if (input.kind === "direct" && history.length >= protocol.maxAttemptsPerGoal) { + throw new Error(`Proof blueprint goal exhausted its direct-attempt budget`) + } + if (input.kind === "decomposition" && !history.length) { + throw new Error(`Proof blueprint requires a direct proof attempt before decomposition`) + } + if (input.kind === "decomposition" && paths.length >= protocol.maxRefinementsPerGoal + 1) { + throw new Error(`Proof blueprint goal exhausted its refinement budget`) + } + const accepted = + input.verification.statementMatched && input.verification.exitCode === 0 && input.verification.warnings === 0 + const failures = [ + ...(!input.verification.statementMatched ? ["compiler did not check the exact leased statement"] : []), + ...(input.verification.exitCode !== 0 ? ["Lean compiler rejected the artifact"] : []), + ...(input.verification.warnings ? ["Lean compiler emitted warnings"] : []), + ] + const ordinal = items(state.attempts).length + if (input.kind === "direct") { + const result = + !accepted || input.claim === "failure" ? "failed" : input.claim === "proof" ? "proved" : "refuted" + const draft = Attempt.parse({ + id: "0".repeat(64), + ordinal, + goalID: lease.goalID, + leaseID: lease.id, + kind: input.kind, + artifactSHA256: input.artifactSHA256, + result, + claim: input.claim, + transcriptSHA256: input.verification.transcriptSHA256, + feedbackSHA256: input.verification.feedbackSHA256, + failures: [...failures, ...(input.claim === "failure" ? ["attempt declared failure"] : [])], + startedAt: input.verification.startedAt, + endedAt: input.verification.endedAt, + recordedAt: now, + }) + const attempt = Attempt.parse({ ...draft, id: attemptID(state, draft) }) + out.attemptID = attempt.id + return integrity( + { + ...state, + attempts: { ...state.attempts, [attempt.id]: attempt }, + leases: { + ...state.leases, + [lease.id]: { ...lease, status: "consumed", consumedAt: now }, + }, + revision: state.revision + 1, + updatedAt: now, + }, + contract, + protocol, + ) + } + if (input.verification.validatorArtifactSHA256 !== protocol.sketchValidatorArtifactSHA256) { + throw new Error(`Proof blueprint sketch changed its frozen validator`) + } + if ( + input.review.reviewerArtifactSHA256 !== protocol.reviewerArtifactSHA256 || + input.review.promptSHA256 !== protocol.reviewerPromptSHA256 + ) { + throw new Error(`Proof blueprint sketch changed its frozen reviewer or rubric`) + } + const children = input.children.map((item) => GoalSpec.parse(item)) + const childIDs = children.map(goalID).toSorted() + if (new Set(childIDs).size !== childIDs.length) throw new Error(`Proof blueprint children must be unique`) + const placeholders = input.verification.placeholderDeclarations + const expected = children.map((item) => item.declaration).toSorted((a, b) => a.localeCompare(b)) + if ( + new Set(placeholders).size !== placeholders.length || + JSON.stringify(placeholders) !== JSON.stringify(expected) + ) { + throw new Error(`Verified sketch placeholders must exactly equal its introduced child declarations`) + } + const reviewed = input.review.relevant && input.review.easier && input.review.plausible + const graph = accepted && reviewed ? graphIssue(state, lease.goalID, childIDs, protocol) : undefined + const approved = accepted && reviewed && !graph + const reviewFailures = [ + ...(!input.review.relevant ? ["reviewer rejected decomposition relevance"] : []), + ...(!input.review.easier ? ["reviewer rejected decomposition difficulty reduction"] : []), + ...(!input.review.plausible ? ["reviewer rejected decomposition plausibility"] : []), + ...(graph ? [graph] : []), + ] + const attemptBase = { + ordinal, + goalID: lease.goalID, + leaseID: lease.id, + kind: input.kind, + artifactSHA256: input.artifactSHA256, + result: approved ? ("accepted" as const) : ("rejected" as const), + transcriptSHA256: input.verification.validatorTranscriptSHA256, + feedbackSHA256: input.verification.feedbackSHA256, + failures: [...failures, ...reviewFailures], + startedAt: input.verification.startedAt, + endedAt: input.verification.endedAt, + recordedAt: now, + } + const branchBase = Decomposition.parse({ + id: "0".repeat(64), + parentID: lease.goalID, + childIDs, + informalPlanSHA256: input.informalPlanSHA256, + sketchArtifactSHA256: input.artifactSHA256, + sketchTranscriptSHA256: input.verification.validatorTranscriptSHA256, + reviewerTranscriptSHA256: input.review.transcriptSHA256, + attemptID: "0".repeat(64), + createdAt: now, + }) + const branchID = approved ? decompositionID(state, branchBase) : undefined + const draft = Attempt.parse({ + id: "0".repeat(64), + ...attemptBase, + ...(branchID ? { decompositionID: branchID } : {}), + }) + const boundAttempt = Attempt.parse({ ...draft, id: attemptID(state, draft) }) + const complete = branchID + ? Decomposition.parse({ ...branchBase, id: branchID, attemptID: boundAttempt.id }) + : undefined + const additions = approved + ? Object.fromEntries( + children.map((child) => { + const id = goalID(child) + return [id, state.goals[id] ?? Goal.parse({ id, ...child, createdAt: now })] + }), + ) + : {} + out.attemptID = boundAttempt.id + out.decompositionID = complete?.id + return integrity( + { + ...state, + goals: { ...state.goals, ...additions }, + decompositions: complete ? { ...state.decompositions, [complete.id]: complete } : state.decompositions, + attempts: { ...state.attempts, [boundAttempt.id]: boundAttempt }, + leases: { ...state.leases, [lease.id]: { ...lease, status: "consumed", consumedAt: now } }, + revision: state.revision + 1, + updatedAt: now, + }, + contract, + protocol, + ) + }) + return { ...out, state: await read(contract.sessionID) } + } + + export function prompt(contract: HarnessContract.Info) { + if (!contract.formalProof?.blueprint) return "" + return [ + '', + "Use the evaluator-leased AND/OR blueprint for search: try a direct proof first, then propose compiler-checked decompositions whose only placeholders are the introduced child lemmas.", + "A decomposition is an AND node and closes only when every child is proved; a goal is an OR node and may close through any accepted branch. Shared statement hashes reuse one goal.", + "Failed attempts and rejected decompositions are durable. Refinement adds alternatives without mutating proved goals or erasing failures.", + "Reviewer relevance, difficulty, and plausibility are planning heuristics only. Blueprint closure is provisional and never replaces the canonical formal-proof-v1 receipt.", + "Do not claim compiler, reviewer, lease, or receipt authority in agent output.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + const protocol = contract?.formalProof?.blueprint + if (!contract || !protocol) return "" + const current = await state(sessionID).catch(() => undefined) + if (!current) return `${prompt(contract)}\nThe evaluator must initialize the blueprint before leasing proof work.` + const snapshot = render(current, protocol) + const goals = snapshot.goals.filter((item) => item.ready).slice(0, 8) + return [ + prompt(contract), + `Blueprint ${snapshot.summary.status}: ${snapshot.summary.proved}/${snapshot.summary.goals} goals proved, ${snapshot.summary.attempts} attempts, ${snapshot.summary.refinements} refinements.`, + ...goals.map((item) => `Ready goal ${item.id}: ${item.declaration} in ${item.module}.`), + ].join("\n") + } +} diff --git a/backend/cli/src/session/harness/claims.ts b/backend/cli/src/session/harness/claims.ts new file mode 100644 index 00000000..cd832c43 --- /dev/null +++ b/backend/cli/src/session/harness/claims.ts @@ -0,0 +1,569 @@ +import fs from "fs/promises" +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessConfirmation } from "./confirmation" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" + +export namespace HarnessClaims { + export const Kind = z.enum(["descriptive", "statistical", "causal", "mechanistic", "theoretical", "performance"]) + export type Kind = z.infer + + export const Status = z.enum(["untested", "provisional", "inconclusive", "supported", "refuted"]) + export type Status = z.infer + + export const Mode = z.enum([ + "heldout_evaluator", + "clean_replay", + "independent_implementation", + "independent_derivation", + "adversarial_review", + ]) + export type Mode = z.infer + + const Requirement = z + .object({ + independentSources: z.number().int().min(1).max(5), + checks: z + .array(z.string().min(1).max(100)) + .max(24) + .refine((items) => new Set(items).size === items.length, "Required checks must be unique"), + }) + .strict() + + export const Claim = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + runID: z.string().min(1), + sessionID: z.string().min(1), + text: z.string().min(1).max(4_000), + kind: Kind, + importance: z.enum(["supporting", "headline"]), + subject: z + .object({ + uri: z.string().min(1).max(2_048), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + provenanceID: z.string().min(1).max(200).optional(), + }) + .strict(), + requirements: Requirement, + createdBy: z + .object({ + actor: z.string().min(1).max(200), + sessionID: z.string().min(1), + messageID: z.string().min(1).optional(), + }) + .strict(), + createdAt: z.number().int().positive(), + }) + .strict() + export type Claim = z.infer + + export const Evidence = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + claimID: z.string().regex(/^[a-f0-9]{64}$/), + origin: z.enum(["observed", "verified"]), + stance: z.enum(["supports", "refutes", "inconclusive"]), + kind: z.enum([ + "observation", + "measurement", + "statistical_test", + "citation", + "artifact", + "review", + "replay", + "derivation", + "evaluator", + ]), + summary: z.string().min(1).max(2_000), + source: z + .object({ + uri: z.string().min(1).max(2_048), + evaluator: z.string().min(1).max(200).optional(), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + actor: z.string().min(1).max(200), + sessionID: z.string().min(1), + runID: z.string().min(1), + mode: Mode.optional(), + independenceKey: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + confirmationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }) + .strict(), + checks: z.array(HarnessEvaluation.Check).max(64), + evidence: z.array(z.string().min(1).max(1_000)).max(32), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "Claim evidence may contain at most 128 metrics"), + createdAt: z.number().int().positive(), + }) + .strict() + export type Evidence = z.infer + + const State = z + .object({ + schemaVersion: z.literal(1), + claims: z.record(z.string(), Claim), + evidence: z.record(z.string(), Evidence), + revision: z.number().int().nonnegative(), + }) + .strict() + export type State = z.infer + + const VerificationBase = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1), + sessionID: z.string().min(1), + claimID: z.string().regex(/^[a-f0-9]{64}$/), + mode: Mode, + producer: z + .object({ + actor: z.string().min(1).max(200), + sessionID: z.string().min(1), + }) + .strict(), + verifier: z + .object({ + actor: z.string().min(1).max(200), + sessionID: z.string().min(1), + model: z.string().min(1).max(200).optional(), + environment: z.string().min(1).max(500).optional(), + }) + .strict(), + isolation: z + .object({ + freshProcess: z.boolean(), + cleanWorkspace: z.boolean(), + outputWithheld: z.boolean(), + codeIndependent: z.boolean(), + hiddenTestsAccessible: z.literal(false), + }) + .strict(), + source: z + .object({ + uri: z.string().min(1).max(2_048), + evaluator: z.string().min(1).max(200).optional(), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + confirmationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }) + .strict(), + status: HarnessEvaluation.Status, + summary: z.string().min(1).max(2_000), + checks: z.array(HarnessEvaluation.Check).min(1).max(64), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "Verification may contain at most 128 metrics"), + evaluatedAt: z.number().int().positive(), + }) + .strict() + + const rules: Parameters[0] = (value, ctx) => { + if (value.producer.actor === value.verifier.actor) { + ctx.addIssue({ code: "custom", path: ["verifier", "actor"], message: "Verifier must differ from producer" }) + } + const clean = ["clean_replay", "independent_implementation", "independent_derivation"].includes(value.mode) + if (clean && value.producer.sessionID === value.verifier.sessionID) { + ctx.addIssue({ + code: "custom", + path: ["verifier", "sessionID"], + message: "Clean-room verification needs a separate session", + }) + } + if (clean && (!value.isolation.freshProcess || !value.isolation.cleanWorkspace)) { + ctx.addIssue({ + code: "custom", + path: ["isolation"], + message: "Clean-room verification needs a fresh process and clean workspace", + }) + } + if (clean && !value.source.sha256) { + ctx.addIssue({ + code: "custom", + path: ["source", "sha256"], + message: "Clean-room verification must bind the exact source bytes", + }) + } + const independent = ["independent_implementation", "independent_derivation"].includes(value.mode) + if (independent && (!value.isolation.outputWithheld || !value.isolation.codeIndependent)) { + ctx.addIssue({ + code: "custom", + path: ["isolation"], + message: "Independent verification must withhold outputs and use independent code or derivation", + }) + } + if (value.mode === "heldout_evaluator" && !value.isolation.outputWithheld) { + ctx.addIssue({ + code: "custom", + path: ["isolation", "outputWithheld"], + message: "Held-out outputs must remain withheld", + }) + } + if (value.status !== "passed") return + const failed = value.checks.find((check) => check.blocking && check.status !== "passed") + if (!failed) return + ctx.addIssue({ + code: "custom", + path: ["status"], + message: `A passed verification cannot contain a non-passing blocking check: ${failed.id}`, + }) + } + + export const VerificationInfo = VerificationBase.superRefine(rules) + export type VerificationInfo = z.infer + + export const Verification = VerificationBase.extend({ id: z.string().regex(/^[a-f0-9]{64}$/) }).superRefine(rules) + export type Verification = z.infer + + export type View = Claim & { + status: Status + evidence: Evidence[] + independentSources: number + passedChecks: string[] + missingChecks: string[] + } + + const root = path.join(Global.Path.data, "harness", "claims") + const verifications = path.join(Global.Path.data, "harness", "verifications") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const folder = (sessionID: string) => path.join(verifications, encodeURIComponent(sessionID)) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const empty = (): State => ({ schemaVersion: 1, claims: {}, evidence: {}, revision: 0 }) + + const defaults: Record> = { + descriptive: { independentSources: 1, checks: [] }, + statistical: { independentSources: 1, checks: ["estimand", "assumptions", "multiplicity"] }, + causal: { independentSources: 2, checks: ["identification", "confounding", "sensitivity"] }, + mechanistic: { independentSources: 2, checks: ["alternative-explanation", "intervention"] }, + theoretical: { independentSources: 2, checks: ["assumptions", "limiting-case", "independent-derivation"] }, + performance: { independentSources: 1, checks: ["held-out", "baseline", "budget"] }, + } + + async function read(sessionID: string) { + const data = await JsonStore.read(file(sessionID)) + const parsed = State.safeParse(data) + return parsed.success ? parsed.data : empty() + } + + function derive(claim: Claim, items: Evidence[]) { + const verified = items.filter((item) => item.origin === "verified") + if (verified.some((item) => item.stance === "refutes")) return "refuted" as const + const supports = verified.filter((item) => item.stance === "supports") + const sources = new Set( + supports.flatMap((item) => (item.source.independenceKey ? [item.source.independenceKey] : [])), + ) + const checks = new Set( + supports.flatMap((item) => item.checks.filter((check) => check.status === "passed").map((check) => check.id)), + ) + const complete = claim.requirements.checks.every((check) => checks.has(check)) + if (sources.size >= claim.requirements.independentSources && complete) return "supported" as const + if (verified.length) return "inconclusive" as const + if (items.length) return "provisional" as const + return "untested" as const + } + + export async function declare(input: { + sessionID: string + actor: string + messageID?: string + text: string + kind: Kind + importance: "supporting" | "headline" + subject: Claim["subject"] + requirements?: { independentSources?: number; checks?: string[] } + }) { + const contract = await HarnessContract.read(input.sessionID) + if (input.kind === "performance" && input.importance === "headline" && !input.subject.sha256) { + throw new Error(`A headline performance claim must bind an immutable subject SHA-256`) + } + const runID = contract?.runID ?? `session:${input.sessionID}` + const base = defaults[input.kind] + const requirements = Requirement.parse({ + independentSources: Math.max(base.independentSources, input.requirements?.independentSources ?? 1), + checks: [...new Set([...base.checks, ...(input.requirements?.checks ?? [])])], + }) + const id = digest({ runID, text: input.text, kind: input.kind, subject: input.subject }) + const claim = Claim.parse({ + id, + runID, + sessionID: input.sessionID, + text: input.text, + kind: input.kind, + importance: input.importance, + subject: input.subject, + requirements, + createdBy: { actor: input.actor, sessionID: input.sessionID, messageID: input.messageID }, + createdAt: Date.now(), + }) + await JsonStore.update(file(input.sessionID), (data) => { + const state = Object.keys(data).length ? State.parse(data) : empty() + if (state.claims[id]) return state + return { ...state, claims: { ...state.claims, [id]: claim }, revision: state.revision + 1 } + }) + return (await read(input.sessionID)).claims[id]! + } + + export async function observe(input: { + sessionID: string + claimID: string + actor: string + kind: Evidence["kind"] + stance: Evidence["stance"] + summary: string + source: { uri: string; sha256?: string } + evidence?: string[] + metrics?: Record + }) { + const state = await read(input.sessionID) + const claim = state.claims[input.claimID] + if (!claim) throw new Error(`Unknown claim ${input.claimID}`) + const id = digest({ + claimID: claim.id, + actor: input.actor, + kind: input.kind, + summary: input.summary, + source: input.source, + }) + const item = Evidence.parse({ + id, + claimID: claim.id, + origin: "observed", + stance: input.stance, + kind: input.kind, + summary: input.summary, + source: { + ...input.source, + actor: input.actor, + sessionID: input.sessionID, + runID: claim.runID, + }, + checks: [], + evidence: input.evidence ?? [], + metrics: input.metrics ?? {}, + createdAt: Date.now(), + }) + await JsonStore.update(file(input.sessionID), (data) => { + const current = State.parse(data) + if (current.evidence[id]) return current + return { ...current, evidence: { ...current.evidence, [id]: item }, revision: current.revision + 1 } + }) + return item + } + + function view(state: State, claim: Claim): View { + const items = Object.values(state.evidence) + .filter((item) => item.claimID === claim.id) + .toSorted((a, b) => a.createdAt - b.createdAt || a.id.localeCompare(b.id)) + const supports = items.filter((item) => item.origin === "verified" && item.stance === "supports") + const sources = new Set( + supports.flatMap((item) => (item.source.independenceKey ? [item.source.independenceKey] : [])), + ) + const passed = new Set( + supports.flatMap((item) => item.checks.filter((check) => check.status === "passed").map((check) => check.id)), + ) + return { + ...claim, + status: derive(claim, items), + evidence: items, + independentSources: sources.size, + passedChecks: [...passed].toSorted(), + missingChecks: claim.requirements.checks.filter((check) => !passed.has(check)), + } + } + + export async function get(sessionID: string, claimID: string) { + const state = await read(sessionID) + const claim = state.claims[claimID] + return claim ? view(state, claim) : null + } + + export async function list(sessionID: string) { + const state = await read(sessionID) + return Object.values(state.claims) + .map((claim) => view(state, claim)) + .toSorted((a, b) => a.createdAt - b.createdAt || a.id.localeCompare(b.id)) + } + + async function heldout(parsed: VerificationInfo) { + if (parsed.mode !== "heldout_evaluator") return + const contract = await HarnessContract.read(parsed.sessionID) + const evaluator = contract?.confirmation?.claim.evaluator.name ?? contract?.benchmark.evaluator + if (!contract || parsed.source.evaluator !== evaluator) { + throw new Error(`Held-out verification does not match the bound benchmark evaluator`) + } + if (contract.confirmation) { + if (!parsed.source.confirmationReceiptID) { + throw new Error(`Confirmation-enabled held-out verification requires its canonical claim receipt`) + } + const receipt = await HarnessConfirmation.assert(contract, parsed.source.confirmationReceiptID) + if (parsed.source.sha256 !== receipt.selection.candidateArtifact.sha256) { + throw new Error(`Held-out verification does not bind the confirmed candidate artifact`) + } + if ( + parsed.status !== receipt.status || + parsed.metrics[contract.confirmation.claim.metric] !== receipt.score || + parsed.evaluatedAt < receipt.recordedAt + ) { + throw new Error(`Held-out verification does not match the sealed claim result`) + } + } + if (!contract.confirmation && parsed.source.confirmationReceiptID) { + throw new Error(`Legacy held-out verification cannot cite a sealed confirmation receipt`) + } + } + + export async function stage(input: VerificationInfo) { + const parsed = VerificationInfo.parse(input) + const state = await read(input.sessionID) + const claim = state.claims[input.claimID] + if (!claim) throw new Error(`Unknown claim ${input.claimID}`) + if (parsed.runID !== claim.runID || parsed.sessionID !== claim.sessionID) { + throw new Error(`Verification belongs to a different claim run`) + } + if (parsed.producer.actor !== claim.createdBy.actor || parsed.producer.sessionID !== claim.createdBy.sessionID) { + throw new Error(`Verification does not identify the claim producer`) + } + await heldout(parsed) + const id = digest(parsed) + const verification = Verification.parse({ ...parsed, id }) + await JsonStore.update(path.join(folder(input.sessionID), `${id}.json`), (data) => { + if (!Object.keys(data).length) return verification + const current = Verification.parse(data) + if (current.id === id) return current + throw new Error(`Verification ${id} is immutable`) + }) + return verification + } + + function kind(mode: Mode): Evidence["kind"] { + if (mode === "heldout_evaluator") return "evaluator" + if (mode === "clean_replay" || mode === "independent_implementation") return "replay" + if (mode === "independent_derivation") return "derivation" + return "review" + } + + export async function reconcile(sessionID: string) { + const names = await fs.readdir(folder(sessionID)).catch(() => []) + const records = await Promise.all( + names + .filter((name) => name.endsWith(".json")) + .map((name) => + Bun.file(path.join(folder(sessionID), name)) + .json() + .then((value) => Verification.parse(value)), + ), + ) + for (const record of records) { + const body = structuredClone(record) as Record + delete body.id + if (digest(body) !== record.id) throw new Error(`Verification ${record.id} is not content-addressed`) + await heldout(record) + } + await JsonStore.update(file(sessionID), (data) => { + const state = Object.keys(data).length ? State.parse(data) : empty() + const additions = records.flatMap((record) => { + const claim = state.claims[record.claimID] + if (!claim || state.evidence[record.id]) return [] + if (record.runID !== claim.runID || record.sessionID !== claim.sessionID) { + throw new Error(`Verification ${record.id} belongs to a different claim run`) + } + if ( + record.producer.actor !== claim.createdBy.actor || + record.producer.sessionID !== claim.createdBy.sessionID + ) { + throw new Error(`Verification ${record.id} does not identify the claim producer`) + } + const stance = + record.status === "passed" + ? ("supports" as const) + : record.status === "failed" + ? ("refutes" as const) + : ("inconclusive" as const) + return [ + Evidence.parse({ + id: record.id, + claimID: claim.id, + origin: "verified", + stance, + kind: kind(record.mode), + summary: record.summary, + source: { + uri: record.source.uri, + evaluator: record.source.evaluator, + sha256: record.source.sha256, + actor: record.verifier.actor, + sessionID: record.verifier.sessionID, + runID: record.runID, + mode: record.mode, + independenceKey: digest({ actor: record.verifier.actor, sessionID: record.verifier.sessionID }), + confirmationReceiptID: record.source.confirmationReceiptID, + }, + checks: record.checks, + evidence: record.evidence, + metrics: record.metrics, + createdAt: record.evaluatedAt, + }), + ] + }) + if (!additions.length) return state + return { + ...state, + evidence: { ...state.evidence, ...Object.fromEntries(additions.map((item) => [item.id, item])) }, + revision: state.revision + additions.length, + } + }) + return list(sessionID) + } + + export async function verify(input: VerificationInfo) { + const verification = await stage(input) + await reconcile(input.sessionID) + return { verification, claim: await get(input.sessionID, input.claimID) } + } + + const escape = (value: string) => + value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """) + + export async function prompt(sessionID: string) { + const claims = (await list(sessionID)).filter((claim) => claim.status !== "supported") + if (!claims.length) return "" + const lines = [ + '', + "Claim status is computed from backend verification. Observations never count as support.", + ] + for (const claim of claims.slice(0, 12)) { + const block = [ + ``, + escape(claim.text).slice(0, 800), + `Independent sources: ${claim.independentSources}/${claim.requirements.independentSources}`, + `Missing checks: ${escape(claim.missingChecks.join(", ") || "none")}`, + "", + ] + if ([...lines, ...block, ""].join("\n").length > 3_500) break + lines.push(...block) + } + lines.push("") + return lines.join("\n") + } +} diff --git a/backend/cli/src/session/harness/confirmation.ts b/backend/cli/src/session/harness/confirmation.ts new file mode 100644 index 00000000..193b56a0 --- /dev/null +++ b/backend/cli/src/session/harness/confirmation.ts @@ -0,0 +1,403 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" +import { HarnessDomain } from "./domain" +import { HarnessEvaluation } from "./evaluation" +import { HarnessMeta } from "./meta" +import { HarnessSearch } from "./search" + +export namespace HarnessConfirmation { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + confirmationToken: Token, + }) + .strict() + export type Access = z.infer + + export const Submit = z + .object({ + schemaVersion: z.literal(1), + sessionID: z.string().min(1).max(240), + confirmationToken: Token, + candidateSHA256: Hash, + manifestSHA256: Hash, + validatorSHA256: Hash, + environmentSHA256: Hash, + outcome: z.enum(["completed", "failed", "inconclusive"]), + score: z.number().finite().optional(), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "A confirmation may contain at most 128 metrics") + .default({}), + checks: z.array(HarnessEvaluation.Check).min(1).max(128), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + usage: HarnessEvaluation.Usage.optional(), + outputSHA256: Hash, + evaluatedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.checks.map((item) => item.id)).size !== value.checks.length) { + ctx.addIssue({ code: "custom", path: ["checks"], message: "Confirmation check IDs must be unique" }) + } + if (!value.checks.some((item) => item.blocking)) { + ctx.addIssue({ code: "custom", path: ["checks"], message: "Confirmation requires a blocking check" }) + } + if (value.outcome === "completed" && value.score === undefined) { + ctx.addIssue({ code: "custom", path: ["score"], message: "A completed confirmation requires a score" }) + } + if (value.outcome !== "completed" && value.score !== undefined) { + ctx.addIssue({ + code: "custom", + path: ["score"], + message: "An incomplete confirmation cannot publish a partial score", + }) + } + }) + export type Submit = z.input + type Submission = z.infer + type Audit = Pick< + Submission, + "manifestSHA256" | "validatorSHA256" | "environmentSHA256" | "outcome" | "score" | "metrics" | "checks" + > + + const Artifact = z + .object({ + uri: z.string().min(1).max(2_048), + sha256: Hash, + }) + .strict() + + const SelectionBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("terminal-verified-best-selection-v1"), + selectionID: Hash, + contractSHA256: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1).max(240), + runID: z.string().min(1).max(240), + searchRevision: z.number().int().nonnegative(), + stopReason: HarnessSearch.Stop, + candidateID: Hash, + candidateArtifact: Artifact, + candidateCreatedAt: z.number().int().positive(), + optimizationResultSHA256: Hash, + optimizationEvaluationSHA256: Hash, + selectedAt: z.number().int().positive(), + }) + .strict() + + export const Selection = SelectionBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.selectionID + if (digest(stable) === value.selectionID) return + ctx.addIssue({ code: "custom", path: ["selectionID"], message: "Confirmation selection hash is invalid" }) + }) + export type Selection = z.infer + + const ReceiptBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("sealed-confirmation-receipt-v1"), + receiptID: Hash, + contractSHA256: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1).max(240), + runID: z.string().min(1).max(240), + selection: Selection, + claim: HarnessContract.ConfirmationClaim, + outcome: z.enum(["completed", "failed", "inconclusive"]), + status: HarnessEvaluation.Status, + score: z.number().finite().optional(), + metrics: z.record(z.string(), z.number().finite()), + checks: z.array(HarnessEvaluation.Check).min(1).max(128), + failures: z.array(z.string().min(1).max(500)).max(256), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(4_224), + usage: HarnessEvaluation.Usage.optional(), + outputSHA256: Hash, + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + + export const Receipt = ReceiptBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Sealed confirmation receipt hash is invalid" }) + }) + export type Receipt = z.infer + + const Claim = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + contractSHA256: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1).max(240), + selectionID: Hash, + }) + .strict() + + const root = path.join(Global.Path.data, "harness", "confirmations") + const file = (receiptID: string) => path.join(root, `${receiptID}.json`) + const claimfile = (sessionID: string) => path.join(root, "sessions", `${digest(sessionID)}.json`) + + async function claim(sessionID: string) { + const data = await JsonStore.read(claimfile(sessionID)) + if (!Object.keys(data).length) return null + return Claim.parse(data) + } + + function audit(protocol: HarnessContract.Confirmation, input: Audit) { + if (input.manifestSHA256 !== protocol.claim.manifestSHA256) { + throw new Error(`Claim evaluation changed the frozen claim manifest`) + } + if (input.validatorSHA256 !== protocol.claim.validatorSHA256) { + throw new Error(`Claim evaluation changed the frozen validator`) + } + if (input.environmentSHA256 !== protocol.claim.environmentSHA256) { + throw new Error(`Claim evaluation changed the frozen environment`) + } + const metric = input.metrics[protocol.claim.metric] + if (input.outcome === "completed" && metric !== input.score) { + throw new Error(`Claim score does not match the bound ${protocol.claim.metric} metric`) + } + if (input.outcome !== "completed" && metric !== undefined) { + throw new Error(`An incomplete confirmation cannot publish a partial claim metric`) + } + const blocked = input.checks.filter((item) => item.blocking && item.status !== "passed") + const target = + input.score !== undefined && + (protocol.claim.direction === "maximize" + ? input.score >= protocol.claim.target + : input.score <= protocol.claim.target) + const status = + input.outcome === "failed" || blocked.some((item) => item.status === "failed") + ? ("failed" as const) + : input.outcome === "inconclusive" || blocked.some((item) => item.status === "inconclusive") + ? ("inconclusive" as const) + : target + ? ("passed" as const) + : ("failed" as const) + const failures = [ + ...(input.outcome === "failed" ? ["claim evaluator failed before producing a complete result"] : []), + ...(input.outcome === "inconclusive" ? ["claim evaluator returned an inconclusive result"] : []), + ...blocked.map((item) => `blocking-check:${item.id}:${item.status}`), + ...(input.outcome === "completed" && !target + ? [`claim score ${input.score} does not satisfy ${protocol.claim.direction} target ${protocol.claim.target}`] + : []), + ] + return { status, failures } + } + + export async function select(contract: HarnessContract.Info) { + const protocol = contract.confirmation + if (!protocol) throw new Error(`Harness contract does not require sealed confirmation`) + await HarnessMeta.assertPromotable(contract) + const state = await HarnessSearch.read(contract.sessionID) + if (state.runID !== contract.runID) throw new Error(`Search state does not match the bound harness run`) + if (state.status !== "completed" || !state.stopReason || !state.bestID) { + throw new Error(`Sealed confirmation requires a terminal search with one verified winner`) + } + if (Object.values(state.reservations).some((item) => item.status === "open")) { + throw new Error(`Sealed confirmation cannot select while candidate reservations remain open`) + } + const candidate = state.candidates[state.bestID] + if (!candidate || candidate.result?.source !== "verified" || candidate.result.status !== "passed") { + throw new Error(`The server-selected confirmation subject is not a verified passing candidate`) + } + const evaluation = (await HarnessEvaluation.list(contract.sessionID)).findLast( + (item) => + item.subject?.type === "candidate" && item.subject.id === candidate.id && HarnessEvaluation.verified(item), + ) + if (!evaluation) throw new Error(`The terminal winner has no durable verified optimization evaluation`) + const stable = { + schemaVersion: 1 as const, + protocolVersion: "terminal-verified-best-selection-v1" as const, + contractSHA256: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + sourceSessionID: contract.sessionID, + runID: contract.runID, + searchRevision: state.revision, + stopReason: state.stopReason, + candidateID: candidate.id, + candidateArtifact: candidate.artifact, + candidateCreatedAt: candidate.createdAt, + optimizationResultSHA256: digest(candidate.result), + optimizationEvaluationSHA256: HarnessEvaluation.fingerprint(evaluation), + selectedAt: state.updatedAt, + } + return Selection.parse({ ...stable, selectionID: digest(stable) }) + } + + function comparable(receipt: Receipt, stable: Omit) { + const current = structuredClone(receipt) as Record + delete current.receiptID + delete current.recordedAt + return JSON.stringify(current) === JSON.stringify(stable) + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const protocol = contract.confirmation + if (!protocol) throw new Error(`Harness contract does not require sealed confirmation`) + if (value.sessionID !== contract.sessionID) throw new Error(`Confirmation session does not match its contract`) + const selection = await select(contract) + if (value.candidateSHA256 !== selection.candidateArtifact.sha256) { + throw new Error(`Claim evaluator did not evaluate the server-selected candidate artifact`) + } + const now = Date.now() + if (value.evaluatedAt < selection.selectedAt || value.evaluatedAt > now) { + throw new Error(`Claim evaluation timestamp is outside the terminal selection interval`) + } + const result = audit(protocol, value) + if (result.status === "passed") HarnessDomain.assert(contract.packs ?? [], value.checks) + const stable = { + schemaVersion: 1 as const, + protocolVersion: "sealed-confirmation-receipt-v1" as const, + contractSHA256: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + sourceSessionID: contract.sessionID, + runID: contract.runID, + selection, + claim: protocol.claim, + outcome: value.outcome, + status: result.status, + score: value.score, + metrics: value.metrics, + checks: value.checks, + failures: result.failures, + evidence: [...new Set([...value.evidence, ...value.checks.flatMap((item) => item.evidence)])].toSorted(), + usage: value.usage, + outputSHA256: value.outputSHA256, + evaluatedAt: value.evaluatedAt, + } + const active = await claim(contract.sessionID) + if (active) { + const receipt = await read(active.receiptID) + if (!receipt) throw new Error(`The session's frozen confirmation receipt is corrupt`) + if (comparable(receipt, stable)) return receipt + throw new Error(`The session already has a frozen confirmation receipt; holdout retries are forbidden`) + } + const body = { ...stable, recordedAt: now } + const receipt = Receipt.parse({ ...body, receiptID: digest(body) }) + await JsonStore.update(file(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Sealed confirmation receipt is immutable once recorded`) + }) + const saved = await read(receipt.receiptID) + if (!saved) throw new Error(`Sealed confirmation receipt was not durable after recording`) + const statement = Claim.parse({ + schemaVersion: 1, + receiptID: saved.receiptID, + contractSHA256: saved.contractSHA256, + protocolSHA256: saved.protocolSHA256, + sourceSessionID: saved.sourceSessionID, + selectionID: saved.selection.selectionID, + }) + await JsonStore.update(claimfile(contract.sessionID), async (data) => { + if (!Object.keys(data).length) return statement + const current = Claim.parse(data) + if (current.receiptID === statement.receiptID) return current + const winner = await read(current.receiptID) + if (winner && comparable(winner, stable)) return current + throw new Error(`The session already has a frozen confirmation receipt; holdout retries are forbidden`) + }) + const frozen = await claim(contract.sessionID) + if (!frozen) throw new Error(`Sealed confirmation receipt was not durably frozen for its session`) + const winner = await read(frozen.receiptID) + if (winner && comparable(winner, stable)) return winner + throw new Error(`The session already has a different frozen confirmation receipt`) + } + + export async function read(receiptID: string) { + const id = Hash.parse(receiptID) + const data = await JsonStore.read(file(id)) + const parsed = Receipt.safeParse(data) + return parsed.success && parsed.data.receiptID === id ? parsed.data : null + } + + export function binds(contract: HarnessContract.Info, input: Receipt) { + const receipt = Receipt.parse(input) + const protocol = contract.confirmation + if (!protocol) throw new Error(`Receipt cites a confirmation protocol that is not bound`) + if ( + receipt.contractSHA256 !== HarnessContract.fingerprint(contract) || + receipt.protocolSHA256 !== digest(protocol) || + receipt.sourceSessionID !== contract.sessionID || + receipt.runID !== contract.runID || + JSON.stringify(receipt.claim) !== JSON.stringify(protocol.claim) + ) { + throw new Error(`Sealed confirmation receipt does not match the bound contract`) + } + return receipt + } + + export async function assert(contract: HarnessContract.Info, receiptID: string) { + const stored = await read(receiptID) + if (!stored) throw new Error(`Unknown or corrupt sealed confirmation receipt ${receiptID}`) + const receipt = binds(contract, stored) + const protocol = contract.confirmation! + const active = await claim(contract.sessionID) + if (active?.receiptID !== receipt.receiptID) { + throw new Error(`Receipt is not the session's canonical sealed confirmation`) + } + const selection = await select(contract) + if (JSON.stringify(selection) !== JSON.stringify(receipt.selection)) { + throw new Error(`Sealed confirmation receipt changed the server-selected terminal winner`) + } + const result = audit(protocol, { + manifestSHA256: receipt.claim.manifestSHA256, + validatorSHA256: receipt.claim.validatorSHA256, + environmentSHA256: receipt.claim.environmentSHA256, + outcome: receipt.outcome, + score: receipt.score, + metrics: receipt.metrics, + checks: receipt.checks, + }) + if (receipt.status !== result.status || JSON.stringify(receipt.failures) !== JSON.stringify(result.failures)) { + throw new Error(`Sealed confirmation receipt does not match the backend-derived verdict`) + } + if (receipt.status === "passed") HarnessDomain.assert(contract.packs ?? [], receipt.checks) + if (receipt.evaluatedAt < receipt.selection.selectedAt || receipt.evaluatedAt > receipt.recordedAt) { + throw new Error(`Sealed confirmation receipt predates its terminal candidate selection`) + } + return receipt + } + + export async function current(contract: HarnessContract.Info) { + if (!contract.confirmation) return null + const active = await claim(contract.sessionID) + return active ? assert(contract, active.receiptID) : null + } + + export function prompt(contract: HarnessContract.Info) { + const protocol = contract.confirmation + if (!protocol) return "" + return [ + "", + "Optimization evaluator results are provisional search signals, not final benchmark evidence.", + `Search may use only the committed ${protocol.optimization.split} optimization split. The distinct ${protocol.claim.split} claim split is reserved for one post-search confirmation.`, + "The backend selects exactly one verified winner only after search is terminal. The agent may not nominate, swap, or retry a claim candidate.", + "Do not access the claim manifest, claim evaluator capability, claim outputs, or partial claim metrics during search.", + "Claim results never feed candidate ranking, adaptive control, hindsight memory, or learned-skill generation. Reports remain provisional until the immutable confirmation receipt exists.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + return contract ? prompt(contract) : "" + } +} diff --git a/backend/cli/src/session/harness/contract.ts b/backend/cli/src/session/harness/contract.ts new file mode 100644 index 00000000..edaf211e --- /dev/null +++ b/backend/cli/src/session/harness/contract.ts @@ -0,0 +1,2025 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessPack } from "./pack" + +export namespace HarnessContract { + export const Profile = z.enum(["react", "optimize", "reproduce", "theory", "numerical", "training", "forecast"]) + export type Profile = z.infer + + export const Family = z.enum(["data", "biology", "physics", "chemistry", "ml", "generalist", "custom"]) + export type Family = z.infer + + export const Objective = z + .object({ + metric: z.string().min(1).max(200), + direction: z.enum(["maximize", "minimize"]), + }) + .strict() + export type Objective = z.infer + + export const Objectives = z + .array(Objective) + .max(8) + .refine( + (items) => new Set(items.map((item) => item.metric)).size === items.length, + "Objective metrics must be unique", + ) + export type Objectives = z.infer + + export const Search = z + .object({ + protocolVersion: z.literal("adaptive-search-v1"), + signal: z + .object({ + source: z.literal("verified-final-evaluations"), + decay: z.literal(0.9), + epsilon: z.literal(1e-8), + }) + .strict(), + local: z + .object({ + minIntensity: z.literal(0.15), + maxIntensity: z.literal(0.5), + }) + .strict(), + global: z + .object({ + exploration: z.literal(Math.SQRT2), + minVisits: z.literal(2), + }) + .strict(), + stagnation: z + .object({ + patience: z.literal(5), + maxSignal: z.literal(0.02), + }) + .strict(), + }) + .strict() + export type Search = z.infer + + export const adaptiveSearch: Search = { + protocolVersion: "adaptive-search-v1", + signal: { source: "verified-final-evaluations", decay: 0.9, epsilon: 1e-8 }, + local: { minIntensity: 0.15, maxIntensity: 0.5 }, + global: { exploration: Math.SQRT2, minVisits: 2 }, + stagnation: { patience: 5, maxSignal: 0.02 }, + } + + export const Topology = z.enum([ + "auto", + "solo", + "centralized", + "fork_join", + "tournament", + "evolution", + "verifier_loop", + ]) + export type Topology = z.infer + + export const Role = z.enum([ + "generation", + "proximity", + "reflection", + "ranking", + "evolution", + "revision", + "verification", + "investigation", + "simulation", + "synthesis", + ]) + export type Role = z.infer + + export const Traits = z + .object({ + decomposability: z.number().min(0).max(1), + sequentiality: z.number().min(0).max(1), + toolIntensity: z.number().min(0).max(1), + uncertainty: z.number().min(0).max(1), + verificationRisk: z.number().min(0).max(1), + novelty: z.number().min(0).max(1), + crossDomain: z.number().min(0).max(1), + }) + .strict() + export type Traits = z.infer + + export const Adaptive = z + .object({ + protocolVersion: z.literal("marginal-utility-v1"), + minRounds: z.number().int().min(1).max(8), + patience: z.number().int().min(1).max(7), + minUtilityGain: z.number().finite().nonnegative().max(1), + maxUncertainty: z.number().finite().min(0).max(1), + targetUtility: z.number().finite().min(0).max(1).optional(), + }) + .strict() + export type Adaptive = z.infer + + export const Repair = z + .object({ + protocolVersion: z.literal("verifier-routed-v1"), + minConfidence: z.number().finite().min(0.5).max(1), + }) + .strict() + export type Repair = z.infer + + export const Orchestration = z + .object({ + topology: Topology, + traits: Traits.optional(), + maxWorkers: z.number().int().min(1).max(2), + maxRounds: z.number().int().min(1).max(8), + roles: z + .array(Role) + .min(1) + .max(Role.options.length) + .refine((items) => new Set(items).size === items.length, "Orchestration roles must be unique") + .optional(), + minIndependentVerifiers: z.number().int().min(1).max(2), + adaptive: Adaptive.optional(), + repair: Repair.optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.adaptive && value.topology !== "evolution") { + ctx.addIssue({ + code: "custom", + path: ["topology"], + message: "Adaptive marginal-utility control requires an explicit evolution topology", + }) + } + if (value.adaptive && value.adaptive.minRounds > value.maxRounds) { + ctx.addIssue({ + code: "custom", + path: ["adaptive", "minRounds"], + message: "Adaptive minimum rounds exceed the orchestration round budget", + }) + } + if (value.adaptive && Math.max(value.adaptive.minRounds, value.adaptive.patience + 1) > value.maxRounds) { + ctx.addIssue({ + code: "custom", + path: ["adaptive", "patience"], + message: "Adaptive patience cannot be observed within the orchestration round budget", + }) + } + if (value.repair && value.topology !== "verifier_loop") { + ctx.addIssue({ + code: "custom", + path: ["topology"], + message: "Verifier-routed repair requires an explicit verifier_loop topology", + }) + } + if (value.topology === "verifier_loop" && !value.repair) { + ctx.addIssue({ + code: "custom", + path: ["repair"], + message: "Verifier loop topology requires a verifier-routed repair contract", + }) + } + if (value.topology === "verifier_loop" && value.minIndependentVerifiers !== 2) { + ctx.addIssue({ + code: "custom", + path: ["minIndependentVerifiers"], + message: "Verifier loop topology requires two independent verifiers", + }) + } + if (value.adaptive && value.repair) { + ctx.addIssue({ + code: "custom", + path: ["repair"], + message: "Adaptive evolution and verifier-routed repair are separate bounded controllers", + }) + } + }) + export type Orchestration = z.infer + + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + + export const AuditTransfer = z + .object({ + protocolVersion: z.literal("score-history-prior-v1"), + poolSHA256: Hash, + sourceManifestSHA256: Hash, + selectionSHA256: Hash, + selectionMethod: z.enum(["pca-gmm-profile-v1", "holdout-embedding-gmm-v1"]), + sourceModels: z + .array(z.string().min(1).max(240)) + .min(3) + .max(64) + .refine((items) => new Set(items).size === items.length, "Audit transfer source models must be unique"), + calibrationSamples: z.number().int().min(2).max(64), + maxCalibrationMAE: z.number().positive().max(1), + }) + .strict() + export type AuditTransfer = z.infer + + export const Audit = z + .object({ + mode: z.enum(["performance", "failure", "hybrid"]), + budget: z.number().int().min(2).max(512), + minSamples: z.number().int().min(2).max(512), + noiseVariance: z.number().positive().max(2).default(0.05), + lengthscale: z.number().positive().max(100).default(1), + beta: z.number().nonnegative().max(10).default(1.96), + failureThreshold: z.number().min(0).max(1).default(0.5), + tolerance: z.number().positive().max(1).default(0.02), + maxUncertainty: z.number().positive().max(1).default(0.05), + estimationWeight: z.number().min(0).max(1).default(0.5), + diversityWeight: z.number().min(0).max(0.5).default(0.2), + coverageWeight: z.number().min(0).max(0.5).default(0.2), + targetFailures: z.number().int().positive().max(512).optional(), + transfer: AuditTransfer.optional(), + promotionRequired: z.boolean().optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.minSamples > value.budget) { + ctx.addIssue({ code: "custom", path: ["minSamples"], message: "Audit minimum samples exceed its budget" }) + } + if (value.diversityWeight + value.coverageWeight > 1) { + ctx.addIssue({ code: "custom", message: "Audit diversity and coverage weights cannot exceed one" }) + } + if (value.targetFailures !== undefined && value.targetFailures > value.budget) { + ctx.addIssue({ code: "custom", path: ["targetFailures"], message: "Audit failure target exceeds its budget" }) + } + if (value.transfer && value.transfer.calibrationSamples > value.budget) { + ctx.addIssue({ + code: "custom", + path: ["transfer", "calibrationSamples"], + message: "Audit transfer calibration exceeds its budget", + }) + } + if (value.promotionRequired && !value.transfer) { + ctx.addIssue({ + code: "custom", + path: ["promotionRequired"], + message: "Audit promotion requires a transfer-qualified proactive audit", + }) + } + if (value.promotionRequired && value.mode === "failure") { + ctx.addIssue({ + code: "custom", + path: ["mode"], + message: "A failure-only audit cannot qualify a population performance evaluation", + }) + } + }) + export type Audit = z.infer + + export const FailureValidatorKind = z.enum(["correctness", "topic", "novelty"]) + export type FailureValidatorKind = z.infer + + const FailureIdentity = z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + promptSHA256: Hash, + configSHA256: Hash, + }) + .strict() + + export const FailureTopic = z + .object({ + id: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$/), + commitment: Hash, + }) + .strict() + export type FailureTopic = z.infer + + export const FailureDiscovery = z + .object({ + protocolVersion: z.literal("topic-aware-failure-v1"), + sourcePoolSHA256: Hash, + topicModel: z + .object({ + kind: z.enum(["predefined", "bertopic"]), + identity: FailureIdentity, + }) + .strict(), + topics: z + .array(FailureTopic) + .min(2) + .max(64) + .refine((items) => new Set(items.map((item) => item.id)).size === items.length, "Failure topics must be unique") + .refine( + (items) => new Set(items.map((item) => item.commitment)).size === items.length, + "Failure topic commitments must be unique", + ) + .refine( + (items) => items.every((item, index) => !index || items[index - 1]!.id < item.id), + "Failure topics must be sorted by ID", + ), + generator: FailureIdentity, + validators: z + .array( + z + .object({ + kind: FailureValidatorKind, + identity: FailureIdentity, + }) + .strict(), + ) + .length(FailureValidatorKind.options.length) + .refine( + (items) => new Set(items.map((item) => item.kind)).size === items.length, + "Failure discovery requires every validator class", + ) + .refine( + (items) => items.every((item, index) => item.kind === FailureValidatorKind.options[index]), + "Failure discovery validators must use canonical kind order", + ), + embedding: z + .object({ + identity: FailureIdentity, + dimensions: z.number().int().min(2).max(64), + regularization: z.number().positive().max(0.01).default(1e-6), + }) + .strict(), + budget: z.number().int().min(2).max(512), + anchorsPerAttempt: z.number().int().min(1).max(8), + exploration: z.number().positive().max(4).default(Math.SQRT2), + failureThreshold: z.number().min(0).max(1), + targetFailures: z.number().int().positive().max(512).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.targetFailures !== undefined && value.targetFailures > value.budget) { + ctx.addIssue({ + code: "custom", + path: ["targetFailures"], + message: "Failure discovery target exceeds its attempt budget", + }) + } + if (value.budget < value.topics.length) { + ctx.addIssue({ + code: "custom", + path: ["budget"], + message: "Failure discovery budget must initialize every topic arm", + }) + } + const actors = [value.generator, ...value.validators.map((item) => item.identity)] + const identities = actors.map((item) => `${item.promptSHA256}:${item.configSHA256}`) + if (new Set(identities).size !== identities.length) { + ctx.addIssue({ + code: "custom", + path: ["validators"], + message: "Failure generator and validators must use distinct prompt/config commitments", + }) + } + }) + export type FailureDiscovery = z.infer + + export const ObjectiveAudit = z + .object({ + schemaVersion: z.literal(1), + planSHA256: Hash, + validatorSHA256: Hash, + contractSHA256: Hash, + guardIDs: z + .array(z.string().min(1).max(200)) + .min(1) + .max(64) + .refine((items) => new Set(items).size === items.length, "Objective audit guards must be unique"), + }) + .strict() + export type ObjectiveAudit = z.infer + + export const IntegrityAuditKind = z.enum(["test_item_contamination", "external_model_use", "benchmark_lookup"]) + export type IntegrityAuditKind = z.infer + + export const IntegrityAuditor = z + .object({ + kind: IntegrityAuditKind, + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + promptSHA256: Hash, + }) + .strict() + + export const Integrity = z + .object({ + protocolVersion: z.literal("benchmark-integrity-v1"), + validatorSHA256: Hash, + traceSchemaSHA256: Hash, + minEvents: z.number().int().min(1).max(10_000_000), + minCoverage: z.number().min(0.9).max(1), + assignedModel: z + .object({ + name: z.string().min(1).max(500), + baseArtifactSHA256: Hash, + configSHA256: Hash, + }) + .strict(), + forbiddenModelArtifacts: z + .array(Hash) + .max(128) + .refine((items) => new Set(items).size === items.length, "Forbidden model artifacts must be unique") + .default([]), + policy: z + .object({ + testItemDerivation: z.literal("forbidden"), + unapprovedExternalModels: z.literal("forbidden"), + benchmarkLookup: z.literal("forbidden"), + }) + .strict(), + auditors: z + .array(IntegrityAuditor) + .length(IntegrityAuditKind.options.length) + .refine( + (items) => new Set(items.map((item) => item.kind)).size === items.length, + "Integrity auditors must be unique", + ), + hiddenCanaryManifestSHA256: Hash, + minHiddenCanaries: z.number().int().min(1).max(10_000), + }) + .strict() + .superRefine((value, ctx) => { + const expected = IntegrityAuditKind.options.toSorted() + const submitted = value.auditors.map((item) => item.kind).toSorted() + if (JSON.stringify(expected) !== JSON.stringify(submitted)) { + ctx.addIssue({ code: "custom", path: ["auditors"], message: "Integrity protocol requires every audit class" }) + } + const identities = value.auditors.map((item) => `${item.name}\n${item.version}\n${item.promptSHA256}`) + if (new Set(identities).size === identities.length) return + ctx.addIssue({ code: "custom", path: ["auditors"], message: "Integrity auditor identities must be distinct" }) + }) + export type Integrity = z.infer + + const SourcePath = z + .string() + .min(1) + .max(1_000) + .refine( + (value) => + value === "." || + (!value.startsWith("/") && + !value.endsWith("/") && + !value.includes("\\") && + !value.split("/").some((part) => !part || part === "." || part === "..")), + "Evolution trace paths must be normalized relative POSIX paths", + ) + + export const Evolution = z + .object({ + protocolVersion: z.literal("evolution-trace-v1"), + validatorSHA256: Hash, + manifestSchemaSHA256: Hash, + lineAlgorithm: z.literal("sha256-exact-line-v1"), + roots: z + .array(SourcePath) + .min(1) + .max(32) + .refine((items) => new Set(items).size === items.length, "Evolution trace roots must be unique"), + extensions: z + .array(z.string().regex(/^\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}$/)) + .min(1) + .max(128) + .refine((items) => new Set(items).size === items.length, "Evolution trace extensions must be unique"), + exclude: z + .array(SourcePath) + .max(128) + .refine((items) => new Set(items).size === items.length, "Evolution trace exclusions must be unique") + .default([]), + maxFiles: z.number().int().min(1).max(100_000), + maxFileBytes: z.number().int().min(1).max(1_000_000_000), + maxTotalBytes: z.number().int().min(1).max(10_000_000_000), + maxSourceLines: z.number().int().min(1).max(10_000_000), + maxChangedLines: z.number().int().min(1).max(2_000_000), + }) + .strict() + .superRefine((value, ctx) => { + if (value.maxFileBytes <= value.maxTotalBytes) return + ctx.addIssue({ + code: "custom", + path: ["maxFileBytes"], + message: "Evolution trace file limit cannot exceed its total byte limit", + }) + }) + export type Evolution = z.infer + + export const MetaComponent = z.enum(["prompt", "memory", "skill", "tool", "middleware", "subagent", "scaffold"]) + export type MetaComponent = z.infer + + export const MetaIdentity = z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + promptSHA256: Hash, + configSHA256: Hash, + }) + .strict() + export type MetaIdentity = z.infer + + export const MetaTask = z + .object({ + id: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$/), + commitment: Hash, + activationRequired: z.boolean(), + }) + .strict() + export type MetaTask = z.infer + + const MetaTasks = z + .array(MetaTask) + .min(1) + .max(256) + .refine((items) => new Set(items.map((item) => item.id)).size === items.length, "Meta-harness tasks must be unique") + .refine( + (items) => new Set(items.map((item) => item.commitment)).size === items.length, + "Meta-harness task commitments must be unique", + ) + .refine( + (items) => items.every((item, index) => !index || items[index - 1]!.id < item.id), + "Meta-harness tasks must be sorted by ID", + ) + .refine( + (items) => items.some((item) => item.activationRequired), + "Each meta-harness split must include an activation-required task", + ) + + export const MetaModel = z + .object({ + id: z.string().min(1).max(240), + commitment: Hash, + }) + .strict() + export type MetaModel = z.infer + + const MetaModels = z + .array(MetaModel) + .min(1) + .max(64) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Meta-harness model IDs must be unique", + ) + .refine( + (items) => new Set(items.map((item) => item.commitment)).size === items.length, + "Meta-harness model commitments must be unique", + ) + .refine( + (items) => items.every((item, index) => !index || items[index - 1]!.id < item.id), + "Meta-harness models must be sorted by ID", + ) + + export const MetaHarness = z + .object({ + protocolVersion: z.literal("meta-harness-v1"), + validatorSHA256: Hash, + archiveSchemaSHA256: Hash, + traceSchemaSHA256: Hash, + baseline: z + .object({ + artifactSHA256: Hash, + manifestSHA256: Hash, + }) + .strict(), + mutable: z + .array( + z + .object({ + root: SourcePath, + component: MetaComponent, + }) + .strict(), + ) + .min(1) + .max(64) + .refine( + (items) => new Set(items.map((item) => item.root)).size === items.length, + "Mutable roots must be unique", + ) + .refine( + (items) => items.every((item, index) => !index || items[index - 1]!.root < item.root), + "Mutable roots must be sorted", + ), + protected: z + .object({ + manifestSHA256: Hash, + roots: z + .array(SourcePath) + .min(1) + .max(128) + .refine((items) => new Set(items).size === items.length, "Protected roots must be unique") + .refine( + (items) => items.every((item, index) => !index || items[index - 1]! < item), + "Protected roots must be sorted", + ), + }) + .strict(), + archive: z + .object({ + contents: z.literal("full-source-scores-traces"), + query: z.literal("filesystem"), + summariesOnly: z.literal(false), + hiddenContent: z.literal("excluded"), + evaluatorContent: z.literal("excluded"), + }) + .strict(), + updater: MetaIdentity, + judge: MetaIdentity, + search: z + .object({ + models: MetaModels, + tasks: MetaTasks, + }) + .strict(), + heldout: z + .object({ + models: MetaModels, + tasks: MetaTasks, + }) + .strict(), + thresholds: z + .object({ + minSearchGain: z.number().finite().nonnegative(), + minHeldoutGain: z.number().finite().nonnegative(), + maxModelRegression: z.number().finite().nonnegative(), + minActivationRate: z.number().finite().min(0).max(1), + minRequiredAdherence: z.number().finite().min(0).max(1), + minFinalAdherence: z.number().finite().min(0).max(1), + maxPhaseDrift: z.number().finite().min(0).max(1), + minPredictionPrecision: z.number().finite().min(0).max(1), + maxRiskRegressions: z.number().int().nonnegative().max(10_000), + maxContextTokens: z.number().int().positive(), + maxMeanContextIncrease: z.number().finite().nonnegative(), + }) + .strict(), + promotionRequired: z.literal(true), + }) + .strict() + .superRefine((value, ctx) => { + const covers = (root: string, target: string) => root === "." || target === root || target.startsWith(`${root}/`) + const mutable = value.mutable.map((item) => item.root) + for (const [index, root] of mutable.entries()) { + if (mutable.some((other, otherIndex) => index !== otherIndex && covers(other, root))) { + ctx.addIssue({ code: "custom", path: ["mutable", index, "root"], message: "Mutable roots cannot overlap" }) + } + if (value.protected.roots.some((other) => covers(root, other) || covers(other, root))) { + ctx.addIssue({ + code: "custom", + path: ["mutable", index, "root"], + message: "Mutable and protected roots cannot overlap", + }) + } + } + for (const [index, root] of value.protected.roots.entries()) { + if (value.protected.roots.some((other, otherIndex) => index !== otherIndex && covers(other, root))) { + ctx.addIssue({ + code: "custom", + path: ["protected", "roots", index], + message: "Protected roots cannot overlap", + }) + } + } + const searchModels = new Set(value.search.models.map((model) => model.id)) + const searchModelCommitments = new Set(value.search.models.map((model) => model.commitment)) + if ( + value.heldout.models.some((model) => searchModels.has(model.id) || searchModelCommitments.has(model.commitment)) + ) { + ctx.addIssue({ + code: "custom", + path: ["heldout", "models"], + message: "Held-out models must be unseen in search", + }) + } + const searchTasks = new Set(value.search.tasks.map((item) => item.id)) + const searchCommitments = new Set(value.search.tasks.map((item) => item.commitment)) + if (value.heldout.tasks.some((task) => searchTasks.has(task.id) || searchCommitments.has(task.commitment))) { + ctx.addIssue({ code: "custom", path: ["heldout", "tasks"], message: "Held-out tasks must be unseen in search" }) + } + if ( + value.updater.name === value.judge.name && + value.updater.version === value.judge.version && + value.updater.promptSHA256 === value.judge.promptSHA256 && + value.updater.configSHA256 === value.judge.configSHA256 + ) { + ctx.addIssue({ code: "custom", path: ["judge"], message: "Updater and adherence judge identities must differ" }) + } + }) + export type MetaHarness = z.infer + + export const InterventionFamily = z.enum([ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer", + ]) + export type InterventionFamily = z.infer + + const InterventionRule = z.discriminatedUnion("mode", [ + z + .object({ + family: z.literal("replay"), + mode: z.literal("max_absolute_effect"), + threshold: z.number().finite().nonnegative(), + }) + .strict(), + z + .object({ + family: z.enum(["retune", "ablation", "repair"]), + mode: z.literal("min_effect"), + threshold: z.number().finite().nonnegative(), + }) + .strict(), + z + .object({ + family: z.enum(["model_transfer", "context_transfer", "evaluator_transfer", "split_transfer"]), + mode: z.literal("max_regression"), + threshold: z.number().finite().nonnegative(), + }) + .strict(), + ]) + + export const Interventions = z + .object({ + protocolVersion: z.literal("intervention-study-v1"), + validatorSHA256: Hash, + requiredForPromotion: z.boolean(), + minPairs: z.number().int().min(3).max(32), + maxPairs: z.number().int().min(3).max(32), + maxTotalPairs: z.number().int().min(3).max(256), + confidence: z.literal(0.95), + required: z + .array(InterventionFamily) + .min(1) + .max(InterventionFamily.options.length) + .refine((items) => new Set(items).size === items.length, "Intervention families must be unique") + .refine( + (items) => JSON.stringify(items) === JSON.stringify(items.toSorted()), + "Intervention families must be sorted", + ), + rules: z + .array(InterventionRule) + .min(1) + .max(InterventionFamily.options.length) + .refine( + (items) => new Set(items.map((item) => item.family)).size === items.length, + "Intervention rules must be unique", + ) + .refine( + (items) => + JSON.stringify(items.map((item) => item.family)) === + JSON.stringify(items.map((item) => item.family).toSorted()), + "Intervention rules must be family-sorted", + ), + }) + .strict() + .superRefine((value, ctx) => { + if (value.minPairs > value.maxPairs) { + ctx.addIssue({ + code: "custom", + path: ["maxPairs"], + message: "Intervention maximum pairs cannot be smaller than its minimum pairs", + }) + } + if (value.maxPairs * value.required.length > value.maxTotalPairs) { + ctx.addIssue({ + code: "custom", + path: ["maxTotalPairs"], + message: "Intervention total pair limit cannot fit every required family", + }) + } + if (JSON.stringify(value.required) === JSON.stringify(value.rules.map((item) => item.family))) return + ctx.addIssue({ + code: "custom", + path: ["rules"], + message: "Intervention rules must cover exactly the required families", + }) + }) + export type Interventions = z.infer + + export const SimulationStress = z.enum([ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability", + ]) + export type SimulationStress = z.infer + + export const SimulationEngine = z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + commandSHA256: Hash, + configSHA256: Hash, + }) + .strict() + + export const SimulationReference = z + .object({ + kind: z.enum(["analytic", "manufactured", "benchmark", "independent_solver", "limiting_case"]), + identity: z.string().min(1).max(500), + sha256: Hash, + }) + .strict() + + export const Simulation = z + .object({ + kind: z.enum(["ode", "pde", "cfd", "materials", "molecular", "agentic"]), + engine: SimulationEngine, + problemSHA256: Hash, + reference: SimulationReference, + validation: z + .object({ + errorNorm: z.string().min(1).max(200), + minLevels: z.number().int().min(3).max(12), + maxLevels: z.number().int().min(3).max(24).default(12), + expectedOrder: z.number().finite().positive().max(20), + orderTolerance: z.number().finite().nonnegative().max(10), + maxResidual: z.number().finite().nonnegative(), + invariantTolerances: z + .record(z.string().min(1).max(100), z.number().finite().nonnegative()) + .refine( + (value) => Object.keys(value).length >= 1 && Object.keys(value).length <= 32, + "A simulator protocol needs 1 to 32 invariant tolerances", + ), + requiredStressTests: z + .array(SimulationStress) + .min(1) + .max(SimulationStress.options.length) + .refine((items) => new Set(items).size === items.length, "Simulation stress tests must be unique"), + }) + .strict() + .superRefine((value, ctx) => { + if (value.minLevels <= value.maxLevels) return + ctx.addIssue({ + code: "custom", + path: ["maxLevels"], + message: "Simulation maximum levels cannot be smaller than its minimum levels", + }) + }), + }) + .strict() + export type Simulation = z.infer + + export const EvaluatorFault = z.enum([ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness", + ]) + export type EvaluatorFault = z.infer + + export const EvaluatorAudit = z + .object({ + protocolVersion: z.literal("evaluator-audit-v1"), + auditor: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "human", "external"]), + }) + .strict(), + suite: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + commitmentSHA256: Hash, + }) + .strict(), + minCleanCases: z.number().int().min(2).max(512), + minCasesPerFault: z.number().int().min(1).max(128), + requiredFaults: z + .array(EvaluatorFault) + .min(1) + .max(EvaluatorFault.options.length) + .refine((items) => new Set(items).size === items.length, "Evaluator audit fault classes must be unique"), + minSensitivity: z.number().min(0.5).max(1), + minSpecificity: z.number().min(0.5).max(1), + minBalancedAccuracy: z.number().min(0.5).max(1), + minFaultRecall: z.number().min(0.5).max(1), + maxBrierScore: z.number().min(0).max(0.5), + }) + .strict() + export type EvaluatorAudit = z.infer + + export const Novelty = z.enum(["not_required", "known", "rediscovery", "minor", "publication", "major"]) + export type Novelty = z.infer + + export const SemanticAudit = z + .object({ + protocolVersion: z.literal("semantic-audit-v1"), + reviewer: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["gate", "human", "external"]), + }) + .strict(), + scope: z + .object({ + objectiveSHA256: Hash, + criteria: z + .array( + z + .object({ + id: z.string().min(1).max(100), + requirement: z.string().min(1).max(500), + }) + .strict(), + ) + .min(1) + .max(24) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Semantic criterion IDs must be unique", + ), + forbiddenShortcuts: z + .array( + z + .object({ + id: z.string().min(1).max(100), + description: z.string().min(1).max(500), + }) + .strict(), + ) + .min(1) + .max(24) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Forbidden semantic shortcut IDs must be unique", + ), + literature: z + .object({ + cutoff: z.iso.date(), + corpusSHA256: Hash, + }) + .strict(), + noveltyFloor: Novelty, + }) + .strict(), + minReviewers: z.number().int().min(2).max(5), + minConfidence: z.number().finite().min(0.5).max(1), + }) + .strict() + export type SemanticAudit = z.infer + + export const SynthesisTool = z.enum(["google_search", "paper_search", "web_browse"]) + export type SynthesisTool = z.infer + + const SynthesisIdentity = z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + promptSHA256: Hash, + configSHA256: Hash, + }) + .strict() + + export const ScientificSynthesis = z + .object({ + protocolVersion: z.literal("scientific-synthesis-v1"), + querySHA256: Hash, + referenceSHA256: Hash, + referenceFactsSHA256: Hash, + referenceFactCount: z.number().int().min(1).max(2_048), + cutoff: z.iso.date(), + tools: z + .array(SynthesisTool) + .min(1) + .max(SynthesisTool.options.length) + .refine((items) => new Set(items).size === items.length, "Scientific synthesis tools must be unique") + .refine( + (items) => + items.every( + (item, index) => + !index || SynthesisTool.options.indexOf(items[index - 1]!) < SynthesisTool.options.indexOf(item), + ), + "Scientific synthesis tools must use canonical order", + ), + traceSchemaSHA256: Hash, + filterPolicySHA256: Hash, + maxToolEvents: z.number().int().min(1).max(10_000), + decomposer: SynthesisIdentity, + judges: z + .object({ + precision: SynthesisIdentity, + recall: SynthesisIdentity, + }) + .strict(), + minGeneratedFacts: z.number().int().min(1).max(512), + minPrecision: z.number().min(0).max(1), + minRecall: z.number().min(0).max(1), + minF1: z.number().min(0).max(1), + cleanRoomRequired: z.literal(true), + judgeFailurePolicy: z.literal("inconclusive"), + }) + .strict() + .superRefine((value, ctx) => { + const identities = [value.decomposer, value.judges.precision, value.judges.recall] + const prompts = identities.map((item) => item.promptSHA256) + if (new Set(prompts).size !== prompts.length) { + ctx.addIssue({ + code: "custom", + path: ["judges"], + message: "Decomposition, precision, and recall require distinct prompt commitments", + }) + } + }) + export type ScientificSynthesis = z.infer + + export const AutonomyLevel = z.enum(["essentially_autonomous", "human_ai_collaboration", "primarily_human"]) + export type AutonomyLevel = z.infer + + export const HumanAIAutonomy = z + .object({ + protocolVersion: z.literal("human-ai-autonomy-v1"), + claimedLevel: AutonomyLevel, + recorder: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + artifactSHA256: Hash, + source: z.literal("evaluator_runtime"), + }) + .strict(), + traceSchemaSHA256: Hash, + classificationPolicySHA256: Hash, + maxEvents: z.number().int().min(2).max(10_000), + rawRetention: z.literal("required"), + disclosure: z.enum(["evaluator_retained", "public_essential_after_release"]), + completeTraceRequired: z.literal(true), + uncertaintyPolicy: z.literal("inconclusive"), + }) + .strict() + export type HumanAIAutonomy = z.infer + + export const FormalTier = z.enum(["kernel", "fresh_recheck", "external_crosscheck"]) + export type FormalTier = z.infer + + export const FormalRelation = z.enum(["exact_proof", "exact_refutation", "repaired_proof"]) + export type FormalRelation = z.infer + + export const FormalForbidden = z.enum(["sorry", "admit", "debug.skipKernelTC", "native_decide"]) + export type FormalForbidden = z.infer + + export const FormalVerifierRole = z.enum([ + "lean_kernel", + "source_auditor", + "axiom_auditor", + "fresh_rechecker", + "sandbox_comparator", + "external_checker", + ]) + export type FormalVerifierRole = z.infer + + const FormalVerifier = z + .object({ + role: FormalVerifierRole, + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + artifactSHA256: Hash, + }) + .strict() + + export const ProofBlueprint = z + .object({ + protocolVersion: z.literal("proof-blueprint-v1"), + graphSchemaSHA256: Hash, + compilerArtifactSHA256: Hash, + sketchValidatorArtifactSHA256: Hash, + reviewerArtifactSHA256: Hash, + reviewerPromptSHA256: Hash, + nodePolicy: z.literal("and-or-monotone-v1"), + failurePolicy: z.literal("preserve-and-refine"), + memoization: z.literal("goal-sha256"), + finalAuthority: z.literal("formal-proof-v1"), + directAttemptFirst: z.literal(true), + verifiedSketchRequired: z.literal(true), + completeFailureHistoryRequired: z.literal(true), + maxNodes: z.number().int().min(2).max(512), + maxDepth: z.number().int().min(1).max(32), + maxParallel: z.number().int().min(1).max(32), + maxAttemptsPerGoal: z.number().int().min(1).max(16), + maxRefinementsPerGoal: z.number().int().min(0).max(16), + leaseDurationMs: z.number().int().min(1_000).max(3_600_000), + }) + .strict() + .superRefine((value, ctx) => { + if (value.maxParallel > value.maxNodes) { + ctx.addIssue({ + code: "custom", + path: ["maxParallel"], + message: "Proof blueprint parallelism cannot exceed its graph node budget", + }) + } + const artifacts = [ + value.graphSchemaSHA256, + value.compilerArtifactSHA256, + value.sketchValidatorArtifactSHA256, + value.reviewerArtifactSHA256, + value.reviewerPromptSHA256, + ] + if (new Set(artifacts).size === artifacts.length) return + ctx.addIssue({ + code: "custom", + path: ["graphSchemaSHA256"], + message: "Proof blueprint schema, compiler, validator, reviewer, and rubric require distinct artifacts", + }) + }) + export type ProofBlueprint = z.infer + + export const FormalProof = z + .object({ + protocolVersion: z.literal("formal-proof-v1"), + language: z.literal("lean4"), + tier: FormalTier, + relation: FormalRelation, + challengeSHA256: Hash, + statementSHA256: Hash, + declaration: z.string().min(1).max(500), + module: z.string().min(1).max(500), + leanVersion: z.string().min(1).max(200), + leanToolchainSHA256: Hash, + lakeManifestSHA256: Hash, + dependencyTreeSHA256: Hash, + verifiers: z.array(FormalVerifier).min(2).max(FormalVerifierRole.options.length), + sandboxImageSHA256: Hash.optional(), + forbiddenConstructs: z.array(FormalForbidden).length(FormalForbidden.options.length), + allowedAxioms: z.array(z.string().min(1).max(300)).max(64), + maxFiles: z.number().int().min(6).max(10_000), + completeManifestRequired: z.literal(true), + warningPolicy: z.literal("fail"), + semanticPolicy: z.literal("formal_statement_only"), + blueprint: ProofBlueprint.optional(), + }) + .strict() + .superRefine((value, ctx) => { + const roles = value.verifiers.map((item) => item.role) + const required = + value.tier === "kernel" + ? ["lean_kernel", "source_auditor", "axiom_auditor"] + : value.tier === "fresh_recheck" + ? ["lean_kernel", "source_auditor", "axiom_auditor", "fresh_rechecker"] + : FormalVerifierRole.options + if (new Set(roles).size !== roles.length || JSON.stringify(roles) !== JSON.stringify(required)) { + ctx.addIssue({ + code: "custom", + path: ["verifiers"], + message: "Formal proof verifier roles must exactly match the frozen trust tier in canonical order", + }) + } + const artifacts = value.verifiers.map((item) => item.artifactSHA256) + if (new Set(artifacts).size !== artifacts.length) { + ctx.addIssue({ + code: "custom", + path: ["verifiers"], + message: "Formal proof trust roles require distinct verifier artifacts", + }) + } + if (Boolean(value.sandboxImageSHA256) !== (value.tier === "external_crosscheck")) { + ctx.addIssue({ + code: "custom", + path: ["sandboxImageSHA256"], + message: "Only the external cross-check tier requires a frozen sandbox image", + }) + } + if (JSON.stringify(value.forbiddenConstructs) !== JSON.stringify(FormalForbidden.options)) { + ctx.addIssue({ + code: "custom", + path: ["forbiddenConstructs"], + message: "Formal proof forbidden constructs must use the complete canonical protocol policy", + }) + } + if ( + new Set(value.allowedAxioms).size !== value.allowedAxioms.length || + value.allowedAxioms.some( + (item, index) => Boolean(index) && value.allowedAxioms[index - 1]!.localeCompare(item) >= 0, + ) + ) { + ctx.addIssue({ + code: "custom", + path: ["allowedAxioms"], + message: "Formal proof allowed axioms must be unique and use canonical order", + }) + } + if (value.allowedAxioms.includes("sorryAx")) { + ctx.addIssue({ + code: "custom", + path: ["allowedAxioms"], + message: "A formal proof protocol can never allow sorryAx", + }) + } + const kernel = value.verifiers.find((item) => item.role === "lean_kernel") + if (value.blueprint && value.blueprint.compilerArtifactSHA256 !== kernel?.artifactSHA256) { + ctx.addIssue({ + code: "custom", + path: ["blueprint", "compilerArtifactSHA256"], + message: "Proof blueprint sketches and attempts must use the formal protocol's frozen Lean kernel", + }) + } + }) + export type FormalProof = z.infer + + export const ReplicationEstimator = z.enum(["mean", "median", "iqm", "pass_rate"]) + export type ReplicationEstimator = z.infer + + const ReplicationAxis = z + .object({ + id: z.string().min(1).max(120), + commitmentSHA256: Hash, + }) + .strict() + + const ReplicationInterval = z.discriminatedUnion("method", [ + z + .object({ + method: z.literal("stratified-bootstrap-percentile-v1"), + confidence: z.literal(0.95), + resamples: z.number().int().min(1_000).max(50_000), + seed: z.number().int().min(0).max(0xffffffff), + }) + .strict(), + z + .object({ + method: z.literal("wilson-score-v1"), + confidence: z.literal(0.95), + }) + .strict(), + ]) + + export const Replication = z + .object({ + protocolVersion: z.literal("replicated-evaluation-v1"), + validatorSHA256: Hash, + environmentSHA256: Hash, + sampling: z + .object({ + design: z.literal("crossed-stratified-cluster-v1"), + stratumKind: z.string().min(1).max(120), + clusterKind: z.string().min(1).max(120), + strata: z + .array(ReplicationAxis) + .min(1) + .max(64) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Replication strata must be unique", + ) + .refine( + (items) => new Set(items.map((item) => item.commitmentSHA256)).size === items.length, + "Replication stratum commitments must be unique", + ) + .refine( + (items) => + JSON.stringify(items.map((item) => item.id)) === + JSON.stringify(items.map((item) => item.id).toSorted()), + "Replication strata must be sorted", + ), + clusters: z + .array(ReplicationAxis) + .min(3) + .max(32) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Replication clusters must be unique", + ) + .refine( + (items) => new Set(items.map((item) => item.commitmentSHA256)).size === items.length, + "Replication cluster commitments must be unique", + ) + .refine( + (items) => + JSON.stringify(items.map((item) => item.id)) === + JSON.stringify(items.map((item) => item.id).toSorted()), + "Replication clusters must be sorted", + ), + }) + .strict() + .refine( + (value) => value.strata.length * value.clusters.length <= 512, + "A replicated evaluation may contain at most 512 frozen units", + ), + estimator: ReplicationEstimator, + interval: ReplicationInterval, + decision: z + .object({ + rule: z.literal("conservative-bound-v1"), + direction: z.enum(["maximize", "minimize", "pass"]), + target: z.number().finite(), + maxIntervalWidth: z.number().finite().nonnegative().optional(), + }) + .strict(), + failurePolicy: z.literal("fail-closed"), + }) + .strict() + .superRefine((value, ctx) => { + const pass = value.estimator === "pass_rate" + if (pass !== (value.interval.method === "wilson-score-v1")) { + ctx.addIssue({ + code: "custom", + path: ["interval", "method"], + message: "Pass-rate replication requires Wilson intervals; numeric estimators require stratified bootstrap", + }) + } + if (pass !== (value.decision.direction === "pass")) { + ctx.addIssue({ + code: "custom", + path: ["decision", "direction"], + message: "Pass-rate replication requires pass direction; numeric estimators require maximize or minimize", + }) + } + if (!pass && value.sampling.clusters.length < 5) { + ctx.addIssue({ + code: "custom", + path: ["sampling", "clusters"], + message: "Numeric stratified bootstrap requires at least five independent clusters", + }) + } + if (pass && (value.decision.target < 0 || value.decision.target > 1)) { + ctx.addIssue({ + code: "custom", + path: ["decision", "target"], + message: "Pass-rate replication target must be between zero and one", + }) + } + if (pass && value.sampling.strata.length !== 1) { + ctx.addIssue({ + code: "custom", + path: ["sampling", "strata"], + message: "Wilson pass-rate intervals require one stratum of independent Bernoulli clusters", + }) + } + }) + export type Replication = z.infer + + export const Split = z.enum(["development", "validation", "held_out", "release"]) + export type Split = z.infer + + export const ConfirmationClaim = z + .object({ + taskID: z.string().min(1).max(500), + split: z.enum(["held_out", "release"]), + manifestSHA256: Hash, + validatorSHA256: Hash, + environmentSHA256: Hash, + evaluator: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "external"]), + }) + .strict(), + source: z + .object({ + repository: z.string().url(), + revision: z.string().regex(/^[a-f0-9]{40}$/), + }) + .strict() + .optional(), + metric: z.string().min(1).max(200), + direction: z.enum(["maximize", "minimize"]), + target: z.number().finite(), + }) + .strict() + + export const Confirmation = z + .object({ + protocolVersion: z.literal("sealed-confirmation-v1"), + optimization: z + .object({ + split: z.enum(["development", "validation"]), + manifestSHA256: Hash, + }) + .strict(), + claim: ConfirmationClaim, + selection: z + .object({ + rule: z.literal("terminal-verified-best-v1"), + subjects: z.literal(1), + }) + .strict(), + exposure: z + .object({ + policy: z.literal("terminal-receipt-only"), + searchFeedback: z.literal(false), + memoryCapture: z.literal(false), + }) + .strict(), + failurePolicy: z.literal("fail-closed"), + }) + .strict() + .refine( + (value) => value.optimization.manifestSHA256 !== value.claim.manifestSHA256, + "Optimization and claim manifests must be distinct", + ) + export type Confirmation = z.infer + + export const Fidelity = z + .object({ + id: z.string().min(1).max(100), + final: z.boolean(), + maxWallTimeMs: z.number().int().positive().optional(), + maxCostUSD: z.number().nonnegative().optional(), + }) + .strict() + export const FidelityPlan = z + .array(Fidelity) + .min(2) + .max(8) + .refine((items) => new Set(items.map((item) => item.id)).size === items.length, "Fidelity stages must be unique") + .refine((items) => items.filter((item) => item.final).length === 1, "A fidelity plan needs exactly one final stage") + .refine((items) => items.at(-1)?.final === true, "The final fidelity stage must be last") + + export const Info = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1), + sessionID: z.string().min(1), + objective: z.string().min(1), + benchmark: z + .object({ + name: z.string().min(1), + title: z.string().min(1).max(240).default("Scientific evaluation"), + family: Family.default("custom"), + task: z.string().min(1).max(4_000).default("Scientific evaluation task"), + version: z.string().min(1), + taskID: z.string().min(1), + split: Split, + evaluator: z.string().min(1), + evaluatorVersion: z.string().min(1).optional(), + evaluatorSource: z.enum(["benchmark", "gate", "human", "external"]).optional(), + fidelities: FidelityPlan.optional(), + metric: z.string().min(1).optional(), + direction: z.enum(["maximize", "minimize", "pass"]).optional(), + target: z.number().finite().optional(), + objectives: Objectives.optional(), + objectiveAudit: ObjectiveAudit.optional(), + }) + .strict(), + profile: Profile, + orchestration: Orchestration.optional(), + search: Search.optional(), + audit: Audit.optional(), + failureDiscovery: FailureDiscovery.optional(), + integrity: Integrity.optional(), + evolution: Evolution.optional(), + metaHarness: MetaHarness.optional(), + interventions: Interventions.optional(), + simulation: Simulation.optional(), + evaluatorAudit: EvaluatorAudit.optional(), + semanticAudit: SemanticAudit.optional(), + synthesis: ScientificSynthesis.optional(), + autonomy: HumanAIAutonomy.optional(), + formalProof: FormalProof.optional(), + replication: Replication.optional(), + confirmation: Confirmation.optional(), + packs: z + .array(HarnessPack.Id) + .max(HarnessPack.Id.options.length) + .refine((items) => new Set(items).size === items.length, "Harness packs must be unique") + .optional(), + model: z + .object({ + provider: z.string().min(1), + name: z.string().min(1), + effort: z.string().min(1).optional(), + }) + .strict(), + tools: z.array(z.string().min(1)).default([]), + skills: z + .array( + z + .object({ + name: z.string().min(1), + version: z.string().min(1).optional(), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }) + .strict(), + ) + .default([]), + budget: z + .object({ + wallTimeMs: z.number().int().positive().optional(), + steps: z.number().int().positive().optional(), + candidates: z.number().int().positive().optional(), + tokens: z.number().int().positive().optional(), + costUSD: z.number().nonnegative().optional(), + cpuHours: z.number().nonnegative().optional(), + gpuHours: z.number().nonnegative().optional(), + }) + .strict(), + seed: z.number().int(), + intervention: z.enum(["autonomous", "human_reprompted"]), + contamination: z + .object({ + policy: z.string().min(1), + hiddenTestsAccessible: z.literal(false), + publicDataCutoff: z.string().min(1).optional(), + }) + .strict(), + createdAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if ( + value.benchmark.objectives?.length && + (!value.benchmark.metric || !["maximize", "minimize"].includes(value.benchmark.direction ?? "")) + ) { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "objectives"], + message: "Secondary objectives require a numeric primary benchmark metric", + }) + } + if (value.benchmark.objectives?.some((item) => item.metric === value.benchmark.metric)) { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "objectives"], + message: "A secondary objective cannot duplicate the primary benchmark metric", + }) + } + if (value.benchmark.objectives?.length && value.profile !== "optimize") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "objectives"], + message: "Secondary objectives require the optimize profile", + }) + } + if (value.search && value.profile !== "optimize") { + ctx.addIssue({ + code: "custom", + path: ["search"], + message: "Adaptive candidate search requires the optimize profile", + }) + } + if ( + value.search && + (!value.benchmark.metric || !["maximize", "minimize"].includes(value.benchmark.direction ?? "")) + ) { + ctx.addIssue({ + code: "custom", + path: ["search"], + message: "Adaptive candidate search requires a numeric benchmark metric and direction", + }) + } + if (value.benchmark.objectiveAudit && !value.benchmark.objectives?.length) { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "objectiveAudit"], + message: "An objective audit requires declared secondary objectives", + }) + } + if ( + (value.failureDiscovery || + value.integrity || + value.evolution || + value.metaHarness || + value.interventions || + value.simulation || + value.evaluatorAudit || + value.semanticAudit || + value.synthesis || + value.autonomy || + value.formalProof || + value.replication || + value.confirmation) && + !value.benchmark.evaluatorVersion + ) { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorVersion"], + message: "Evaluator-controlled validation needs an evaluator version", + }) + } + if (value.failureDiscovery && !value.audit) { + ctx.addIssue({ + code: "custom", + path: ["failureDiscovery"], + message: "Topic-aware failure discovery requires a bound active audit", + }) + } + if ( + value.failureDiscovery && + value.audit && + value.failureDiscovery.failureThreshold !== value.audit.failureThreshold + ) { + ctx.addIssue({ + code: "custom", + path: ["failureDiscovery", "failureThreshold"], + message: "Failure discovery must use the active audit failure threshold", + }) + } + if (value.failureDiscovery && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Topic-aware failure discovery requires a capability-authenticated evaluator source", + }) + } + if ( + (value.failureDiscovery || + value.integrity || + value.evolution || + value.metaHarness || + value.interventions || + value.simulation || + value.evaluatorAudit || + value.semanticAudit || + value.synthesis || + value.autonomy || + value.formalProof || + value.replication || + value.confirmation) && + !value.benchmark.evaluatorSource + ) { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Evaluator-controlled validation needs an evaluator source", + }) + } + if (value.simulation && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Simulator validation requires a capability-authenticated evaluator source", + }) + } + if (value.integrity && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Runtime integrity validation requires a capability-authenticated evaluator source", + }) + } + if (value.evolution && value.profile !== "optimize") { + ctx.addIssue({ + code: "custom", + path: ["evolution"], + message: "Evolution trace validation requires the optimize profile", + }) + } + if (value.evolution && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Evolution trace validation requires a capability-authenticated evaluator source", + }) + } + if (value.metaHarness && value.profile !== "optimize") { + ctx.addIssue({ + code: "custom", + path: ["metaHarness"], + message: "Meta-harness qualification requires the optimize profile", + }) + } + if (value.metaHarness && (!value.search || !value.evolution || !value.confirmation)) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness"], + message: + "Meta-harness qualification requires adaptive search, exact evolution provenance, and sealed confirmation", + }) + } + if (value.metaHarness && value.evolution) { + const covers = (root: string, target: string) => + root === "." || target === root || target.startsWith(`${root}/`) + const roots = [...value.metaHarness.mutable.map((item) => item.root), ...value.metaHarness.protected.roots] + if (roots.some((root) => !value.evolution!.roots.some((source) => covers(source, root)))) { + ctx.addIssue({ + code: "custom", + path: ["evolution", "roots"], + message: "Evolution source capture must cover every mutable and protected meta-harness root", + }) + } + } + if ( + value.metaHarness && + (!value.benchmark.metric || !["maximize", "minimize"].includes(value.benchmark.direction ?? "")) + ) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness"], + message: "Meta-harness qualification requires a numeric benchmark metric and direction", + }) + } + if (value.metaHarness && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Meta-harness qualification requires capability-authenticated evaluation", + }) + } + if (value.metaHarness && !value.metaHarness.search.models.some((model) => model.id === value.model.name)) { + ctx.addIssue({ + code: "custom", + path: ["model", "name"], + message: "The bound beneficiary model must be present in the meta-harness search model set", + }) + } + if ( + value.metaHarness && + value.metaHarness.judge.name === value.benchmark.evaluator && + value.metaHarness.judge.version === value.benchmark.evaluatorVersion + ) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness", "judge"], + message: "Adherence judging requires an identity distinct from the optimization score evaluator", + }) + } + if ( + value.metaHarness && + value.confirmation && + value.metaHarness.judge.name === value.confirmation.claim.evaluator.name && + value.metaHarness.judge.version === value.confirmation.claim.evaluator.version + ) { + ctx.addIssue({ + code: "custom", + path: ["metaHarness", "judge"], + message: "Adherence judging requires an identity distinct from the sealed claim evaluator", + }) + } + if (value.interventions && value.profile !== "optimize") { + ctx.addIssue({ + code: "custom", + path: ["interventions"], + message: "Controlled replay interventions require the optimize profile", + }) + } + if (value.interventions && !value.evolution) { + ctx.addIssue({ + code: "custom", + path: ["interventions"], + message: "Controlled replay interventions require exact evolutionary provenance", + }) + } + if ( + value.interventions && + (!value.benchmark.metric || !["maximize", "minimize"].includes(value.benchmark.direction ?? "")) + ) { + ctx.addIssue({ + code: "custom", + path: ["interventions"], + message: "Controlled replay interventions require a numeric benchmark metric and direction", + }) + } + if (value.interventions && !["held_out", "release"].includes(value.benchmark.split)) { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "split"], + message: "Controlled replay interventions require a held-out or release benchmark split", + }) + } + if (value.interventions && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Controlled replay interventions require a capability-authenticated evaluator source", + }) + } + if ( + value.integrity?.auditors.some( + (auditor) => + auditor.name === value.benchmark.evaluator && auditor.version === value.benchmark.evaluatorVersion, + ) + ) { + ctx.addIssue({ + code: "custom", + path: ["integrity", "auditors"], + message: "Runtime integrity auditing requires identities distinct from the score evaluator", + }) + } + if ( + value.evaluatorAudit && + value.evaluatorAudit.auditor.name === value.benchmark.evaluator && + value.evaluatorAudit.auditor.version === value.benchmark.evaluatorVersion && + value.evaluatorAudit.auditor.source === value.benchmark.evaluatorSource + ) { + ctx.addIssue({ + code: "custom", + path: ["evaluatorAudit", "auditor"], + message: "Evaluator qualification requires an independent auditor identity", + }) + } + if ( + value.semanticAudit && + value.semanticAudit.scope.objectiveSHA256 !== + new Bun.CryptoHasher("sha256").update(value.objective).digest("hex") + ) { + ctx.addIssue({ + code: "custom", + path: ["semanticAudit", "scope", "objectiveSHA256"], + message: "Semantic audit objective commitment does not match the bound objective", + }) + } + if ( + value.semanticAudit && + value.semanticAudit.reviewer.name === value.benchmark.evaluator && + value.semanticAudit.reviewer.version === value.benchmark.evaluatorVersion + ) { + ctx.addIssue({ + code: "custom", + path: ["semanticAudit", "reviewer"], + message: "Semantic review requires an identity distinct from the score evaluator", + }) + } + if ( + value.semanticAudit && + value.evaluatorAudit && + value.semanticAudit.reviewer.name === value.evaluatorAudit.auditor.name && + value.semanticAudit.reviewer.version === value.evaluatorAudit.auditor.version + ) { + ctx.addIssue({ + code: "custom", + path: ["semanticAudit", "reviewer"], + message: "Semantic review and evaluator qualification require distinct identities", + }) + } + if (value.synthesis && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Clean-room synthesis requires a capability-authenticated evaluator source", + }) + } + if (value.synthesis && (value.benchmark.metric !== "factual_f1" || value.benchmark.direction !== "maximize")) { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "metric"], + message: "Scientific synthesis requires factual_f1 as a maximized primary metric", + }) + } + if (value.synthesis && !value.evaluatorAudit) { + ctx.addIssue({ + code: "custom", + path: ["evaluatorAudit"], + message: "Scientific synthesis requires independent evaluator qualification", + }) + } + if ( + value.synthesis && + value.evaluatorAudit && + ["wrong_answer", "unsupported_claim", "data_leakage"].some( + (fault) => !value.evaluatorAudit?.requiredFaults.includes(fault as EvaluatorFault), + ) + ) { + ctx.addIssue({ + code: "custom", + path: ["evaluatorAudit", "requiredFaults"], + message: "Scientific synthesis qualification must test wrong answers, unsupported claims, and data leakage", + }) + } + if (value.synthesis && value.contamination.publicDataCutoff !== value.synthesis.cutoff) { + ctx.addIssue({ + code: "custom", + path: ["contamination", "publicDataCutoff"], + message: "Scientific synthesis cutoff must match the frozen contamination policy", + }) + } + if (value.synthesis && value.synthesis.tools.some((tool) => !value.tools.includes(tool))) { + ctx.addIssue({ + code: "custom", + path: ["tools"], + message: "Scientific synthesis retrieval tools must be present in the run tool allowlist", + }) + } + if (value.autonomy && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Human-AI autonomy tracing requires a capability-authenticated evaluator source", + }) + } + if ( + value.autonomy?.claimedLevel !== undefined && + value.autonomy.claimedLevel !== "essentially_autonomous" && + value.intervention !== "human_reprompted" + ) { + ctx.addIssue({ + code: "custom", + path: ["intervention"], + message: "Collaborative or primarily-human claims require the human_reprompted intervention label", + }) + } + if (value.formalProof && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Formal proof validation requires a capability-authenticated evaluator source", + }) + } + if ( + value.replication && + (!value.benchmark.metric || value.benchmark.direction === undefined || value.benchmark.target === undefined) + ) { + ctx.addIssue({ + code: "custom", + path: ["replication"], + message: "Replicated evaluation requires a benchmark metric, direction, and target", + }) + } + if ( + value.replication && + (value.replication.decision.direction !== value.benchmark.direction || + value.replication.decision.target !== value.benchmark.target) + ) { + ctx.addIssue({ + code: "custom", + path: ["replication", "decision"], + message: "Replicated evaluation decision must match the bound benchmark direction and target", + }) + } + if (value.replication && value.benchmark.evaluatorSource === "human") { + ctx.addIssue({ + code: "custom", + path: ["benchmark", "evaluatorSource"], + message: "Replicated evaluation requires a capability-authenticated evaluator source", + }) + } + if (value.confirmation && value.profile !== "optimize") { + ctx.addIssue({ + code: "custom", + path: ["confirmation"], + message: "Sealed confirmation requires the optimize profile", + }) + } + if (value.confirmation && !value.search) { + ctx.addIssue({ + code: "custom", + path: ["confirmation"], + message: "Sealed confirmation requires backend-managed adaptive search", + }) + } + if (value.confirmation && value.confirmation.optimization.split !== value.benchmark.split) { + ctx.addIssue({ + code: "custom", + path: ["confirmation", "optimization", "split"], + message: "The confirmation optimization split must match the bound benchmark split", + }) + } + if ( + value.confirmation && + (value.confirmation.claim.metric !== value.benchmark.metric || + value.confirmation.claim.direction !== value.benchmark.direction || + value.confirmation.claim.target !== value.benchmark.target) + ) { + ctx.addIssue({ + code: "custom", + path: ["confirmation", "claim"], + message: "The sealed claim metric, direction, and target must match the optimization objective", + }) + } + if ( + value.confirmation && + value.confirmation.claim.evaluator.name === value.benchmark.evaluator && + value.confirmation.claim.evaluator.version === value.benchmark.evaluatorVersion && + value.confirmation.claim.evaluator.source === value.benchmark.evaluatorSource + ) { + ctx.addIssue({ + code: "custom", + path: ["confirmation", "claim", "evaluator"], + message: "Sealed confirmation requires an evaluator identity distinct from optimization", + }) + } + }) + export type Info = z.infer + + const root = path.join(Global.Path.data, "harness", "contracts") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + + export async function bind(input: Info) { + const contract = Info.parse(input) + await JsonStore.update(file(contract.sessionID), (data) => { + if (!Object.keys(data).length) return contract + const current = Info.parse(data) + if (fingerprint(current) === fingerprint(contract)) return current + throw new Error(`Harness contract for session ${contract.sessionID} is immutable once bound`) + }) + return contract + } + + export async function read(sessionID: string): Promise { + const data = await Bun.file(file(sessionID)) + .json() + .catch(() => null) + const parsed = Info.safeParse(data) + return parsed.success ? parsed.data : null + } + + export function fingerprint(input: Info) { + const contract = Info.parse(input) + return new Bun.CryptoHasher("sha256").update(JSON.stringify(contract)).digest("hex") + } +} diff --git a/backend/cli/src/session/harness/domain.ts b/backend/cli/src/session/harness/domain.ts new file mode 100644 index 00000000..54003728 --- /dev/null +++ b/backend/cli/src/session/harness/domain.ts @@ -0,0 +1,311 @@ +import z from "zod" +import { HarnessContract } from "./contract" +import { HarnessPack } from "./pack" + +export namespace HarnessDomain { + export const Check = z + .object({ + id: z.string().min(1).max(100), + severity: z.enum(["blocking", "advisory"]), + requirement: z.string().min(1).max(500), + }) + .strict() + export type Check = z.infer + + export const Info = z + .object({ + id: HarnessPack.Id, + title: z.string().min(1).max(100), + purpose: z.string().min(1).max(500), + checks: z + .array(Check) + .min(1) + .max(24) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Pack check IDs must be unique", + ), + }) + .strict() + export type Info = z.infer + + export type Selection = { ids: HarnessPack.Id[]; source: "contract" | "recommended" | "none" } + export type Actual = { + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + evidence: string[] + } + + const gate = (id: string, requirement: string): Check => ({ id, severity: "blocking", requirement }) + const advise = (id: string, requirement: string): Check => ({ id, severity: "advisory", requirement }) + const estimand = gate("estimand", "State the target quantity, population, comparison, and time horizon.") + const assumptions = gate("assumptions", "Check the assumptions that make the chosen method valid.") + const multiplicity = gate("multiplicity", "Control multiple testing or document why it is not applicable.") + const uncertainty = gate("uncertainty", "Report calibrated uncertainty, interval estimates, or justified bounds.") + const heldout = gate("held-out", "Evaluate only on the declared untouched split or withheld outputs.") + const baseline = gate("baseline", "Compare against the declared relevant baseline under the same setting.") + const budget = gate("budget", "Report and respect the declared model, data, tool, and compute budget.") + + export const catalog: Record = { + statistics: Info.parse({ + id: "statistics", + title: "Statistical methodology", + purpose: "Guard estimands, assumptions, effect interpretation, uncertainty, and multiplicity.", + checks: [ + estimand, + assumptions, + gate("effect-size", "Report effect size and practical scale, not significance alone."), + uncertainty, + multiplicity, + advise("sensitivity", "Probe sensitivity to defensible alternative specifications or assumptions."), + gate("stat-replay", "Preserve exact inputs, exclusions, transformations, test definition, and seed."), + ], + }), + biology: Info.parse({ + id: "biology", + title: "Computational biology", + purpose: "Guard biological identity, design, QC, batch structure, multiplicity, and interpretation.", + checks: [ + gate("bio-identifiers", "Validate organism, assembly, feature namespace, aliases, and identifier versions."), + gate("bio-design", "Recover groups, pairing, replicates, sampling unit, and experimental design."), + gate("bio-qc", "Run modality-appropriate sample and feature QC without outcome-informed deletion."), + gate("bio-batch", "Model or justify batch, donor, site, lane, and other technical structure."), + gate("bio-covariates", "Predeclare relevant covariates and avoid post-outcome adjustment choices."), + estimand, + multiplicity, + gate("bio-validity", "Separate statistical evidence from biological mechanism and validate key annotations."), + ], + }), + physics: Info.parse({ + id: "physics", + title: "Theoretical and physical reasoning", + purpose: "Guard assumptions, dimensions, conventions, limits, conservation, and independent derivation.", + checks: [ + assumptions, + gate("units", "Verify dimensional consistency of every material equation and reported quantity."), + gate("sign-convention", "Pin coordinate, metric, Fourier, phase, and sign conventions."), + gate("limiting-case", "Recover known limits, asymptotics, bounds, or special cases."), + gate("physics-conservation", "Check applicable conservation laws, symmetries, and physical bounds."), + gate( + "independent-derivation", + "Verify the headline result through an independent derivation or equivalent route.", + ), + ], + }), + pde: Info.parse({ + id: "pde", + title: "Numerical PDE and simulation", + purpose: "Guard problem specification, discretization, stability, convergence, conservation, and error claims.", + checks: [ + gate("pde-equation", "Pin the exact PDE, coefficients, source terms, and nondimensionalization."), + gate("pde-domain", "Pin geometry, coordinate system, mesh domain, and material regions."), + gate("pde-bc-ic", "Pin boundary and initial conditions and verify their compatibility."), + gate("pde-discretization", "Record scheme, order, mesh, timestep, solver, tolerances, and stopping rules."), + gate("pde-convergence", "Demonstrate mesh/time/order convergence in a declared error norm."), + gate("pde-stability", "Check CFL, conditioning, solver convergence, or the applicable stability criterion."), + gate( + "pde-conservation", + "Measure conservation, residual, positivity, maximum principle, or relevant invariants.", + ), + gate("pde-reference", "Compare with an analytic, manufactured, benchmark, or known-limit solution."), + gate("pde-error", "Report error norms and tolerances on the claimed quantity, not visuals alone."), + ], + }), + chemistry: Info.parse({ + id: "chemistry", + title: "Chemistry and materials", + purpose: "Guard chemical identity, representation, conditions, splits, physical validity, and uncertainty.", + checks: [ + gate("chem-identity", "Pin compound/material identity, composition, phase, protonation, charge, and version."), + gate( + "chem-standardization", + "Document canonicalization, salts, tautomers, duplicates, and structure normalization.", + ), + gate("chem-valence", "Validate valence, aromaticity, charge balance, sanitization, and impossible structures."), + gate("chem-stereo", "Preserve or explicitly marginalize stereochemistry and regiochemistry."), + gate("chem-units", "Pin units, assay/measurement conditions, temperature, pressure, solvent, and protocol."), + heldout, + gate("chem-split", "Audit scaffold, temporal, composition, and near-duplicate leakage across splits."), + gate("chem-physical", "Check physical bounds, conservation, symmetry, and domain applicability."), + uncertainty, + ], + }), + ml: Info.parse({ + id: "ml", + title: "Machine learning", + purpose: "Guard data identity, held-out evaluation, leakage, baselines, metrics, variance, and compute.", + checks: [ + gate("ml-data", "Pin dataset version, schema, target, exclusions, preprocessing, and sample unit."), + heldout, + gate( + "ml-leakage", + "Audit target, split, temporal, group, preprocessing, retrieval, and benchmark contamination.", + ), + baseline, + gate("ml-metric", "Pin metric implementation, direction, aggregation, decoding, and tie handling."), + gate("ml-seed-variance", "Report seeds and uncertainty or justify deterministic evaluation."), + budget, + gate("ml-model", "Pin model/checkpoint identity, templates, tokenizer, config, dependencies, and code state."), + advise("ml-ablation", "Ablate the claimed improvement against the strongest measured parent or baseline."), + ], + }), + forecast: Info.parse({ + id: "forecast", + title: "Weather and spatiotemporal forecasting", + purpose: "Guard forecast configuration, lead-dependent metrics, baselines, calibration, leakage, and compute.", + checks: [ + gate("forecast-data", "Pin dataset, observation/reanalysis source, variables, units, and revision."), + gate("forecast-init", "Pin initialization time, analysis cycle, ensemble members, and latency assumptions."), + gate("forecast-grid", "Pin region, mask, vertical levels, grid/resolution, regridding, and weighting."), + gate("forecast-leads", "Report the declared lead times without selecting only favorable horizons."), + heldout, + baseline, + gate("forecast-metrics", "Report the full lead-dependent deterministic or probabilistic metric portfolio."), + gate("forecast-mode", "Distinguish deterministic, ensemble, probabilistic, and nowcast settings."), + gate("forecast-calibration", "Check calibration or explicitly document deterministic non-applicability."), + gate( + "forecast-leakage", + "Audit future information, reanalysis revisions, temporal overlap, and normalization leakage.", + ), + budget, + uncertainty, + ], + }), + formal: Info.parse({ + id: "formal", + title: "Formal theorem verification", + purpose: + "Guard statement identity, proof relation, environment closure, kernel acceptance, and trust assumptions.", + checks: [ + gate("formal-challenge", "Bind the exact trusted challenge, canonical statement, declaration, and module."), + gate("formal-relation", "Distinguish an exact proof, exact refutation, and repaired-statement proof."), + gate("formal-environment", "Pin Lean, toolchain, package manifest, dependency tree, and checker artifacts."), + gate("formal-manifest", "Commit the complete challenge, statement, proof, environment, and support manifest."), + gate( + "formal-kernel", + "Build successfully with no warnings and replay the proof through the frozen Lean kernel.", + ), + gate( + "formal-source", + "Audit every manifest source with the frozen policy and reject all unchecked escape constructs.", + ), + gate( + "formal-axioms", + "Audit the transitive axiom closure, including axiom types, against the frozen allowlist.", + ), + gate("formal-tier", "Satisfy the contract's kernel, fresh-recheck, or independent external-checker tier."), + advise( + "formal-semantics", + "Review that the frozen formal statement and definitions express the intended informal mathematics.", + ), + ], + }), + } + + export function compose(ids: HarnessPack.Id[]) { + const checks = new Map() + for (const id of ids) { + for (const check of catalog[id].checks) { + const current = checks.get(check.id) + if (current && JSON.stringify(current) !== JSON.stringify(check)) { + throw new Error(`Domain pack check ${check.id} has conflicting definitions`) + } + checks.set(check.id, check) + } + } + return [...checks.values()] + } + + export function audit(ids: HarnessPack.Id[], actual: Actual[]) { + const expected = compose(ids) + const duplicates = actual.filter((item, index) => actual.findIndex((other) => other.id === item.id) !== index) + const byID = new Map(actual.map((item) => [item.id, item])) + const missing = expected.filter((check) => check.severity === "blocking" && !byID.has(check.id)) + const failed = expected.flatMap((check) => { + if (check.severity !== "blocking") return [] + const item = byID.get(check.id) + if (!item) return [] + if (item.status !== "passed") return [{ check, actual: item, reason: `status:${item.status}` }] + if (!item.blocking) return [{ check, actual: item, reason: "not-marked-blocking" }] + if (!item.evidence.length) return [{ check, actual: item, reason: "missing-evidence" }] + return [] + }) + const advisory = expected.filter((check) => check.severity === "advisory" && !byID.has(check.id)) + return { expected, missing, failed, advisory, duplicates } + } + + export function assert(ids: HarnessPack.Id[], actual: Actual[]) { + const result = audit(ids, actual) + if (!result.missing.length && !result.failed.length && !result.duplicates.length) return result + const issues = [ + ...result.missing.map((check) => `${check.id}:missing`), + ...result.failed.map((item) => `${item.check.id}:${item.reason}`), + ...result.duplicates.map((item) => `${item.id}:duplicate`), + ] + throw new Error(`Domain verification pack failed: ${issues.join(", ")}`) + } + + export function recommend(input: { agent?: string; profile: HarnessContract.Profile; text: string }) { + const text = input.text.toLowerCase() + const active = + /\b(analy[sz]e|calculate|compute|estimate|evaluate|fit|implement|model|run|simulate|train|benchmark)\b/.test( + text, + ) || /\btest\s+(whether|if)\b/.test(text) + if (!active && input.profile === "react") return [] + const ids: HarnessPack.Id[] = [] + const add = (...items: HarnessPack.Id[]) => { + for (const item of items) if (!ids.includes(item)) ids.push(item) + } + const stats = + /\b(chi[- ]?square|p[- ]?value|hypothesis test|anova|regression|confidence interval|effect size|multiple testing|statistics?|statistical)\b/.test( + text, + ) + if (stats) add("statistics") + const biology = + input.agent === "biology" && /\b(data|gene|genom|protein|cell|variant|omics|assay|cohort|sample)\b/.test(text) + if (biology && active) add("biology") + if (input.profile === "theory") add("physics") + if (input.profile === "numerical") add("physics", "pde") + const chemistry = /\b(chemi|molecul|compounds?|reactions?|materials?|crystal|polymer|cataly|smiles|inchi)/.test( + text, + ) + if (chemistry && active) add("chemistry") + if (input.profile === "training" || (input.profile === "optimize" && input.agent === "ml")) add("ml") + if (input.profile === "forecast") add("ml", "forecast") + if (/\b(lean\s*4?|formal(?:ize|ization| proof)?|theorem prover|proof assistant|kernel[- ]checked)\b/.test(text)) { + add("formal") + } + return ids + } + + export async function resolve(input: { + sessionID: string + agent?: string + profile: HarnessContract.Profile + text: string + }): Promise { + const contract = await HarnessContract.read(input.sessionID) + if (contract?.packs?.length) return { ids: contract.packs, source: "contract" } + const ids = recommend(input) + return { ids, source: ids.length ? "recommended" : "none" } + } + + export function prompt(selection: Selection) { + if (!selection.ids.length) return "" + const lines = [ + ``, + selection.source === "contract" + ? "These checks are part of the immutable evaluator contract; every blocking ID needs a passing evidence-backed check." + : "Use these checks for material claims when applicable; do not turn a simple task into process ceremony.", + ] + for (const id of selection.ids) { + lines.push(`## ${catalog[id].title} (${id})`) + for (const check of catalog[id].checks) { + lines.push(`- [${check.severity}] ${check.id}: ${check.requirement}`) + } + } + lines.push("") + return lines.join("\n") + } +} diff --git a/backend/cli/src/session/harness/evaluation.ts b/backend/cli/src/session/harness/evaluation.ts new file mode 100644 index 00000000..9ea3633b --- /dev/null +++ b/backend/cli/src/session/harness/evaluation.ts @@ -0,0 +1,519 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" +import { HarnessDomain } from "./domain" +import { HarnessJudge } from "./judge" +import { HarnessIntegrity } from "./integrity" +import { HarnessEvolution } from "./evolution" +import { HarnessIntervention } from "./intervention" +import { HarnessSimulation } from "./simulation" +import { HarnessSemantic } from "./semantic" +import { HarnessReplication } from "./replication" + +export namespace HarnessEvaluation { + export const Status = z.enum(["passed", "failed", "inconclusive"]) + export type Status = z.infer + + export const Check = z + .object({ + id: z.string().min(1).max(200), + status: Status, + blocking: z.boolean(), + score: z.number().finite().optional(), + evidence: z.array(z.string().min(1).max(1_000)).max(32).default([]), + note: z.string().max(4_000).optional(), + }) + .strict() + export type Check = z.infer + + export const Usage = z + .object({ + wallTimeMs: z.number().nonnegative().optional(), + costUSD: z.number().nonnegative().optional(), + }) + .strict() + .refine( + (value) => value.wallTimeMs !== undefined || value.costUSD !== undefined, + "Evaluation usage cannot be empty", + ) + export type Usage = z.infer + + export const Info = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1), + sessionID: z.string().min(1), + subject: z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1), + }) + .strict() + .optional(), + fidelity: z + .object({ + stage: z.string().min(1).max(100), + final: z.boolean(), + }) + .strict() + .optional(), + simulationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + integrityReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evolutionReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + interventionReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evaluatorAuditReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + semanticReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + replicationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + auditReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + failureDiscoveryReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + synthesisReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + autonomyReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + proofReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evaluator: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "human", "external"]), + }) + .strict(), + status: Status, + score: z.number().finite().optional(), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "An evaluation may contain at most 128 metrics") + .default({}), + checks: z.array(Check).max(128), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + usage: Usage.optional(), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive().optional(), + notes: z.string().max(8_000).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.status !== "passed") return + const failed = value.checks.find((check) => check.blocking && check.status !== "passed") + if (!failed) return + ctx.addIssue({ + code: "custom", + path: ["status"], + message: `A passed evaluation cannot contain a non-passing blocking check: ${failed.id}`, + }) + }) + export type Info = z.infer + + const State = z + .object({ + schemaVersion: z.literal(1), + items: z.record(z.string(), Info), + order: z.array(z.string()), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.order).size !== value.order.length) { + ctx.addIssue({ code: "custom", path: ["order"], message: "Evaluation journal order must be unique" }) + } + for (const key of value.order) { + if (value.items[key]) continue + ctx.addIssue({ code: "custom", path: ["order"], message: `Evaluation journal is missing ${key}` }) + } + }) + type State = z.infer + + const root = path.join(Global.Path.data, "harness", "evaluations") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const key = (input: Pick) => { + const subject = input.subject ? `${input.subject.type}:${input.subject.id}` : "run" + return input.fidelity ? `${subject}@${input.fidelity.stage}` : subject + } + const empty = (): State => ({ schemaVersion: 1, items: {}, order: [] }) + + function state(input: Record) { + const legacy = Info.safeParse(input) + if (!legacy.success) return State.parse(Object.keys(input).length ? input : empty()) + const id = key(legacy.data) + return State.parse({ schemaVersion: 1, items: { [id]: legacy.data }, order: [id] }) + } + + export function fingerprint(input: Info) { + return new Bun.CryptoHasher("sha256").update(JSON.stringify(Info.parse(input))).digest("hex") + } + + export function passed(input: Info) { + const evaluation = Info.parse(input) + return ( + evaluation.status === "passed" && evaluation.checks.every((check) => !check.blocking || check.status === "passed") + ) + } + + export const final = (input: Info) => Info.parse(input).fidelity?.final !== false + export const verified = (input: Info) => passed(input) && final(input) + + export async function record(input: Info) { + const submitted = Info.parse(input) + const evaluation = Info.parse({ ...submitted, recordedAt: Date.now() }) + const contract = await HarnessContract.read(evaluation.sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${evaluation.sessionID}`) + if (contract.runID !== evaluation.runID) { + throw new Error(`Evaluation run ${evaluation.runID} does not match contract run ${contract.runID}`) + } + if (contract.benchmark.evaluator !== evaluation.evaluator.name) { + throw new Error( + `Evaluation source ${evaluation.evaluator.name} does not match contract evaluator ${contract.benchmark.evaluator}`, + ) + } + if ( + contract.benchmark.evaluatorVersion !== undefined && + contract.benchmark.evaluatorVersion !== evaluation.evaluator.version + ) { + throw new Error( + `Evaluation version ${evaluation.evaluator.version} does not match contract evaluator version ${contract.benchmark.evaluatorVersion}`, + ) + } + if ( + contract.benchmark.evaluatorSource !== undefined && + contract.benchmark.evaluatorSource !== evaluation.evaluator.source + ) { + throw new Error( + `Evaluation source ${evaluation.evaluator.source} does not match contract source ${contract.benchmark.evaluatorSource}`, + ) + } + const plan = contract.benchmark.fidelities + if (!plan && evaluation.fidelity) throw new Error(`Evaluation fidelity is not declared by the bound contract`) + if (plan && !evaluation.fidelity) throw new Error(`Evaluation must name a fidelity stage`) + const stage = evaluation.fidelity ? plan?.find((item) => item.id === evaluation.fidelity?.stage) : undefined + if (evaluation.fidelity && !stage) throw new Error(`Evaluation fidelity stage is not in the bound contract`) + if (stage && stage.final !== evaluation.fidelity?.final) { + throw new Error(`Evaluation fidelity finality does not match the bound contract`) + } + if (evaluation.simulationReceiptID && !contract.simulation) { + throw new Error(`Evaluation references a simulation receipt without a bound simulator protocol`) + } + if (evaluation.integrityReceiptID && !contract.integrity) { + throw new Error(`Evaluation references an integrity receipt without a bound runtime integrity protocol`) + } + if (contract.integrity && evaluation.status === "passed" && final(evaluation) && !evaluation.integrityReceiptID) { + throw new Error(`A passing final evaluation must reference a runtime integrity receipt`) + } + if (evaluation.integrityReceiptID) { + await HarnessIntegrity.assert({ + contract, + receiptID: evaluation.integrityReceiptID, + subject: + evaluation.subject?.type === "candidate" + ? { type: "candidate", id: evaluation.subject.id } + : { type: "run", id: contract.runID }, + requirePassed: evaluation.status === "passed" && final(evaluation), + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + }) + } + if (evaluation.evolutionReceiptID && !contract.evolution) { + throw new Error(`Evaluation references an evolution receipt without a bound evolution trace protocol`) + } + if (evaluation.evolutionReceiptID && evaluation.subject?.type !== "candidate") { + throw new Error(`Only a candidate evaluation may reference an evolution receipt`) + } + if ( + contract.evolution && + evaluation.subject?.type === "candidate" && + evaluation.status === "passed" && + final(evaluation) && + !evaluation.evolutionReceiptID + ) { + throw new Error(`A passing final candidate evaluation must reference an evolution trace receipt`) + } + if (evaluation.evolutionReceiptID && evaluation.subject?.type === "candidate") { + await HarnessEvolution.assert({ + contract, + receiptID: evaluation.evolutionReceiptID, + candidateID: evaluation.subject.id, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + }) + } + if (evaluation.interventionReceiptID && !contract.interventions) { + throw new Error(`Evaluation references an intervention receipt without a bound intervention protocol`) + } + if (evaluation.interventionReceiptID && evaluation.subject?.type !== "candidate") { + throw new Error(`Only a candidate evaluation may reference an intervention receipt`) + } + if (evaluation.interventionReceiptID && !evaluation.evolutionReceiptID) { + throw new Error(`An intervention-bearing evaluation must reference its exact evolution receipt`) + } + if ( + contract.interventions?.requiredForPromotion && + evaluation.subject?.type === "candidate" && + evaluation.status === "passed" && + final(evaluation) && + !evaluation.interventionReceiptID + ) { + throw new Error(`A passing final candidate evaluation must reference a controlled intervention receipt`) + } + if (evaluation.interventionReceiptID && evaluation.subject?.type === "candidate") { + await HarnessIntervention.assert({ + contract, + receiptID: evaluation.interventionReceiptID, + candidateID: evaluation.subject.id, + evolutionReceiptID: evaluation.evolutionReceiptID!, + requirePassed: evaluation.status === "passed" && final(evaluation), + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + }) + } + if (contract.simulation && evaluation.status === "passed" && final(evaluation) && !evaluation.simulationReceiptID) { + throw new Error(`A passing final simulation evaluation must reference a simulator validation receipt`) + } + if (evaluation.simulationReceiptID) { + await HarnessSimulation.assert({ + contract, + receiptID: evaluation.simulationReceiptID, + candidateID: evaluation.subject?.type === "candidate" ? evaluation.subject.id : undefined, + requirePassed: evaluation.status === "passed" && final(evaluation), + evaluatedAt: evaluation.evaluatedAt, + }) + } + if (evaluation.evaluatorAuditReceiptID && !contract.evaluatorAudit) { + throw new Error(`Evaluation references an auditor receipt without a bound evaluator audit protocol`) + } + if ( + contract.evaluatorAudit && + evaluation.status === "passed" && + final(evaluation) && + !evaluation.evaluatorAuditReceiptID + ) { + throw new Error(`A passing final evaluation must reference a qualified evaluator audit receipt`) + } + if (evaluation.evaluatorAuditReceiptID) { + await HarnessJudge.assert({ + contract, + receiptID: evaluation.evaluatorAuditReceiptID, + recordedAt: evaluation.recordedAt!, + requirePassed: evaluation.status === "passed" && final(evaluation), + }) + } + if (evaluation.semanticReceiptID && !contract.semanticAudit) { + throw new Error(`Evaluation references a semantic receipt without a bound semantic audit protocol`) + } + if ( + contract.semanticAudit && + evaluation.status === "passed" && + final(evaluation) && + !evaluation.semanticReceiptID + ) { + throw new Error(`A passing final evaluation must reference a semantic audit receipt`) + } + if (evaluation.semanticReceiptID) { + await HarnessSemantic.assert({ + contract, + receiptID: evaluation.semanticReceiptID, + subject: + evaluation.subject?.type === "candidate" + ? { type: "candidate", id: evaluation.subject.id } + : { type: "run", id: contract.runID }, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + requirePassed: evaluation.status === "passed" && final(evaluation), + }) + } + if (evaluation.replicationReceiptID && !contract.replication) { + throw new Error(`Evaluation references a replication receipt without a bound replicated evaluation protocol`) + } + if ( + contract.replication && + evaluation.status === "passed" && + final(evaluation) && + !evaluation.replicationReceiptID + ) { + throw new Error(`A passing final evaluation must reference a replicated evaluation receipt`) + } + if (evaluation.replicationReceiptID) { + await HarnessReplication.assert({ + contract, + receiptID: evaluation.replicationReceiptID, + subject: + evaluation.subject?.type === "candidate" + ? { type: "candidate", id: evaluation.subject.id } + : { type: "run", id: contract.runID }, + score: evaluation.score, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + requirePassed: evaluation.status === "passed" && final(evaluation), + }) + } + if (evaluation.auditReceiptID && !contract.audit) { + throw new Error(`Evaluation references an active audit receipt without a bound active audit protocol`) + } + if ( + contract.audit?.promotionRequired && + evaluation.status === "passed" && + final(evaluation) && + !evaluation.auditReceiptID + ) { + throw new Error(`A passing final evaluation must reference a qualified active audit receipt`) + } + if (evaluation.auditReceiptID) { + const { HarnessAudit } = await import("./audit") + await HarnessAudit.assert({ + contract, + receiptID: evaluation.auditReceiptID, + subject: + evaluation.subject?.type === "candidate" + ? { type: "candidate", id: evaluation.subject.id } + : { type: "run", id: contract.runID }, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + requireQualified: evaluation.status === "passed" && final(evaluation), + }) + } + if (evaluation.failureDiscoveryReceiptID && !contract.failureDiscovery) { + throw new Error(`Evaluation references a failure discovery receipt without a bound protocol`) + } + if (evaluation.failureDiscoveryReceiptID) { + const { HarnessFailure } = await import("./failure") + await HarnessFailure.assert({ + contract, + receiptID: evaluation.failureDiscoveryReceiptID, + subject: evaluation.subject ?? { type: "run", id: evaluation.runID }, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + }) + } + if (evaluation.synthesisReceiptID && !contract.synthesis) { + throw new Error(`Evaluation references a synthesis receipt without a bound scientific synthesis protocol`) + } + if (contract.synthesis && evaluation.status === "passed" && final(evaluation) && !evaluation.synthesisReceiptID) { + throw new Error(`A passing final evaluation must reference a scientific synthesis receipt`) + } + if (evaluation.synthesisReceiptID) { + const { HarnessSynthesis } = await import("./synthesis") + await HarnessSynthesis.assert({ + contract, + receiptID: evaluation.synthesisReceiptID, + subject: evaluation.subject ?? { type: "run", id: evaluation.runID }, + score: evaluation.score, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + requirePassed: evaluation.status === "passed" && final(evaluation), + }) + } + if (evaluation.autonomyReceiptID && !contract.autonomy) { + throw new Error(`Evaluation references an autonomy receipt without a bound human-AI autonomy protocol`) + } + if (contract.autonomy && evaluation.status === "passed" && final(evaluation) && !evaluation.autonomyReceiptID) { + throw new Error(`A passing final evaluation must reference a human-AI autonomy receipt`) + } + if (evaluation.autonomyReceiptID) { + const { HarnessAutonomy } = await import("./autonomy") + await HarnessAutonomy.assert({ + contract, + receiptID: evaluation.autonomyReceiptID, + subject: evaluation.subject ?? { type: "run", id: evaluation.runID }, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + requirePassed: evaluation.status === "passed" && final(evaluation), + }) + } + if (evaluation.proofReceiptID && !contract.formalProof) { + throw new Error(`Evaluation references a proof receipt without a bound formal proof protocol`) + } + if (contract.formalProof && evaluation.status === "passed" && final(evaluation) && !evaluation.proofReceiptID) { + throw new Error(`A passing final evaluation must reference a formal proof receipt`) + } + if (evaluation.proofReceiptID) { + const { HarnessFormal } = await import("./formal") + await HarnessFormal.assert({ + contract, + receiptID: evaluation.proofReceiptID, + subject: evaluation.subject ?? { type: "run", id: evaluation.runID }, + evaluatedAt: evaluation.evaluatedAt, + recordedAt: evaluation.recordedAt!, + requirePassed: evaluation.status === "passed" && final(evaluation), + }) + } + if (evaluation.status === "passed" && final(evaluation)) { + HarnessDomain.assert(contract.packs ?? [], evaluation.checks) + } + await JsonStore.update(file(evaluation.sessionID), (data) => { + const current = state(data) + const id = key(evaluation) + const existing = current.items[id] + const prior = existing ? structuredClone(existing) : undefined + const retry = structuredClone(evaluation) + if (prior) delete prior.recordedAt + delete retry.recordedAt + if (prior && JSON.stringify(prior) === JSON.stringify(retry)) return current + if (existing) throw new Error(`Evaluation for ${id} is immutable once recorded`) + if (stage && plan) { + const index = plan.findIndex((item) => item.id === stage.id) + const prior = plan + .slice(0, index) + .map( + (item) => + current.items[key({ subject: evaluation.subject, fidelity: { stage: item.id, final: item.final } })], + ) + if (prior.some((item) => !item || !passed(item))) { + throw new Error(`Evaluation cannot advance before every prior fidelity stage passes`) + } + } + return State.parse({ + ...current, + items: { ...current.items, [id]: evaluation }, + order: [...current.order, id], + }) + }) + const stored = (await list(evaluation.sessionID)).find((item) => key(item) === key(evaluation)) + if (!stored) throw new Error(`Evaluation was not durable after recording`) + return stored + } + + export async function list(sessionID: string): Promise { + const data = await JsonStore.read(file(sessionID)) + const parsed = state(data) + return parsed.order.map((id) => parsed.items[id]!) + } + + export async function read(sessionID: string, subject?: Info["subject"]): Promise { + const items = await list(sessionID) + if (!subject) return items.at(-1) ?? null + return items.findLast((item) => item.subject?.type === subject.type && item.subject.id === subject.id) ?? null + } +} diff --git a/backend/cli/src/session/harness/evolution.ts b/backend/cli/src/session/harness/evolution.ts new file mode 100644 index 00000000..24d96427 --- /dev/null +++ b/backend/cli/src/session/harness/evolution.ts @@ -0,0 +1,660 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessEvolution { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const Relative = z + .string() + .min(1) + .max(1_000) + .refine( + (value) => + !value.startsWith("/") && + !value.endsWith("/") && + !value.includes("\\") && + !value.split("/").some((part) => !part || part === "." || part === ".."), + "Evolution manifest paths must be normalized relative POSIX paths", + ) + + const stable = (input: unknown): unknown => { + if (Array.isArray(input)) return input.map(stable) + if (!input || typeof input !== "object") return input + return Object.fromEntries( + Object.entries(input as Record) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => [key, stable(value)]), + ) + } + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(stable(input))).digest("hex") + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + export const Artifact = z + .object({ + uri: z.string().min(1).max(2_048), + sha256: Hash, + }) + .strict() + + export const Subject = z + .object({ + type: z.literal("candidate"), + id: Hash, + artifact: Artifact, + }) + .strict() + + export const File = z + .object({ + path: Relative, + sha256: Hash, + bytes: z.number().int().nonnegative().max(1_000_000_000), + lineHashes: z.array(Hash).max(1_000_000), + }) + .strict() + + export const Files = z + .array(File) + .min(1) + .max(100_000) + .superRefine((items, ctx) => { + const paths = items.map((item) => item.path) + if (new Set(paths).size !== paths.length) { + ctx.addIssue({ code: "custom", message: "Evolution manifest paths must be unique" }) + } + if (same(paths, paths.toSorted())) return + ctx.addIssue({ code: "custom", message: "Evolution manifest files must be path-sorted" }) + }) + + export const Snapshot = z + .object({ + artifact: Artifact, + schemaSHA256: Hash, + files: Files, + }) + .strict() + + export const Parent = z + .object({ + id: Hash, + artifact: Artifact, + receiptID: Hash, + snapshotSHA256: Hash, + delta: Artifact, + }) + .strict() + + export const Parents = z + .array(Parent) + .max(2) + .refine((items) => new Set(items.map((item) => item.id)).size === items.length, "Trace parents must be unique") + .refine( + (items) => + same( + items.map((item) => item.id), + items.map((item) => item.id).toSorted(), + ), + "Trace parents must be ID-sorted", + ) + + export const Validator = z + .object({ + name: z.literal("trace-evolutionary-candidate"), + version: z.literal(1), + scriptSHA256: Hash, + }) + .strict() + + export const Submit = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + protocol: HarnessContract.Evolution, + subject: Subject, + snapshot: Snapshot, + parents: Parents, + validator: Validator, + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + }) + .strict() + export type Submit = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + + export const ParentDelta = z + .object({ + id: Hash, + receiptID: Hash, + filesChanged: z.number().int().nonnegative(), + addedLines: z.number().int().nonnegative(), + deletedLines: z.number().int().nonnegative(), + }) + .strict() + + export const Diagnostics = z + .object({ + files: z.number().int().positive(), + bytes: z.number().int().nonnegative(), + sourceLines: z.number().int().nonnegative(), + depth: z.number().int().nonnegative(), + ancestors: z.number().int().nonnegative(), + addedLines: z.number().int().nonnegative(), + deletedLines: z.number().int().nonnegative(), + ancestralDeletedLines: z.number().int().nonnegative(), + reintroducedLines: z.number().int().nonnegative(), + reintroducedHashes: z.number().int().nonnegative(), + reintroducedFraction: z.number().min(0).max(1), + novelLines: z.number().int().nonnegative(), + sourceChanged: z.boolean(), + cycleDetected: z.boolean(), + parents: z.array(ParentDelta).max(2), + }) + .strict() + export type Diagnostics = z.infer + + export const Info = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + submissionID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + subject: Subject, + evaluator: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "external"]), + }) + .strict(), + protocol: HarnessContract.Evolution, + snapshot: Snapshot, + parents: Parents, + validator: Validator, + diagnostics: Diagnostics, + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + export type Info = z.infer + + type Delta = { + files: Array<{ + path: string + status: "added" | "deleted" | "modified" + beforeSHA256?: string + afterSHA256?: string + }> + added: string[] + deleted: string[] + } + + const lines = (files: z.infer) => { + const counts = new Map() + for (const file of files) { + for (const hash of file.lineHashes) counts.set(hash, (counts.get(hash) ?? 0) + 1) + } + return counts + } + + const maximum = (snapshots: z.infer[]) => { + const counts = new Map() + for (const snapshot of snapshots) { + for (const [hash, count] of lines(snapshot.files)) counts.set(hash, Math.max(counts.get(hash) ?? 0, count)) + } + return counts + } + + const expand = (left: Map, right: Map) => + [...left.entries()] + .toSorted(([a], [b]) => a.localeCompare(b)) + .flatMap(([hash, count]) => Array.from({ length: Math.max(0, count - (right.get(hash) ?? 0)) }, () => hash)) + + const change = (base: z.infer, target: z.infer): Delta => { + const before = new Map(base.files.map((file) => [file.path, file])) + const after = new Map(target.files.map((file) => [file.path, file])) + const files = [...new Set([...before.keys(), ...after.keys()])] + .toSorted() + .reduce((result, file) => { + const prior = before.get(file) + const next = after.get(file) + if (prior?.sha256 === next?.sha256) return result + if (!prior) return [...result, { path: file, status: "added", afterSHA256: next!.sha256 }] + if (!next) return [...result, { path: file, status: "deleted", beforeSHA256: prior.sha256 }] + return [ + ...result, + { + path: file, + status: "modified", + beforeSHA256: prior.sha256, + afterSHA256: next.sha256, + }, + ] + }, []) + const prior = lines(base.files) + const next = lines(target.files) + return { files, added: expand(next, prior), deleted: expand(prior, next) } + } + + const transition = (snapshot: z.infer, parents: Info[]): Pick => { + if (!parents.length) return { added: [], deleted: [] } + const current = lines(snapshot.files) + const base = maximum(parents.map((parent) => parent.snapshot)) + return { added: expand(current, base), deleted: expand(base, current) } + } + + const intersect = (left: string[], right: string[]) => { + const available = new Map() + for (const hash of right) available.set(hash, (available.get(hash) ?? 0) + 1) + return left.filter((hash) => { + const count = available.get(hash) ?? 0 + if (!count) return false + available.set(hash, count - 1) + return true + }) + } + + export function manifestSHA256(protocol: HarnessContract.Evolution, files: z.infer) { + const value = HarnessContract.Evolution.parse(protocol) + return digest({ schemaVersion: 1, lineAlgorithm: value.lineAlgorithm, files: Files.parse(files) }) + } + + export function deltaSHA256(input: { + subject: z.infer + snapshot: z.infer + parent: Pick + }) { + const delta = change(input.parent.snapshot, input.snapshot) + return digest({ + schemaVersion: 1, + parent: { + id: input.parent.subject.id, + artifactSHA256: input.parent.subject.artifact.sha256, + snapshotSHA256: input.parent.snapshot.artifact.sha256, + }, + candidate: { + id: input.subject.id, + artifactSHA256: input.subject.artifact.sha256, + snapshotSHA256: input.snapshot.artifact.sha256, + }, + files: delta.files, + addedLineHashes: delta.added, + deletedLineHashes: delta.deleted, + }) + } + + function validate(snapshot: z.infer, protocol: HarnessContract.Evolution) { + if (snapshot.schemaSHA256 !== protocol.manifestSchemaSHA256) { + throw new Error(`Evolution snapshot schema does not match the immutable harness contract`) + } + if (snapshot.artifact.sha256 !== manifestSHA256(protocol, snapshot.files)) { + throw new Error(`Evolution snapshot manifest content hash is invalid`) + } + if (snapshot.files.length > protocol.maxFiles) throw new Error(`Evolution snapshot exceeds its file limit`) + if (snapshot.files.some((file) => file.bytes > protocol.maxFileBytes)) { + throw new Error(`Evolution snapshot exceeds its per-file byte limit`) + } + const bytes = snapshot.files.reduce((sum, file) => sum + file.bytes, 0) + if (bytes > protocol.maxTotalBytes) throw new Error(`Evolution snapshot exceeds its total byte limit`) + const count = snapshot.files.reduce((sum, file) => sum + file.lineHashes.length, 0) + if (count > protocol.maxSourceLines) throw new Error(`Evolution snapshot exceeds its source-line limit`) + for (const file of snapshot.files) { + const rooted = protocol.roots.some( + (root) => root === "." || file.path === root || file.path.startsWith(`${root}/`), + ) + if (!rooted) throw new Error(`Evolution snapshot file ${file.path} is outside the committed roots`) + if (!protocol.extensions.some((extension) => file.path.endsWith(extension))) { + throw new Error(`Evolution snapshot file ${file.path} has an uncommitted extension`) + } + const excluded = protocol.exclude.some((item) => file.path === item || file.path.startsWith(`${item}/`)) + if (excluded) throw new Error(`Evolution snapshot includes excluded file ${file.path}`) + } + } + + const request = (input: { + runID: string + sessionID: string + protocol: HarnessContract.Evolution + subject: z.infer + snapshot: z.infer + parents: z.infer[] + validator: z.infer + evidence: string[] + evaluatedAt: number + }) => digest(input) + + const lineage = (parents: Info[], items: Record) => { + const seen = new Map() + const visit = (receipt: Info) => { + if (seen.has(receipt.receiptID)) return + seen.set(receipt.receiptID, receipt) + for (const parent of receipt.parents) { + const prior = items[parent.receiptID] + if (prior) visit(prior) + } + } + for (const parent of parents) visit(parent) + return [...seen.values()].toSorted((left, right) => left.receiptID.localeCompare(right.receiptID)) + } + + function derive(snapshot: z.infer, parents: Info[], items: Record) { + const current = transition(snapshot, parents) + const ancestors = lineage(parents, items) + const deleted = ancestors.flatMap((receipt) => { + const prior = receipt.parents.map((parent) => items[parent.receiptID]!).filter(Boolean) + return transition(receipt.snapshot, prior).deleted + }) + const reintroduced = intersect(current.added, deleted) + const bytes = snapshot.files.reduce((sum, file) => sum + file.bytes, 0) + const sourceLines = snapshot.files.reduce((sum, file) => sum + file.lineHashes.length, 0) + const deltas = parents + .map((parent) => { + const delta = change(parent.snapshot, snapshot) + return { + id: parent.subject.id, + receiptID: parent.receiptID, + filesChanged: delta.files.length, + addedLines: delta.added.length, + deletedLines: delta.deleted.length, + } + }) + .toSorted((left, right) => left.id.localeCompare(right.id)) + return Diagnostics.parse({ + files: snapshot.files.length, + bytes, + sourceLines, + depth: parents.length ? Math.max(...parents.map((parent) => parent.diagnostics.depth)) + 1 : 0, + ancestors: ancestors.length, + addedLines: current.added.length, + deletedLines: current.deleted.length, + ancestralDeletedLines: deleted.length, + reintroducedLines: reintroduced.length, + reintroducedHashes: new Set(reintroduced).size, + reintroducedFraction: current.added.length ? reintroduced.length / current.added.length : 0, + novelLines: current.added.length - reintroduced.length, + sourceChanged: deltas.some((parent) => parent.filesChanged > 0), + cycleDetected: reintroduced.length > 0, + parents: deltas, + }) + } + + const State = z + .object({ + schemaVersion: z.literal(1), + items: z.record(Hash, Info), + order: z.array(Hash), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.order).size !== value.order.length) { + ctx.addIssue({ code: "custom", path: ["order"], message: "Evolution receipt order must be unique" }) + } + const subjects = new Set() + const seen: Record = {} + for (const id of value.order) { + const receipt = value.items[id] + if (!receipt) { + ctx.addIssue({ code: "custom", path: ["order"], message: `Evolution receipt ${id} is missing` }) + continue + } + if (receipt.receiptID !== id) { + ctx.addIssue({ code: "custom", path: ["items", id], message: "Evolution receipt key does not match its ID" }) + } + const payload = structuredClone(receipt) as Record + delete payload.receiptID + if (digest(payload) !== id) { + ctx.addIssue({ code: "custom", path: ["items", id], message: "Evolution receipt content hash is invalid" }) + } + if (subjects.has(receipt.subject.id)) { + ctx.addIssue({ code: "custom", path: ["items", id], message: "Candidate evolution receipt is not unique" }) + } + subjects.add(receipt.subject.id) + try { + validate(receipt.snapshot, receipt.protocol) + } catch (error) { + ctx.addIssue({ + code: "custom", + path: ["items", id], + message: error instanceof Error ? error.message : "Evolution snapshot is invalid", + }) + } + const parents = receipt.parents.map((parent) => seen[parent.receiptID]!).filter(Boolean) + if (parents.length !== receipt.parents.length) { + ctx.addIssue({ + code: "custom", + path: ["items", id, "parents"], + message: "Evolution parents must reference earlier receipts", + }) + } else { + for (const parent of receipt.parents) { + const prior = seen[parent.receiptID]! + if ( + prior.subject.id !== parent.id || + !same(prior.subject.artifact, parent.artifact) || + prior.snapshot.artifact.sha256 !== parent.snapshotSHA256 + ) { + ctx.addIssue({ + code: "custom", + path: ["items", id, "parents"], + message: "Evolution parent identity does not match its referenced receipt", + }) + } + if ( + parent.delta.sha256 !== + deltaSHA256({ subject: receipt.subject, snapshot: receipt.snapshot, parent: prior }) + ) { + ctx.addIssue({ + code: "custom", + path: ["items", id, "parents"], + message: "Evolution parent delta content hash is invalid", + }) + } + } + const diagnostics = derive(receipt.snapshot, parents, seen) + if (!same(receipt.diagnostics, diagnostics)) { + ctx.addIssue({ + code: "custom", + path: ["items", id, "diagnostics"], + message: "Evolution diagnostics derivation drifted", + }) + } + } + if ( + receipt.submissionID !== + request({ + runID: receipt.runID, + sessionID: receipt.sessionID, + protocol: receipt.protocol, + subject: receipt.subject, + snapshot: receipt.snapshot, + parents: receipt.parents, + validator: receipt.validator, + evidence: receipt.evidence, + evaluatedAt: receipt.evaluatedAt, + }) + ) { + ctx.addIssue({ + code: "custom", + path: ["items", id], + message: "Evolution submission content hash is invalid", + }) + } + seen[id] = receipt + } + for (const id of Object.keys(value.items)) { + if (value.order.includes(id)) continue + ctx.addIssue({ code: "custom", path: ["items", id], message: "Evolution receipt is absent from journal order" }) + } + }) + type State = z.infer + + const root = path.join(Global.Path.data, "harness", "evolution") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const empty = (): State => ({ schemaVersion: 1, items: {}, order: [] }) + const state = (input: Record) => State.parse(Object.keys(input).length ? input : empty()) + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const bound = HarnessContract.Info.parse(contract) + const protocol = bound.evolution + if (!protocol) throw new Error(`No evolution trace protocol is bound to session ${value.sessionID}`) + if (bound.sessionID !== value.sessionID || bound.runID !== value.runID) { + throw new Error(`Evolution receipt does not match the bound harness run`) + } + if (!same(value.protocol, protocol)) + throw new Error(`Evolution protocol does not match the immutable harness contract`) + if (value.validator.scriptSHA256 !== protocol.validatorSHA256) { + throw new Error(`Evolution validator does not match the immutable harness contract`) + } + if (value.evaluatedAt > Date.now() + 300_000) throw new Error(`Evolution receipt is implausibly future-dated`) + validate(value.snapshot, protocol) + const search = await import("./search").then((module) => module.HarnessSearch.read(value.sessionID)) + if (search.runID !== bound.runID) throw new Error(`Evolution receipt search belongs to a different harness run`) + const candidate = search.candidates[value.subject.id] + if (!candidate) throw new Error(`Evolution receipt candidate does not exist in the bound search`) + if (!same(candidate.artifact, value.subject.artifact)) { + throw new Error(`Evolution receipt artifact does not match the candidate artifact`) + } + if (value.evaluatedAt < candidate.createdAt) throw new Error(`Evolution validation predates the candidate`) + const expected = candidate.parentIDs.toSorted() + if ( + !same( + value.parents.map((parent) => parent.id), + expected, + ) + ) { + throw new Error(`Evolution receipt parents do not match the candidate lineage`) + } + const evidence = value.evidence.toSorted() + const output: { value?: Info } = {} + await JsonStore.update(file(value.sessionID), (data) => { + const current = state(data) + const parents = value.parents.map((parent) => { + const receipt = current.items[parent.receiptID] + if (!receipt) throw new Error(`Evolution parent receipt ${parent.receiptID} does not exist`) + if ( + receipt.subject.id !== parent.id || + !same(receipt.subject.artifact, parent.artifact) || + receipt.snapshot.artifact.sha256 !== parent.snapshotSHA256 + ) { + throw new Error(`Evolution parent does not match its immutable trace receipt`) + } + if (receipt.evaluatedAt > value.evaluatedAt) { + throw new Error(`Evolution candidate trace predates its parent trace`) + } + const expectedDelta = deltaSHA256({ subject: value.subject, snapshot: value.snapshot, parent: receipt }) + if (parent.delta.sha256 !== expectedDelta) { + throw new Error(`Evolution parent delta content hash is invalid`) + } + const delta = change(receipt.snapshot, value.snapshot) + if (delta.added.length + delta.deleted.length > protocol.maxChangedLines) { + throw new Error(`Evolution parent delta exceeds its changed-line limit`) + } + return receipt + }) + const submissionID = request({ + runID: value.runID, + sessionID: value.sessionID, + protocol, + subject: value.subject, + snapshot: value.snapshot, + parents: value.parents, + validator: value.validator, + evidence, + evaluatedAt: value.evaluatedAt, + }) + const existing = current.order + .map((id) => current.items[id]!) + .find((item) => item.subject.id === value.subject.id) + if (existing) { + if (existing.submissionID !== submissionID) { + throw new Error(`Evolution receipt for candidate ${value.subject.id} is immutable once recorded`) + } + output.value = existing + return current + } + const diagnostics = derive(value.snapshot, parents, current.items) + const payload = { + schemaVersion: 1 as const, + submissionID, + runID: value.runID, + sessionID: value.sessionID, + contractFingerprint: HarnessContract.fingerprint(bound), + subject: value.subject, + evaluator: { + name: bound.benchmark.evaluator, + version: bound.benchmark.evaluatorVersion!, + source: bound.benchmark.evaluatorSource!, + }, + protocol, + snapshot: value.snapshot, + parents: value.parents, + validator: value.validator, + diagnostics, + evidence, + evaluatedAt: value.evaluatedAt, + recordedAt: Date.now(), + } + const receipt = Info.parse({ ...payload, receiptID: digest(payload) }) + output.value = receipt + return State.parse({ + ...current, + items: { ...current.items, [receipt.receiptID]: receipt }, + order: [...current.order, receipt.receiptID], + }) + }) + if (!output.value) throw new Error(`Evolution receipt was not durable after recording`) + return output.value + } + + export async function read(sessionID: string, receiptID: string) { + const current = state(await JsonStore.read(file(sessionID))) + return current.items[Hash.parse(receiptID)] ?? null + } + + export async function list(sessionID: string) { + const current = state(await JsonStore.read(file(sessionID))) + return current.order.map((id) => current.items[id]!) + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + candidateID: string + evaluatedAt: number + recordedAt: number + }) { + const receipt = await read(input.contract.sessionID, input.receiptID) + if (!receipt) throw new Error(`Evolution receipt ${input.receiptID} does not exist`) + if (receipt.runID !== input.contract.runID) throw new Error(`Evolution receipt does not match the harness run`) + if (receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract)) { + throw new Error(`Evolution receipt does not match the immutable harness contract`) + } + if (!same(receipt.protocol, input.contract.evolution)) { + throw new Error(`Evolution receipt does not match the bound protocol`) + } + if (receipt.subject.id !== input.candidateID) { + throw new Error(`Evolution receipt does not match the evaluated candidate`) + } + if (receipt.evaluatedAt > input.evaluatedAt) { + throw new Error(`Benchmark evaluation predates its referenced evolution receipt`) + } + if (receipt.recordedAt > input.recordedAt) { + throw new Error(`Benchmark evaluation was recorded before its evolution receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/failure.ts b/backend/cli/src/session/harness/failure.ts new file mode 100644 index 00000000..f7f97007 --- /dev/null +++ b/backend/cli/src/session/harness/failure.ts @@ -0,0 +1,782 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessAdapter } from "./adapter" +import { HarnessAudit } from "./audit" +import { HarnessContract } from "./contract" + +export namespace HarnessFailure { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const compare = (left: string, right: string) => (left < right ? -1 : left > right ? 1 : 0) + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + export const Subject = HarnessAudit.Subject + export type Subject = HarnessAudit.Subject + + const Anchor = z + .object({ + id: z.string().min(1).max(240), + commitment: Hash, + loss: z.number().min(0).max(1), + }) + .strict() + type Anchor = z.infer + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + export type Access = z.infer + + export const Initialize = Access.extend({ + subject: Subject, + auditReceiptID: Hash, + }).strict() + export type Initialize = z.infer + + const Allocation = z + .object({ + phase: z.enum(["initialization", "ucb1"]), + pulls: z.number().int().nonnegative(), + rewards: z.number().int().nonnegative(), + score: z.number().finite(), + }) + .strict() + + export const Selection = z + .object({ + selectionID: Hash, + round: z.number().int().positive(), + topic: HarnessContract.FailureTopic, + anchors: z.array(Anchor).min(1).max(8), + allocation: Allocation, + selectedAt: z.number().int().positive(), + }) + .strict() + export type Selection = z.infer + + export const Validation = z + .object({ + kind: HarnessContract.FailureValidatorKind, + status: z.enum(["passed", "failed", "inconclusive"]), + score: z.number().finite().optional(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + note: z.string().max(4_000).optional(), + }) + .strict() + export type Validation = z.infer + + const Failed = z + .object({ + status: z.literal("failed"), + mode: z.enum(["generator_error", "timeout", "invalid_output", "other"]), + outputSHA256: Hash.optional(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + const Generated = z + .object({ + status: z.literal("generated"), + caseSHA256: Hash, + outputSHA256: Hash, + embedding: z.array(z.number().finite()).min(2).max(64), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + export const Generation = z.discriminatedUnion("status", [Failed, Generated]) + export type Generation = z.infer + + export const Outcome = z + .object({ + loss: z.number().min(0).max(1), + failure: z.boolean(), + outputSHA256: Hash, + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + export type Outcome = z.infer + + export const Observe = Access.extend({ + selectionID: Hash, + generation: Generation, + validations: z.array(Validation).max(HarnessContract.FailureValidatorKind.options.length), + outcome: Outcome.optional(), + evaluatedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.generation.status === "failed") { + if (value.validations.length) { + ctx.addIssue({ code: "custom", path: ["validations"], message: "A failed generation cannot be validated" }) + } + if (value.outcome) { + ctx.addIssue({ + code: "custom", + path: ["outcome"], + message: "A failed generation cannot have a target outcome", + }) + } + return + } + const kinds = value.validations.map((item) => item.kind) + if ( + kinds.length !== HarnessContract.FailureValidatorKind.options.length || + new Set(kinds).size !== kinds.length || + HarnessContract.FailureValidatorKind.options.some((kind) => !kinds.includes(kind)) + ) { + ctx.addIssue({ + code: "custom", + path: ["validations"], + message: "A generated case requires every frozen validator class exactly once", + }) + } + const passed = value.validations.every((item) => item.status === "passed") + if (passed && !value.outcome) { + ctx.addIssue({ code: "custom", path: ["outcome"], message: "A validated case requires a target outcome" }) + } + if (!passed && value.outcome) { + ctx.addIssue({ + code: "custom", + path: ["outcome"], + message: "An invalid or inconclusive generated case cannot enter target evaluation", + }) + } + }) + export type Observe = z.infer + + export const Attempt = z + .object({ + attemptID: Hash, + selection: Selection, + generation: Generation, + validations: z.array(Validation).max(HarnessContract.FailureValidatorKind.options.length), + outcome: Outcome.optional(), + admissible: z.boolean(), + reward: z.union([z.literal(0), z.literal(1)]), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + export type Attempt = z.infer + + const Arm = z + .object({ + pulls: z.number().int().nonnegative(), + rewards: z.number().int().nonnegative(), + rate: z.number().min(0).max(1), + }) + .strict() + + export const Statistics = z + .object({ + attempts: z.number().int().nonnegative(), + generated: z.number().int().nonnegative(), + admissible: z.number().int().nonnegative(), + failures: z.number().int().nonnegative(), + invalid: z.number().int().nonnegative(), + samplesToFirstFailure: z.number().int().positive().optional(), + failureRate: z.number().min(0).max(1), + topicEntropy: z.number().min(0).max(1), + embeddingLogDet: z.number().finite(), + topics: z.record(z.string(), Arm), + }) + .strict() + export type Statistics = z.infer + + export const Stop = z.enum(["budget_exhausted", "failure_target_reached"]) + export type Stop = z.infer + + export const State = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("topic-aware-failure-v1"), + streamID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + subject: Subject, + auditReceiptID: Hash, + sourcePoolSHA256: Hash, + anchors: z.array(Anchor).min(1).max(512), + config: HarnessContract.FailureDiscovery, + status: z.enum(["active", "completed"]), + stopReason: Stop.optional(), + pending: Selection.optional(), + attempts: z.array(Attempt).max(512), + statistics: Statistics, + revision: z.number().int().nonnegative(), + createdAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.anchors.map((item) => item.id)).size !== value.anchors.length) { + ctx.addIssue({ code: "custom", path: ["anchors"], message: "Failure anchors must be unique" }) + } + if (new Set(value.attempts.map((item) => item.attemptID)).size !== value.attempts.length) { + ctx.addIssue({ code: "custom", path: ["attempts"], message: "Failure attempts must be unique" }) + } + if (value.pending && value.status !== "active") { + ctx.addIssue({ code: "custom", path: ["pending"], message: "A completed failure stream cannot be pending" }) + } + if ((value.status === "completed") !== Boolean(value.stopReason)) { + ctx.addIssue({ code: "custom", path: ["stopReason"], message: "Failure stream terminal state is inconsistent" }) + } + }) + export type State = z.infer + + const ReceiptBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("topic-aware-failure-receipt-v1"), + receiptID: Hash, + streamID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + subject: Subject, + auditReceiptID: Hash, + sourcePoolSHA256: Hash, + config: HarnessContract.FailureDiscovery, + attemptIDs: z.array(Hash).min(2).max(512), + statistics: Statistics, + stopReason: Stop, + revision: z.number().int().positive(), + completedAt: z.number().int().positive(), + sealedAt: z.number().int().positive(), + }) + .strict() + + export const Receipt = ReceiptBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Failure discovery receipt content hash is invalid" }) + }) + export type Receipt = z.infer + + const root = path.join(Global.Path.data, "harness", "failures") + const receiptRoot = path.join(Global.Path.data, "harness", "failure-receipts") + const file = (sessionID: string, streamID: string) => + path.join(root, encodeURIComponent(sessionID), `${streamID}.json`) + const receiptFile = (receiptID: string) => path.join(receiptRoot, `${receiptID}.json`) + + function logdet(input: number[][], dimensions: number, regularization: number) { + if (!input.length) return 0 + const matrix = Array.from({ length: dimensions }, (_, row) => + Array.from({ length: dimensions }, (_, column) => (row === column ? 1 : 0)), + ) + for (const embedding of input) { + embedding.forEach((left, row) => + embedding.forEach((right, column) => { + matrix[row]![column]! += (left * right) / regularization + }), + ) + } + const lower = Array.from({ length: dimensions }, () => Array.from({ length: dimensions }, () => 0)) + Array.from({ length: dimensions }).forEach((_, row) => + Array.from({ length: row + 1 }).forEach((__, column) => { + const prior = Array.from({ length: column }, (_, index) => index).reduce( + (sum, index) => sum + lower[row]![index]! * lower[column]![index]!, + 0, + ) + const value = matrix[row]![column]! - prior + lower[row]![column] = + row === column ? Math.sqrt(Math.max(value, Number.EPSILON)) : value / lower[column]![column]! + }), + ) + const determinant = 2 * lower.reduce((sum, row, index) => sum + Math.log(row[index]!), 0) + return (input.length * Math.log(regularization) + determinant) / input.length + } + + function statistics(config: HarnessContract.FailureDiscovery, attempts: Attempt[]) { + const topics = Object.fromEntries( + config.topics.map((topic) => { + const selected = attempts.filter((attempt) => attempt.selection.topic.id === topic.id) + const rewards = selected.reduce((sum, attempt) => sum + attempt.reward, 0) + return [topic.id, { pulls: selected.length, rewards, rate: selected.length ? rewards / selected.length : 0 }] + }), + ) + const generated = attempts.filter((attempt) => attempt.generation.status === "generated") + const admissible = attempts.filter((attempt) => attempt.admissible) + const failures = admissible.filter((attempt) => attempt.reward === 1) + const topicCounts = config.topics.map( + (topic) => failures.filter((attempt) => attempt.selection.topic.id === topic.id).length, + ) + const entropy = failures.length + ? topicCounts.reduce((sum, count) => { + if (!count) return sum + const probability = count / failures.length + return sum - probability * Math.log(probability) + }, 0) / Math.log(config.topics.length) + : 0 + const embeddings = failures.flatMap((attempt) => + attempt.generation.status === "generated" ? [attempt.generation.embedding] : [], + ) + return Statistics.parse({ + attempts: attempts.length, + generated: generated.length, + admissible: admissible.length, + failures: failures.length, + invalid: attempts.length - admissible.length, + samplesToFirstFailure: failures[0]?.selection.round, + failureRate: admissible.length ? failures.length / admissible.length : 0, + topicEntropy: entropy, + embeddingLogDet: logdet(embeddings, config.embedding.dimensions, config.embedding.regularization), + topics, + }) + } + + function derive( + config: HarnessContract.FailureDiscovery, + prior: Attempt[], + generation: Generation, + validations: Validation[], + outcome: Outcome | undefined, + ): { valid: true; admissible: boolean; reward: 0 | 1 } | { valid: false; error: string } { + if (generation.status === "failed") { + if (validations.length) return { valid: false, error: "A failed generation cannot be validated" } + if (outcome) return { valid: false, error: "A failed generation cannot have a target outcome" } + return { valid: true, admissible: false, reward: 0 } + } + if ( + validations.length !== HarnessContract.FailureValidatorKind.options.length || + validations.some((item, index) => item.kind !== HarnessContract.FailureValidatorKind.options[index]) + ) { + return { valid: false, error: "A generated case requires every frozen validator class exactly once" } + } + if (generation.embedding.length !== config.embedding.dimensions) { + return { valid: false, error: "Failure discovery embedding does not match the frozen dimension" } + } + const norm = Math.sqrt(generation.embedding.reduce((sum, item) => sum + item * item, 0)) + if (Math.abs(norm - 1) > 1e-6) { + return { valid: false, error: "Failure discovery embedding must be L2-normalized" } + } + const duplicate = prior.some( + (attempt) => attempt.generation.status === "generated" && attempt.generation.caseSHA256 === generation.caseSHA256, + ) + const novelty = validations.find((item) => item.kind === "novelty")! + if (duplicate && novelty.status === "passed") { + return { valid: false, error: "An exact duplicate generated case cannot pass novelty validation" } + } + const passed = validations.every((item) => item.status === "passed") + if (passed && !outcome) return { valid: false, error: "A validated case requires a target outcome" } + if (!passed && outcome) { + return { valid: false, error: "An invalid or inconclusive generated case cannot enter target evaluation" } + } + if (outcome && outcome.failure !== outcome.loss >= config.failureThreshold) { + return { valid: false, error: "Failure discovery label does not match the frozen loss threshold" } + } + const admissible = passed && outcome !== undefined + return { valid: true, admissible, reward: admissible && outcome.failure ? 1 : 0 } + } + + function stopped(config: HarnessContract.FailureDiscovery, attempts: Attempt[]) { + const initialized = new Set(attempts.map((attempt) => attempt.selection.topic.id)).size === config.topics.length + const failures = attempts.reduce((sum, attempt) => sum + attempt.reward, 0) + if (initialized && config.targetFailures !== undefined && failures >= config.targetFailures) { + return "failure_target_reached" as const + } + if (attempts.length >= config.budget) return "budget_exhausted" as const + return undefined + } + + function allocation(state: Pick) { + const arms = statistics(state.config, state.attempts).topics + const unpulled = state.config.topics.find((topic) => !arms[topic.id]!.pulls) + if (unpulled) { + return { + topic: unpulled, + allocation: { + phase: "initialization" as const, + pulls: arms[unpulled.id]!.pulls, + rewards: arms[unpulled.id]!.rewards, + score: Number.MAX_VALUE, + }, + } + } + const total = Math.max(state.attempts.length, 1) + const ranked = state.config.topics + .map((topic) => { + const arm = arms[topic.id]! + return { + topic, + allocation: { + phase: "ucb1" as const, + pulls: arm.pulls, + rewards: arm.rewards, + score: arm.rate + state.config.exploration * Math.sqrt(Math.log(total) / arm.pulls), + }, + tie: digest({ streamID: state.streamID, round: state.attempts.length + 1, topic: topic.commitment }), + } + }) + .toSorted((left, right) => right.allocation.score - left.allocation.score || compare(left.tie, right.tie)) + return ranked[0]! + } + + function select(state: Pick, selectedAt: number) { + const choice = allocation(state) + const round = state.attempts.length + 1 + const anchors = state.anchors + .map((anchor) => ({ anchor, tie: digest({ streamID: state.streamID, round, anchor: anchor.commitment }) })) + .toSorted((left, right) => right.anchor.loss - left.anchor.loss || compare(left.tie, right.tie)) + .slice(0, state.config.anchorsPerAttempt) + .map((item) => item.anchor) + const stable = { round, topic: choice.topic, anchors, allocation: choice.allocation } + return Selection.parse({ + ...stable, + selectionID: digest({ streamID: state.streamID, ...stable }), + selectedAt, + }) + } + + function replay(state: State) { + const origin = { + runID: state.runID, + sessionID: state.sessionID, + contractFingerprint: state.contractFingerprint, + subject: state.subject, + auditReceiptID: state.auditReceiptID, + sourcePoolSHA256: state.sourcePoolSHA256, + config: state.config, + anchors: state.anchors, + } + if (digest(origin) !== state.streamID) return false + const prior: Attempt[] = [] + for (const attempt of state.attempts) { + if (stopped(state.config, prior)) return false + const expected = select({ ...state, attempts: prior }, attempt.selection.selectedAt) + if (!same(expected, attempt.selection)) return false + const result = derive(state.config, prior, attempt.generation, attempt.validations, attempt.outcome) + if (!result.valid || result.admissible !== attempt.admissible || result.reward !== attempt.reward) return false + const last = prior.at(-1) + if ( + attempt.selection.selectedAt < (last?.recordedAt ?? state.createdAt) || + attempt.evaluatedAt < attempt.selection.selectedAt || + attempt.evaluatedAt > attempt.recordedAt + 300_000 || + attempt.recordedAt < attempt.selection.selectedAt + ) { + return false + } + const stable = { + selection: attempt.selection, + generation: attempt.generation, + validations: attempt.validations, + outcome: attempt.outcome, + admissible: attempt.admissible, + reward: attempt.reward, + evaluatedAt: attempt.evaluatedAt, + recordedAt: attempt.recordedAt, + } + if (digest(stable) !== attempt.attemptID) return false + prior.push(attempt) + } + if (!same(statistics(state.config, state.attempts), state.statistics)) return false + if (state.pending && !same(select(state, state.pending.selectedAt), state.pending)) return false + if (state.revision !== state.attempts.length * 2 + (state.pending ? 1 : 0)) return false + const updatedAt = state.pending?.selectedAt ?? state.attempts.at(-1)?.recordedAt ?? state.createdAt + if (state.updatedAt !== updatedAt) return false + const stop = stopped(state.config, state.attempts) + return state.status === (stop ? "completed" : "active") && state.stopReason === stop + } + + function parse(input: Record) { + const state = State.parse(input) + if (!replay(state)) throw new Error(`Failure discovery state cannot be replayed from its immutable journal`) + return state + } + + async function read(sessionID: string, streamID: string) { + return parse(await JsonStore.read(file(sessionID, Hash.parse(streamID)))) + } + + function match(state: State, contract: HarnessContract.Info) { + if (state.sessionID !== contract.sessionID || state.runID !== contract.runID) { + throw new Error(`Failure discovery stream belongs to a different harness run`) + } + if ( + state.contractFingerprint !== HarnessContract.fingerprint(contract) || + !same(state.config, contract.failureDiscovery) + ) { + throw new Error(`Failure discovery stream does not match the bound harness contract`) + } + } + + export async function initialize(input: Initialize) { + const value = Initialize.parse(input) + const contract = await HarnessAdapter.authorize(value.sessionID, value.evaluatorToken) + const config = contract.failureDiscovery + if (!config) throw new Error(`No topic-aware failure discovery protocol is bound to session ${value.sessionID}`) + const now = Date.now() + const receipt = await HarnessAudit.assert({ + contract, + receiptID: value.auditReceiptID, + subject: value.subject, + evaluatedAt: now, + recordedAt: now, + requireQualified: false, + }) + if (receipt.poolFingerprint !== config.sourcePoolSHA256) { + throw new Error(`Failure discovery source pool does not match the frozen audit pool`) + } + if (!same(receipt.subject, value.subject)) { + throw new Error(`Failure discovery subject artifact does not match its source audit receipt`) + } + const audit = await HarnessAudit.status(receipt.auditID, { + sessionID: value.sessionID, + evaluatorToken: value.evaluatorToken, + }) + const anchors = audit.order.flatMap((id) => { + const entry = audit.pool[id] + return entry?.observation?.failure + ? [{ id: entry.id, commitment: entry.commitment, loss: entry.observation.loss }] + : [] + }) + if (anchors.length < config.anchorsPerAttempt) { + throw new Error(`Failure discovery needs at least ${config.anchorsPerAttempt} authenticated failure anchors`) + } + const origin = { + runID: contract.runID, + sessionID: contract.sessionID, + contractFingerprint: HarnessContract.fingerprint(contract), + subject: value.subject, + auditReceiptID: receipt.receiptID, + sourcePoolSHA256: receipt.poolFingerprint, + config, + anchors, + } + const state = State.parse({ + schemaVersion: 1, + protocolVersion: "topic-aware-failure-v1", + streamID: digest(origin), + ...origin, + status: "active", + attempts: [], + statistics: statistics(config, []), + revision: 0, + createdAt: now, + updatedAt: now, + }) + await JsonStore.update(file(state.sessionID, state.streamID), (data) => { + if (!Object.keys(data).length) return state + const current = parse(data) + if (current.streamID === state.streamID) return current + throw new Error(`Failure discovery stream is immutable once initialized`) + }) + return read(state.sessionID, state.streamID) + } + + export async function status(streamID: string, input: Access) { + const access = Access.parse(input) + const contract = await HarnessAdapter.authorize(access.sessionID, access.evaluatorToken) + const state = await read(access.sessionID, streamID) + match(state, contract) + return state + } + + export async function next(streamID: string, input: Access) { + const access = Access.parse(input) + const contract = await HarnessAdapter.authorize(access.sessionID, access.evaluatorToken) + const result = { selection: undefined as Selection | undefined } + await JsonStore.update(file(access.sessionID, Hash.parse(streamID)), (data) => { + const state = parse(data) + match(state, contract) + if (state.status !== "active") throw new Error(`Failure discovery stream is already complete`) + if (state.pending) { + result.selection = state.pending + return state + } + const now = Math.max(Date.now(), state.updatedAt) + const selection = select(state, now) + result.selection = selection + return State.parse({ ...state, pending: selection, revision: state.revision + 1, updatedAt: now }) + }) + if (!result.selection) throw new Error(`Failure discovery selection was not durable after recording`) + return result.selection + } + + export async function observe(streamID: string, input: Observe) { + const value = Observe.parse(input) + const validations = value.validations.toSorted( + (left, right) => + HarnessContract.FailureValidatorKind.options.indexOf(left.kind) - + HarnessContract.FailureValidatorKind.options.indexOf(right.kind), + ) + const contract = await HarnessAdapter.authorize(value.sessionID, value.evaluatorToken) + const submittedAt = Date.now() + if (value.evaluatedAt > submittedAt + 300_000) { + throw new Error(`Failure discovery outcome is unreasonably future-dated`) + } + await JsonStore.update(file(value.sessionID, Hash.parse(streamID)), (data) => { + const state = parse(data) + match(state, contract) + const previous = state.attempts.find((attempt) => attempt.selection.selectionID === value.selectionID) + if (previous) { + const submitted = { + generation: value.generation, + validations, + outcome: value.outcome, + evaluatedAt: value.evaluatedAt, + } + const recorded = { + generation: previous.generation, + validations: previous.validations, + outcome: previous.outcome, + evaluatedAt: previous.evaluatedAt, + } + if (same(submitted, recorded)) return state + throw new Error(`Failure discovery attempt is immutable once recorded`) + } + if (state.status !== "active" || !state.pending) throw new Error(`Failure discovery has no pending selection`) + if (state.pending.selectionID !== value.selectionID) { + throw new Error(`Failure discovery attempt does not match the server-selected topic and anchors`) + } + if (value.evaluatedAt < state.pending.selectedAt) { + throw new Error(`Failure discovery outcome predates its selection`) + } + const result = derive(state.config, state.attempts, value.generation, validations, value.outcome) + if (!result.valid) throw new Error(result.error) + const now = Math.max(submittedAt, state.updatedAt) + const stable = { + selection: state.pending, + generation: value.generation, + validations, + outcome: value.outcome, + admissible: result.admissible, + reward: result.reward, + evaluatedAt: value.evaluatedAt, + recordedAt: now, + } + const attempt = Attempt.parse({ ...stable, attemptID: digest(stable) }) + const attempts = [...state.attempts, attempt] + const stats = statistics(state.config, attempts) + const stop = stopped(state.config, attempts) + return State.parse({ + ...state, + pending: undefined, + attempts, + statistics: stats, + status: stop ? "completed" : "active", + stopReason: stop, + revision: state.revision + 1, + updatedAt: now, + }) + }) + return read(value.sessionID, streamID) + } + + export async function seal(streamID: string, input: Access) { + const access = Access.parse(input) + const contract = await HarnessAdapter.authorize(access.sessionID, access.evaluatorToken) + const state = await read(access.sessionID, streamID) + match(state, contract) + if (state.status !== "completed" || !state.stopReason) { + throw new Error(`Failure discovery must reach a terminal state before sealing`) + } + const stable = { + schemaVersion: 1 as const, + protocolVersion: "topic-aware-failure-receipt-v1" as const, + streamID: state.streamID, + runID: state.runID, + sessionID: state.sessionID, + contractFingerprint: state.contractFingerprint, + subject: state.subject, + auditReceiptID: state.auditReceiptID, + sourcePoolSHA256: state.sourcePoolSHA256, + config: state.config, + attemptIDs: state.attempts.map((attempt) => attempt.attemptID), + statistics: state.statistics, + stopReason: state.stopReason, + revision: state.revision, + completedAt: state.updatedAt, + sealedAt: state.updatedAt, + } + const receipt = Receipt.parse({ ...stable, receiptID: digest(stable) }) + await JsonStore.update(receiptFile(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Failure discovery receipt is immutable once recorded`) + }) + const saved = await readReceipt(receipt.receiptID) + if (!saved) throw new Error(`Failure discovery receipt was not durable after recording`) + return saved + } + + export async function readReceipt(receiptID: string) { + const id = Hash.parse(receiptID) + const parsed = Receipt.safeParse(await JsonStore.read(receiptFile(id))) + if (!parsed.success || parsed.data.receiptID !== id) return null + const state = await JsonStore.read(file(parsed.data.sessionID, parsed.data.streamID)) + const current = State.safeParse(state) + if (!current.success || !replay(current.data) || current.data.status !== "completed") return null + const source = await HarnessAudit.readReceipt(current.data.auditReceiptID) + if ( + !source || + source.contractFingerprint !== current.data.contractFingerprint || + source.poolFingerprint !== current.data.sourcePoolSHA256 || + !same(source.subject, current.data.subject) + ) { + return null + } + const snapshot = { + streamID: current.data.streamID, + runID: current.data.runID, + sessionID: current.data.sessionID, + contractFingerprint: current.data.contractFingerprint, + subject: current.data.subject, + auditReceiptID: current.data.auditReceiptID, + sourcePoolSHA256: current.data.sourcePoolSHA256, + config: current.data.config, + attemptIDs: current.data.attempts.map((attempt) => attempt.attemptID), + statistics: current.data.statistics, + stopReason: current.data.stopReason, + revision: current.data.revision, + completedAt: current.data.updatedAt, + sealedAt: current.data.updatedAt, + } + const receipt = { ...parsed.data } + delete (receipt as Partial).schemaVersion + delete (receipt as Partial).protocolVersion + delete (receipt as Partial).receiptID + return same(snapshot, receipt) ? parsed.data : null + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: { type: "run" | "candidate"; id: string } + evaluatedAt: number + recordedAt: number + }) { + const receipt = await readReceipt(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt failure discovery receipt ${input.receiptID}`) + if (!input.contract.failureDiscovery) { + throw new Error(`Evaluation cites a failure discovery receipt without a bound protocol`) + } + if ( + receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract) || + receipt.sessionID !== input.contract.sessionID || + receipt.runID !== input.contract.runID + ) { + throw new Error(`Failure discovery receipt belongs to a different harness run`) + } + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Failure discovery receipt belongs to a different evaluation subject`) + } + if (receipt.completedAt < input.contract.createdAt) { + throw new Error(`Failure discovery receipt predates the bound harness contract`) + } + if (receipt.completedAt > input.evaluatedAt || receipt.sealedAt > input.recordedAt) { + throw new Error(`Evaluation predates its failure discovery receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/formal.ts b/backend/cli/src/session/harness/formal.ts new file mode 100644 index 00000000..2df1d3d9 --- /dev/null +++ b/backend/cli/src/session/harness/formal.ts @@ -0,0 +1,614 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessFormal { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(240), + }) + .strict() + export type Subject = z.infer + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + export type Access = z.infer + + export const FileRole = z.enum([ + "challenge", + "statement", + "proof", + "lean_toolchain", + "lake_manifest", + "dependency_tree", + "config", + "support", + ]) + export type FileRole = z.infer + + export const File = z + .object({ + path: z + .string() + .min(1) + .max(500) + .refine((value) => !path.isAbsolute(value) && !value.split(/[\\/]/).includes(".."), "Path must be relative"), + role: FileRole, + sha256: Hash, + }) + .strict() + export type File = z.infer + + const Build = z + .object({ + verifierArtifactSHA256: Hash, + exitCode: z.number().int(), + warnings: z.number().int().nonnegative(), + transcriptSHA256: Hash, + }) + .strict() + + const Finding = z + .object({ + construct: HarnessContract.FormalForbidden, + path: File.shape.path, + line: z.number().int().positive(), + }) + .strict() + + const Source = z + .object({ + verifierArtifactSHA256: Hash, + complete: z.boolean(), + findings: z.array(Finding).max(128), + transcriptSHA256: Hash, + }) + .strict() + + const Axioms = z + .object({ + verifierArtifactSHA256: Hash, + complete: z.boolean(), + typesTraversed: z.boolean(), + observed: z.array(z.string().min(1).max(300)).max(128), + transcriptSHA256: Hash, + }) + .strict() + + const Fresh = z + .object({ + verifierArtifactSHA256: Hash, + fresh: z.boolean(), + exitCode: z.number().int(), + transcriptSHA256: Hash, + }) + .strict() + + const Crosscheck = z + .object({ + role: z.enum(["lean_kernel", "external_checker"]), + verifierArtifactSHA256: Hash, + accepted: z.boolean(), + transcriptSHA256: Hash, + }) + .strict() + + const External = z + .object({ + comparatorArtifactSHA256: Hash, + sandboxImageSHA256: Hash, + sandboxed: z.boolean(), + challengeMatched: z.boolean(), + proofTermSHA256: Hash, + transcriptSHA256: Hash, + checks: z.array(Crosscheck).min(2).max(2), + }) + .strict() + + export const Submit = Access.extend({ + subject: Subject, + artifactSHA256: Hash, + relation: HarnessContract.FormalRelation, + challengeSHA256: Hash, + statementSHA256: Hash, + declaration: z.string().min(1).max(500), + module: z.string().min(1).max(500), + environment: z + .object({ + leanVersion: z.string().min(1).max(200), + leanToolchainSHA256: Hash, + lakeManifestSHA256: Hash, + dependencyTreeSHA256: Hash, + }) + .strict(), + manifest: z + .object({ + complete: z.boolean(), + files: z.array(File).min(6).max(10_000), + }) + .strict(), + verification: z + .object({ + startedAt: z.number().int().positive(), + endedAt: z.number().int().positive(), + build: Build, + source: Source, + axioms: Axioms, + fresh: Fresh.optional(), + external: External.optional(), + }) + .strict(), + }).strict() + export type Submit = z.infer + + export const Metrics = z + .object({ + files: z.number().int().positive(), + warnings: z.number().int().nonnegative(), + observedAxioms: z.number().int().nonnegative(), + disallowedAxioms: z.array(z.string().min(1).max(300)).max(128), + manifestComplete: z.boolean(), + buildAccepted: z.boolean(), + sourceAuditAccepted: z.boolean(), + forbiddenFindings: z.array(Finding).max(128), + axiomAuditAccepted: z.boolean(), + freshRecheckAccepted: z.boolean(), + externalCrosscheckAccepted: z.boolean(), + statementMatched: z.boolean(), + }) + .strict() + export type Metrics = z.infer + + export const Receipt = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("formal-proof-receipt-v1"), + receiptID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + protocolSHA256: Hash, + subject: Subject, + artifactSHA256: Hash, + relation: HarnessContract.FormalRelation, + challengeSHA256: Hash, + statementSHA256: Hash, + declaration: z.string().min(1).max(500), + module: z.string().min(1).max(500), + environment: Submit.shape.environment, + manifestSHA256: Hash, + files: z.array(File).min(6).max(10_000), + verification: Submit.shape.verification, + tier: HarnessContract.FormalTier, + metrics: Metrics, + status: z.enum(["passed", "failed"]), + failures: z.array(z.string().min(1).max(500)).max(32), + recordedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + delete stable.recordedAt + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Formal proof receipt hash is invalid" }) + }) + export type Receipt = z.infer + + export function prompt(contract: HarnessContract.Info) { + const protocol = contract.formalProof + if (!protocol) return "" + return [ + '', + `Produce a Lean 4 ${protocol.relation} for the frozen declaration ${protocol.declaration} in ${protocol.module}.`, + `The trusted challenge, canonical statement, toolchain, dependency graph, verifier artifacts, and ${protocol.tier} trust tier are immutable.`, + "A successful build alone is insufficient: transitive axiom use is audited, and higher tiers require a fresh kernel replay or sandboxed independent cross-check.", + "Do not use sorry, debug.skipKernelTC, an undeclared axiom, a substituted statement, or a repaired theorem when the contract requires an exact proof.", + "Formal verification proves the frozen Lean statement only; semantic correspondence to informal mathematics remains a separate review obligation.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + return contract ? prompt(contract) : "" + } + + const root = path.join(Global.Path.data, "harness", "formal") + const receiptFile = (receiptID: string) => path.join(root, "receipts", `${receiptID}.json`) + const subjectFile = (sessionID: string, subject: Subject) => + path.join( + root, + "subjects", + encodeURIComponent(sessionID), + `${encodeURIComponent(`${subject.type}:${subject.id}`)}.json`, + ) + + async function target(contract: HarnessContract.Info, subject: Subject) { + if (subject.type === "run") { + if (subject.id !== contract.runID) throw new Error(`Formal proof run subject does not match its contract`) + return { createdAt: contract.createdAt } + } + const state = await import("./search") + .then((module) => module.HarnessSearch.read(contract.sessionID)) + .catch(() => null) + const candidate = state?.runID === contract.runID ? state.candidates[subject.id] : undefined + if (!candidate) throw new Error(`Formal proof candidate does not exist in the bound search`) + return { createdAt: candidate.createdAt, artifactSHA256: candidate.artifact.sha256 } + } + + const verifier = (protocol: HarnessContract.FormalProof, role: HarnessContract.FormalVerifierRole) => { + const item = protocol.verifiers.find((entry) => entry.role === role) + if (!item) throw new Error(`Formal proof protocol has no ${role} verifier`) + return item + } + + function manifest(files: File[], max: number) { + if (files.length > max) throw new Error(`Formal proof manifest exceeds its frozen file budget`) + const parsed = files.map((item) => File.parse(item)) + if (new Set(parsed.map((item) => item.path)).size !== parsed.length) { + throw new Error(`Formal proof manifest paths must be unique`) + } + if (parsed.some((item, index) => Boolean(index) && parsed[index - 1]!.path.localeCompare(item.path) >= 0)) { + throw new Error(`Formal proof manifest files must use canonical path order`) + } + for (const role of [ + "challenge", + "statement", + "proof", + "lean_toolchain", + "lake_manifest", + "dependency_tree", + ] as const) { + if (parsed.filter((item) => item.role === role).length !== 1) { + throw new Error(`Formal proof manifest requires exactly one ${role} file`) + } + } + return parsed + } + + function identities(input: Submit, protocol: HarnessContract.FormalProof) { + if (input.verification.build.verifierArtifactSHA256 !== verifier(protocol, "lean_kernel").artifactSHA256) { + throw new Error(`Formal proof build used an unbound Lean kernel`) + } + if (input.verification.source.verifierArtifactSHA256 !== verifier(protocol, "source_auditor").artifactSHA256) { + throw new Error(`Formal proof source audit used an unbound verifier`) + } + if (input.verification.axioms.verifierArtifactSHA256 !== verifier(protocol, "axiom_auditor").artifactSHA256) { + throw new Error(`Formal proof axiom audit used an unbound verifier`) + } + const fresh = protocol.tier !== "kernel" + if (Boolean(input.verification.fresh) !== fresh) { + throw new Error(`Formal proof submission does not match its frozen fresh-recheck tier`) + } + if ( + input.verification.fresh && + input.verification.fresh.verifierArtifactSHA256 !== verifier(protocol, "fresh_rechecker").artifactSHA256 + ) { + throw new Error(`Formal proof fresh replay used an unbound verifier`) + } + const external = protocol.tier === "external_crosscheck" + if (Boolean(input.verification.external) !== external) { + throw new Error(`Formal proof submission does not match its frozen external-check tier`) + } + if (!input.verification.external) return + if ( + input.verification.external.comparatorArtifactSHA256 !== + verifier(protocol, "sandbox_comparator").artifactSHA256 || + input.verification.external.sandboxImageSHA256 !== protocol.sandboxImageSHA256 + ) { + throw new Error(`Formal proof external replay changed its comparator or sandbox`) + } + const checks = input.verification.external.checks + if (new Set(checks.map((item) => item.role)).size !== checks.length) { + throw new Error(`Formal proof external checker roles must be unique`) + } + for (const role of ["lean_kernel", "external_checker"] as const) { + const check = checks.find((item) => item.role === role) + if (!check || check.verifierArtifactSHA256 !== verifier(protocol, role).artifactSHA256) { + throw new Error(`Formal proof external replay changed its ${role} verifier`) + } + } + } + + function assess(input: { protocol: HarnessContract.FormalProof; value: Submit; files: File[] }) { + const observed = input.value.verification.axioms.observed + const disallowedAxioms = observed.filter((item) => !input.protocol.allowedAxioms.includes(item)) + const buildAccepted = input.value.verification.build.exitCode === 0 && input.value.verification.build.warnings === 0 + const sourceAuditAccepted = + input.value.verification.source.complete && !input.value.verification.source.findings.length + const axiomAuditAccepted = + input.value.verification.axioms.complete && + input.value.verification.axioms.typesTraversed && + !disallowedAxioms.length + const freshRecheckAccepted = + input.protocol.tier === "kernel" || + Boolean(input.value.verification.fresh?.fresh && input.value.verification.fresh.exitCode === 0) + const external = input.value.verification.external + const statementMatched = input.protocol.tier !== "external_crosscheck" || Boolean(external?.challengeMatched) + const externalCrosscheckAccepted = + input.protocol.tier !== "external_crosscheck" || + Boolean(external?.sandboxed && external.challengeMatched && external.checks.every((item) => item.accepted)) + const metrics = Metrics.parse({ + files: input.files.length, + warnings: input.value.verification.build.warnings, + observedAxioms: observed.length, + disallowedAxioms, + manifestComplete: input.value.manifest.complete, + buildAccepted, + sourceAuditAccepted, + forbiddenFindings: input.value.verification.source.findings, + axiomAuditAccepted, + freshRecheckAccepted, + externalCrosscheckAccepted, + statementMatched, + }) + const failures = [ + ...(!metrics.manifestComplete ? ["proof manifest is not complete"] : []), + ...(!metrics.buildAccepted ? ["Lean build failed or emitted warnings"] : []), + ...(!input.value.verification.source.complete ? ["source audit is incomplete"] : []), + ...input.value.verification.source.findings.map( + (item) => `forbidden construct ${item.construct} at ${item.path}:${item.line}`, + ), + ...(!input.value.verification.axioms.complete ? ["transitive axiom inventory is incomplete"] : []), + ...(!input.value.verification.axioms.typesTraversed ? ["axiom audit did not traverse axiom types"] : []), + ...disallowedAxioms.map((item) => `disallowed axiom: ${item}`), + ...(!metrics.freshRecheckAccepted ? ["fresh kernel replay failed"] : []), + ...(!metrics.statementMatched ? ["external comparator did not match the trusted challenge"] : []), + ...(!metrics.externalCrosscheckAccepted ? ["sandboxed independent cross-check failed"] : []), + ] + return { metrics, status: failures.length ? ("failed" as const) : ("passed" as const), failures } + } + + function verify(receipt: Receipt, protocol: HarnessContract.FormalProof) { + if ( + receipt.relation !== protocol.relation || + receipt.challengeSHA256 !== protocol.challengeSHA256 || + receipt.statementSHA256 !== protocol.statementSHA256 || + receipt.declaration !== protocol.declaration || + receipt.module !== protocol.module || + receipt.tier !== protocol.tier + ) { + throw new Error(`Formal proof receipt changed its frozen theorem identity`) + } + if ( + receipt.environment.leanVersion !== protocol.leanVersion || + receipt.environment.leanToolchainSHA256 !== protocol.leanToolchainSHA256 || + receipt.environment.lakeManifestSHA256 !== protocol.lakeManifestSHA256 || + receipt.environment.dependencyTreeSHA256 !== protocol.dependencyTreeSHA256 + ) { + throw new Error(`Formal proof receipt changed its frozen Lean environment`) + } + const files = manifest(receipt.files, protocol.maxFiles) + if (digest(files) !== receipt.manifestSHA256) { + throw new Error(`Formal proof receipt does not match its file manifest`) + } + const value = Submit.parse({ + sessionID: receipt.sessionID, + evaluatorToken: "receipt-verification-token-0000000000000000", + subject: receipt.subject, + artifactSHA256: receipt.artifactSHA256, + relation: receipt.relation, + challengeSHA256: receipt.challengeSHA256, + statementSHA256: receipt.statementSHA256, + declaration: receipt.declaration, + module: receipt.module, + environment: receipt.environment, + manifest: { complete: receipt.metrics.manifestComplete, files }, + verification: receipt.verification, + }) + identities(value, protocol) + const result = assess({ protocol, value, files }) + if ( + !same(result.metrics, receipt.metrics) || + result.status !== receipt.status || + !same(result.failures, receipt.failures) + ) { + throw new Error(`Formal proof receipt does not match backend-derived verification`) + } + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + if (value.sessionID !== contract.sessionID) { + throw new Error(`Formal proof session does not match its bound harness contract`) + } + const protocol = contract.formalProof + if (!protocol) throw new Error(`Harness contract does not require formal proof validation`) + if ( + value.relation !== protocol.relation || + value.challengeSHA256 !== protocol.challengeSHA256 || + value.statementSHA256 !== protocol.statementSHA256 || + value.declaration !== protocol.declaration || + value.module !== protocol.module + ) { + throw new Error(`Formal proof submission changed the frozen theorem or claim relation`) + } + if ( + value.environment.leanVersion !== protocol.leanVersion || + value.environment.leanToolchainSHA256 !== protocol.leanToolchainSHA256 || + value.environment.lakeManifestSHA256 !== protocol.lakeManifestSHA256 || + value.environment.dependencyTreeSHA256 !== protocol.dependencyTreeSHA256 + ) { + throw new Error(`Formal proof submission changed the frozen Lean environment`) + } + if (value.verification.endedAt < value.verification.startedAt) { + throw new Error(`Formal proof verification ends before it starts`) + } + const recordedAt = Date.now() + const subject = await target(contract, value.subject) + if (value.verification.startedAt < subject.createdAt || value.verification.endedAt > recordedAt) { + throw new Error(`Formal proof verification falls outside its bound subject interval`) + } + if (subject.artifactSHA256 && subject.artifactSHA256 !== value.artifactSHA256) { + throw new Error(`Formal proof receipt changed the candidate artifact`) + } + identities(value, protocol) + const files = manifest(value.manifest.files, protocol.maxFiles) + const byRole = (role: FileRole) => files.find((item) => item.role === role)! + if ( + byRole("challenge").sha256 !== value.challengeSHA256 || + byRole("statement").sha256 !== value.statementSHA256 || + byRole("proof").sha256 !== value.artifactSHA256 || + byRole("lean_toolchain").sha256 !== value.environment.leanToolchainSHA256 || + byRole("lake_manifest").sha256 !== value.environment.lakeManifestSHA256 || + byRole("dependency_tree").sha256 !== value.environment.dependencyTreeSHA256 + ) { + throw new Error(`Formal proof manifest does not bind its challenge, proof, or environment artifacts`) + } + if ( + new Set(value.verification.axioms.observed).size !== value.verification.axioms.observed.length || + value.verification.axioms.observed.some( + (item, index) => Boolean(index) && value.verification.axioms.observed[index - 1]!.localeCompare(item) >= 0, + ) + ) { + throw new Error(`Formal proof observed axioms must be unique and use canonical order`) + } + const findings = value.verification.source.findings + const keys = findings.map( + (item) => `${item.path}\u0000${item.line.toString().padStart(12, "0")}\u0000${item.construct}`, + ) + if ( + new Set(keys).size !== keys.length || + keys.some((item, index) => Boolean(index) && keys[index - 1]!.localeCompare(item) >= 0) + ) { + throw new Error(`Formal proof source findings must be unique and use canonical order`) + } + if (findings.some((item) => !files.some((file) => file.path === item.path))) { + throw new Error(`Formal proof source finding references a file outside the complete manifest`) + } + const result = assess({ protocol, value, files }) + const stable = { + schemaVersion: 1 as const, + protocolVersion: "formal-proof-receipt-v1" as const, + runID: contract.runID, + sessionID: contract.sessionID, + contractFingerprint: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + subject: value.subject, + artifactSHA256: value.artifactSHA256, + relation: value.relation, + challengeSHA256: value.challengeSHA256, + statementSHA256: value.statementSHA256, + declaration: value.declaration, + module: value.module, + environment: value.environment, + manifestSHA256: digest(files), + files, + verification: value.verification, + tier: protocol.tier, + metrics: result.metrics, + status: result.status, + failures: result.failures, + } + const receipt = Receipt.parse({ ...stable, receiptID: digest(stable), recordedAt }) + const claimed = await JsonStore.read(subjectFile(receipt.sessionID, receipt.subject)) + if (Object.keys(claimed).length) { + const current = Receipt.parse(claimed) + if (current.receiptID !== receipt.receiptID) { + throw new Error(`Formal proof subject already has a canonical receipt`) + } + } + await JsonStore.update(receiptFile(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Formal proof receipt is immutable once recorded`) + }) + await JsonStore.update(subjectFile(receipt.sessionID, receipt.subject), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Formal proof subject already has a canonical receipt`) + }) + const saved = await readReceipt(receipt.receiptID) + if (!saved) throw new Error(`Formal proof receipt was not durable after recording`) + return saved + } + + export async function readReceipt(receiptID: string) { + const id = Hash.parse(receiptID) + const parsed = Receipt.safeParse(await JsonStore.read(receiptFile(id))) + if (!parsed.success || parsed.data.receiptID !== id) return null + const canonical = Receipt.safeParse(await JsonStore.read(subjectFile(parsed.data.sessionID, parsed.data.subject))) + if (!canonical.success || canonical.data.receiptID !== id || !same(canonical.data, parsed.data)) return null + return parsed.data + } + + export async function read(receiptID: string, contract: HarnessContract.Info) { + const receipt = await readReceipt(receiptID) + if (!receipt || receipt.sessionID !== contract.sessionID) + throw new Error(`Unknown formal proof receipt ${receiptID}`) + const protocol = contract.formalProof + if (!protocol || receipt.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Formal proof receipt belongs to a different harness run`) + } + verify(receipt, protocol) + const subject = await target(contract, receipt.subject) + if ( + receipt.verification.startedAt < subject.createdAt || + receipt.verification.endedAt > receipt.recordedAt || + (subject.artifactSHA256 && subject.artifactSHA256 !== receipt.artifactSHA256) + ) { + throw new Error(`Formal proof receipt changed its bound subject, artifact, or interval`) + } + return receipt + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: Subject + evaluatedAt: number + recordedAt: number + requirePassed: boolean + }) { + const receipt = await readReceipt(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt formal proof receipt ${input.receiptID}`) + const protocol = input.contract.formalProof + if (!protocol) throw new Error(`Evaluation cites a formal proof receipt without a bound protocol`) + if ( + receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract) || + receipt.protocolSHA256 !== digest(protocol) || + receipt.sessionID !== input.contract.sessionID || + receipt.runID !== input.contract.runID + ) { + throw new Error(`Formal proof receipt belongs to a different harness run`) + } + verify(receipt, protocol) + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Formal proof receipt belongs to a different evaluation subject`) + } + const subject = await target(input.contract, input.subject) + if ( + receipt.verification.startedAt < subject.createdAt || + receipt.verification.endedAt > receipt.recordedAt || + (subject.artifactSHA256 && subject.artifactSHA256 !== receipt.artifactSHA256) + ) { + throw new Error(`Formal proof receipt belongs to a different subject artifact or interval`) + } + if ( + receipt.verification.endedAt > input.evaluatedAt || + receipt.recordedAt > input.evaluatedAt || + receipt.recordedAt > input.recordedAt + ) { + throw new Error(`Evaluation predates its formal proof receipt`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A passing final evaluation requires a passing formal proof receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/integrity.ts b/backend/cli/src/session/harness/integrity.ts new file mode 100644 index 00000000..2a36fd42 --- /dev/null +++ b/backend/cli/src/session/harness/integrity.ts @@ -0,0 +1,457 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessIntegrity { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + export const Artifact = z + .object({ + uri: z.string().min(1).max(2_048), + sha256: Hash, + }) + .strict() + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(240), + artifact: Artifact, + }) + .strict() + + export const Trace = z + .object({ + artifact: Artifact, + schemaSHA256: Hash, + events: z.number().int().min(1).max(10_000_000), + dropped: z.number().int().min(0).max(10_000_000), + startedAt: z.number().int().positive(), + endedAt: z.number().int().positive(), + }) + .strict() + .refine((value) => value.endedAt >= value.startedAt, "Trace end cannot predate its start") + + export const Model = z + .object({ + name: z.string().min(1).max(500), + baseArtifactSHA256: Hash, + configSHA256: Hash, + outputArtifactSHA256: Hash, + lineageVerified: z.boolean(), + }) + .strict() + + export const Audit = z + .object({ + kind: HarnessContract.IntegrityAuditKind, + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + promptSHA256: Hash, + decision: z.enum(["clean", "flagged", "abstain"]), + confidence: z.number().min(0).max(1), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + export const Activity = z + .object({ + unapprovedExternalModelCalls: z.number().int().min(0).max(10_000_000), + benchmarkLookupEvents: z.number().int().min(0).max(10_000_000), + hiddenCanaryManifestSHA256: Hash, + hiddenCanariesTested: z.number().int().min(0).max(10_000), + hiddenCanaryViolations: z.number().int().min(0).max(10_000), + }) + .strict() + .refine( + (value) => value.hiddenCanaryViolations <= value.hiddenCanariesTested, + "Hidden canary violations cannot exceed tested canaries", + ) + + export const Validator = z + .object({ + name: z.literal("verify-benchmark-integrity"), + version: z.literal(1), + scriptSHA256: Hash, + }) + .strict() + + export const Submit = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + protocol: HarnessContract.Integrity, + subject: Subject, + trace: Trace, + model: Model, + audits: z + .array(Audit) + .length(HarnessContract.IntegrityAuditKind.options.length) + .refine( + (items) => new Set(items.map((item) => item.kind)).size === items.length, + "Integrity audits must be unique", + ), + activity: Activity, + validator: Validator, + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + }) + .strict() + export type Submit = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + + export const Failure = z.enum([ + "trace_schema", + "trace_event_floor", + "trace_coverage", + "model_name", + "model_base_artifact", + "model_config", + "model_lineage", + "forbidden_model_artifact", + "test_item_contamination", + "external_model_use", + "benchmark_lookup", + "hidden_canary_manifest", + "hidden_canary_coverage", + "hidden_canary_violation", + ]) + + export const Checks = z + .object({ + traceCompleteness: z.boolean(), + modelIdentity: z.boolean(), + testItemContamination: z.boolean(), + externalModelUse: z.boolean(), + benchmarkLookup: z.boolean(), + hiddenCanary: z.boolean(), + }) + .strict() + + export const Info = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + submissionID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + subject: Subject, + evaluator: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "external"]), + }) + .strict(), + protocol: HarnessContract.Integrity, + trace: Trace, + traceCoverage: z.number().min(0).max(1), + model: Model, + audits: z.array(Audit).length(HarnessContract.IntegrityAuditKind.options.length), + activity: Activity, + validator: Validator, + checks: Checks, + status: z.enum(["passed", "failed"]), + failures: z.array(Failure).max(Failure.options.length), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + export type Info = z.infer + + const request = (input: { + runID: string + sessionID: string + protocol: HarnessContract.Integrity + subject: z.infer + trace: z.infer + model: z.infer + audits: z.infer[] + activity: z.infer + validator: z.infer + evidence: string[] + evaluatedAt: number + }) => digest(input) + + function outcome( + protocol: HarnessContract.Integrity, + trace: z.infer, + model: z.infer, + audits: z.infer[], + activity: z.infer, + ) { + const coverage = trace.events / (trace.events + trace.dropped) + const decision = (kind: HarnessContract.IntegrityAuditKind) => + audits.find((item) => item.kind === kind)?.decision === "clean" + const traceFailures = [ + ...(trace.schemaSHA256 === protocol.traceSchemaSHA256 ? [] : ["trace_schema" as const]), + ...(trace.events >= protocol.minEvents ? [] : ["trace_event_floor" as const]), + ...(coverage >= protocol.minCoverage ? [] : ["trace_coverage" as const]), + ] + const modelFailures = [ + ...(model.name === protocol.assignedModel.name ? [] : ["model_name" as const]), + ...(model.baseArtifactSHA256 === protocol.assignedModel.baseArtifactSHA256 + ? [] + : ["model_base_artifact" as const]), + ...(model.configSHA256 === protocol.assignedModel.configSHA256 ? [] : ["model_config" as const]), + ...(model.lineageVerified ? [] : ["model_lineage" as const]), + ...(protocol.forbiddenModelArtifacts.includes(model.outputArtifactSHA256) + ? ["forbidden_model_artifact" as const] + : []), + ] + const auditFailures = [ + ...(decision("test_item_contamination") ? [] : ["test_item_contamination" as const]), + ...(decision("external_model_use") && activity.unapprovedExternalModelCalls === 0 + ? [] + : ["external_model_use" as const]), + ...(decision("benchmark_lookup") && activity.benchmarkLookupEvents === 0 ? [] : ["benchmark_lookup" as const]), + ] + const canaryFailures = [ + ...(activity.hiddenCanaryManifestSHA256 === protocol.hiddenCanaryManifestSHA256 + ? [] + : ["hidden_canary_manifest" as const]), + ...(activity.hiddenCanariesTested >= protocol.minHiddenCanaries ? [] : ["hidden_canary_coverage" as const]), + ...(activity.hiddenCanaryViolations === 0 ? [] : ["hidden_canary_violation" as const]), + ] + const failures = Failure.array().parse([...traceFailures, ...modelFailures, ...auditFailures, ...canaryFailures]) + const checks = Checks.parse({ + traceCompleteness: traceFailures.length === 0, + modelIdentity: modelFailures.length === 0, + testItemContamination: !auditFailures.includes("test_item_contamination"), + externalModelUse: !auditFailures.includes("external_model_use"), + benchmarkLookup: !auditFailures.includes("benchmark_lookup"), + hiddenCanary: canaryFailures.length === 0, + }) + return { + traceCoverage: coverage, + checks, + status: failures.length ? ("failed" as const) : ("passed" as const), + failures, + } + } + + const State = z + .object({ + schemaVersion: z.literal(1), + items: z.record(Hash, Info), + order: z.array(Hash), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.order).size !== value.order.length) { + ctx.addIssue({ code: "custom", path: ["order"], message: "Integrity receipt order must be unique" }) + } + for (const id of value.order) { + const receipt = value.items[id] + if (!receipt) { + ctx.addIssue({ code: "custom", path: ["order"], message: `Integrity receipt ${id} is missing` }) + continue + } + if (receipt.receiptID !== id) { + ctx.addIssue({ code: "custom", path: ["items", id], message: "Integrity receipt key does not match its ID" }) + } + const payload = structuredClone(receipt) as Record + delete payload.receiptID + if (digest(payload) !== id) { + ctx.addIssue({ code: "custom", path: ["items", id], message: "Integrity receipt content hash is invalid" }) + } + const derived = outcome(receipt.protocol, receipt.trace, receipt.model, receipt.audits, receipt.activity) + if ( + !same( + { + traceCoverage: receipt.traceCoverage, + checks: receipt.checks, + status: receipt.status, + failures: receipt.failures, + }, + derived, + ) + ) { + ctx.addIssue({ code: "custom", path: ["items", id], message: "Integrity receipt outcome derivation drifted" }) + } + if ( + receipt.submissionID !== + request({ + runID: receipt.runID, + sessionID: receipt.sessionID, + protocol: receipt.protocol, + subject: receipt.subject, + trace: receipt.trace, + model: receipt.model, + audits: receipt.audits, + activity: receipt.activity, + validator: receipt.validator, + evidence: receipt.evidence, + evaluatedAt: receipt.evaluatedAt, + }) + ) { + ctx.addIssue({ code: "custom", path: ["items", id], message: "Integrity submission content hash is invalid" }) + } + } + for (const id of Object.keys(value.items)) { + if (value.order.includes(id)) continue + ctx.addIssue({ code: "custom", path: ["items", id], message: "Integrity receipt is absent from journal order" }) + } + }) + type State = z.infer + + const root = path.join(Global.Path.data, "harness", "integrity") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const empty = (): State => ({ schemaVersion: 1, items: {}, order: [] }) + const state = (input: Record) => State.parse(Object.keys(input).length ? input : empty()) + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const bound = HarnessContract.Info.parse(contract) + const protocol = bound.integrity + if (!protocol) throw new Error(`No runtime integrity protocol is bound to session ${value.sessionID}`) + if (bound.sessionID !== value.sessionID || bound.runID !== value.runID) { + throw new Error(`Integrity receipt does not match the bound harness run`) + } + if (value.trace.startedAt < bound.createdAt) throw new Error(`Integrity trace predates the harness contract`) + if (value.evaluatedAt < value.trace.endedAt) throw new Error(`Integrity validation predates the trace end`) + if (value.evaluatedAt > Date.now() + 300_000) throw new Error(`Integrity receipt is implausibly future-dated`) + if (!same(value.protocol, protocol)) + throw new Error(`Integrity protocol does not match the immutable harness contract`) + if (value.validator.scriptSHA256 !== protocol.validatorSHA256) { + throw new Error(`Integrity validator does not match the immutable harness contract`) + } + const expected = protocol.auditors.toSorted((left, right) => left.kind.localeCompare(right.kind)) + const audits = value.audits + .map((item) => ({ ...item, evidence: item.evidence.toSorted() })) + .toSorted((left, right) => left.kind.localeCompare(right.kind)) + const identities = audits.map((item) => ({ + kind: item.kind, + name: item.name, + version: item.version, + promptSHA256: item.promptSHA256, + })) + if (!same(expected, identities)) throw new Error(`Integrity auditors do not match the immutable harness contract`) + if (value.subject.type === "run" && value.subject.id !== bound.runID) { + throw new Error(`Run integrity receipt subject does not match the contract run`) + } + if (value.subject.type === "candidate") { + const search = await import("./search").then((module) => module.HarnessSearch.read(value.sessionID)) + const candidate = search.candidates[value.subject.id] + if (!candidate) throw new Error(`Integrity receipt candidate does not exist in the bound search`) + if (!same(candidate.artifact, value.subject.artifact)) { + throw new Error(`Integrity receipt artifact does not match the candidate artifact`) + } + } + const evidence = value.evidence.toSorted() + const submissionID = request({ + runID: value.runID, + sessionID: value.sessionID, + protocol, + subject: value.subject, + trace: value.trace, + model: value.model, + audits, + activity: value.activity, + validator: value.validator, + evidence, + evaluatedAt: value.evaluatedAt, + }) + const result = outcome(protocol, value.trace, value.model, audits, value.activity) + const payload = { + schemaVersion: 1 as const, + submissionID, + runID: value.runID, + sessionID: value.sessionID, + contractFingerprint: HarnessContract.fingerprint(bound), + subject: value.subject, + evaluator: { + name: bound.benchmark.evaluator, + version: bound.benchmark.evaluatorVersion!, + source: bound.benchmark.evaluatorSource!, + }, + protocol, + trace: value.trace, + traceCoverage: result.traceCoverage, + model: value.model, + audits, + activity: value.activity, + validator: value.validator, + checks: result.checks, + status: result.status, + failures: result.failures, + evidence, + evaluatedAt: value.evaluatedAt, + recordedAt: Date.now(), + } + const receipt = Info.parse({ ...payload, receiptID: digest(payload) }) + const out = { value: receipt } + await JsonStore.update(file(value.sessionID), (data) => { + const current = state(data) + const existing = current.order.map((id) => current.items[id]!).find((item) => item.submissionID === submissionID) + if (existing) { + out.value = existing + return current + } + return State.parse({ + ...current, + items: { ...current.items, [receipt.receiptID]: receipt }, + order: [...current.order, receipt.receiptID], + }) + }) + return out.value + } + + export async function read(sessionID: string, receiptID: string) { + const current = state(await JsonStore.read(file(sessionID))) + return current.items[Hash.parse(receiptID)] ?? null + } + + export async function list(sessionID: string) { + const current = state(await JsonStore.read(file(sessionID))) + return current.order.map((id) => current.items[id]!) + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: { type: "run" | "candidate"; id: string } + requirePassed: boolean + evaluatedAt: number + recordedAt: number + }) { + const receipt = await read(input.contract.sessionID, input.receiptID) + if (!receipt) throw new Error(`Runtime integrity receipt ${input.receiptID} does not exist`) + if (receipt.runID !== input.contract.runID) + throw new Error(`Runtime integrity receipt does not match the harness run`) + if (receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract)) { + throw new Error(`Runtime integrity receipt does not match the immutable harness contract`) + } + if (!same(receipt.protocol, input.contract.integrity)) { + throw new Error(`Runtime integrity receipt does not match the bound protocol`) + } + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Runtime integrity receipt does not match the evaluated subject`) + } + if (receipt.evaluatedAt > input.evaluatedAt) { + throw new Error(`Benchmark evaluation predates its referenced runtime integrity receipt`) + } + if (receipt.recordedAt > input.recordedAt) { + throw new Error(`Benchmark evaluation was recorded before its runtime integrity receipt`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A benchmark result requires a passing runtime integrity receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/intervention.ts b/backend/cli/src/session/harness/intervention.ts new file mode 100644 index 00000000..504390ef --- /dev/null +++ b/backend/cli/src/session/harness/intervention.ts @@ -0,0 +1,761 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" +import { HarnessEvolution } from "./evolution" + +export namespace HarnessIntervention { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const Role = z.enum(["control", "arm"]) + const Status = z.enum(["passed", "failed", "inconclusive"]) + const stable = (input: unknown): unknown => { + if (Array.isArray(input)) return input.map(stable) + if (!input || typeof input !== "object") return input + return Object.fromEntries( + Object.entries(input as Record) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => [key, stable(value)]), + ) + } + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(stable(input))).digest("hex") + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + const Model = z + .object({ + provider: z.string().min(1).max(200), + name: z.string().min(1).max(500), + version: z.string().min(1).max(200), + }) + .strict() + + const Evaluator = z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "external"]), + }) + .strict() + + const Split = z + .object({ + name: z.string().min(1).max(200), + manifest: HarnessEvolution.Artifact, + }) + .strict() + + export const Condition = z + .object({ + seed: z.number().int(), + model: Model, + context: HarnessEvolution.Artifact, + evaluator: Evaluator, + split: Split, + environment: HarnessEvolution.Artifact, + budget: HarnessEvolution.Artifact, + }) + .strict() + export type Condition = z.infer + + export const Target = z + .object({ + artifact: HarnessEvolution.Artifact, + condition: Condition, + }) + .strict() + export type Target = z.infer + + const PairInput = z + .object({ + family: HarnessContract.InterventionFamily, + index: z.number().int().nonnegative().max(31), + control: Target, + arm: Target, + change: HarnessEvolution.Artifact, + }) + .strict() + + export const Pair = PairInput.extend({ pairID: Hash }).strict() + export type Pair = z.infer + + export const Validator = z + .object({ + name: z.literal("design-replay-interventions"), + version: z.literal(1), + scriptSHA256: Hash, + }) + .strict() + + export const Initialize = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + subject: HarnessEvolution.Subject, + evolutionReceiptID: Hash, + validator: Validator, + pairs: z.array(PairInput).min(3).max(256), + }) + .strict() + export type Initialize = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + + export const Observe = z + .object({ + schemaVersion: z.literal(1), + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + pairID: Hash, + role: Role, + targetSHA256: Hash, + status: Status, + score: z.number().finite().optional(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.status !== "passed" || value.score !== undefined) return + ctx.addIssue({ code: "custom", path: ["score"], message: "A passing intervention observation requires a score" }) + }) + export type Observe = z.input + + export const Plan = z + .object({ + schemaVersion: z.literal(1), + planID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + protocol: HarnessContract.Interventions, + benchmark: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + taskID: z.string().min(1), + metric: z.string().min(1), + direction: z.enum(["maximize", "minimize"]), + }) + .strict(), + subject: HarnessEvolution.Subject, + evolutionReceiptID: Hash, + validator: Validator, + pairs: z.array(Pair).min(3).max(256), + createdAt: z.number().int().positive(), + }) + .strict() + export type Plan = z.infer + + export const Outcome = z + .object({ + schemaVersion: z.literal(1), + outcomeID: Hash, + submissionID: Hash, + pairID: Hash, + role: Role, + targetSHA256: Hash, + status: Status, + score: z.number().finite().optional(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + export type Outcome = z.infer + + const Family = z + .object({ + family: HarnessContract.InterventionFamily, + mode: z.enum(["max_absolute_effect", "min_effect", "max_regression"]), + threshold: z.number().finite(), + pairs: z.number().int().min(3).max(32), + validPairs: z.number().int().nonnegative().max(32), + meanEffect: z.number().finite().optional(), + standardDeviation: z.number().finite().nonnegative().optional(), + standardError: z.number().finite().nonnegative().optional(), + confidence95: z.tuple([z.number().finite(), z.number().finite()]).optional(), + maxAbsoluteEffect: z.number().finite().nonnegative().optional(), + regressions: z.number().int().nonnegative(), + verdict: Status, + }) + .strict() + + export const Receipt = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + planID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + subject: HarnessEvolution.Subject, + evolutionReceiptID: Hash, + families: z.array(Family).min(1).max(HarnessContract.InterventionFamily.options.length), + status: Status, + observedAt: z.number().int().positive(), + assessedAt: z.number().int().positive(), + }) + .strict() + export type Receipt = z.infer + + const slot = (pairID: string, role: z.infer) => `${pairID}:${role}` + const find = (outcomes: Record, pairID: string, role: z.infer) => + Object.values(outcomes).find((outcome) => outcome.pairID === pairID && outcome.role === role) + const submission = ( + sessionID: string, + candidateID: string, + outcome: Pick, + ) => + digest({ + schemaVersion: 1, + sessionID, + candidateID, + pairID: outcome.pairID, + role: outcome.role, + targetSHA256: outcome.targetSHA256, + status: outcome.status, + score: outcome.score, + evidence: outcome.evidence, + evaluatedAt: outcome.evaluatedAt, + }) + + export const State = z + .object({ + schemaVersion: z.literal(1), + plan: Plan, + outcomes: z.record(z.string(), Outcome), + order: z.array(z.string()), + receipt: Receipt.optional(), + }) + .strict() + .superRefine((value, ctx) => { + const plan = structuredClone(value.plan) as Record + delete plan.planID + if (digest(plan) !== value.plan.planID) { + ctx.addIssue({ code: "custom", path: ["plan", "planID"], message: "Intervention plan content hash is invalid" }) + } + try { + validatePlan(value.plan) + } catch (error) { + ctx.addIssue({ + code: "custom", + path: ["plan"], + message: error instanceof Error ? error.message : "Intervention plan is invalid", + }) + } + for (const pair of value.plan.pairs) { + const payload = structuredClone(pair) as Record + delete payload.pairID + if (digest(payload) !== pair.pairID) { + ctx.addIssue({ code: "custom", path: ["plan", "pairs"], message: "Intervention pair hash is invalid" }) + } + try { + validatePair(value.plan.subject, pair) + } catch (error) { + ctx.addIssue({ + code: "custom", + path: ["plan", "pairs"], + message: error instanceof Error ? error.message : "Intervention pair is invalid", + }) + } + } + if (new Set(value.order).size !== value.order.length) { + ctx.addIssue({ code: "custom", path: ["order"], message: "Intervention outcome order must be unique" }) + } + const slots = new Set() + for (const id of value.order) { + const outcome = value.outcomes[id] + if (!outcome) { + ctx.addIssue({ code: "custom", path: ["order"], message: `Intervention outcome ${id} is missing` }) + continue + } + if (outcome.outcomeID !== id) { + ctx.addIssue({ + code: "custom", + path: ["outcomes", id], + message: "Intervention outcome key does not match its ID", + }) + } + const payload = structuredClone(outcome) as Record + delete payload.outcomeID + if (digest(payload) !== id) { + ctx.addIssue({ code: "custom", path: ["outcomes", id], message: "Intervention outcome hash is invalid" }) + } + const key = slot(outcome.pairID, outcome.role) + if (slots.has(key)) { + ctx.addIssue({ + code: "custom", + path: ["outcomes", id], + message: `Intervention outcome slot ${key} is duplicated`, + }) + } + slots.add(key) + if (outcome.submissionID !== submission(value.plan.sessionID, value.plan.subject.id, outcome)) { + ctx.addIssue({ + code: "custom", + path: ["outcomes", id], + message: "Intervention submission derivation is invalid", + }) + } + if (outcome.evaluatedAt < value.plan.createdAt || outcome.recordedAt < outcome.evaluatedAt) { + ctx.addIssue({ code: "custom", path: ["outcomes", id], message: "Intervention outcome timing is invalid" }) + } + const pair = value.plan.pairs.find((item) => item.pairID === outcome.pairID) + const target = pair?.[outcome.role] + if (!target || digest(target) !== outcome.targetSHA256) { + ctx.addIssue({ + code: "custom", + path: ["outcomes", id], + message: "Intervention outcome does not match its frozen target", + }) + } + } + for (const id of Object.keys(value.outcomes)) { + if (value.order.includes(id)) continue + ctx.addIssue({ + code: "custom", + path: ["outcomes", id], + message: "Intervention outcome is absent from journal order", + }) + } + if (!value.receipt) return + const receipt = structuredClone(value.receipt) as Record + delete receipt.receiptID + if (digest(receipt) !== value.receipt.receiptID) { + ctx.addIssue({ code: "custom", path: ["receipt"], message: "Intervention receipt content hash is invalid" }) + } + if ( + value.receipt.planID !== value.plan.planID || + value.receipt.runID !== value.plan.runID || + value.receipt.sessionID !== value.plan.sessionID || + value.receipt.contractFingerprint !== value.plan.contractFingerprint || + !same(value.receipt.subject, value.plan.subject) || + value.receipt.evolutionReceiptID !== value.plan.evolutionReceiptID + ) { + ctx.addIssue({ + code: "custom", + path: ["receipt"], + message: "Intervention receipt does not match its frozen plan", + }) + } + const expected = value.plan.pairs.flatMap((pair) => [slot(pair.pairID, "control"), slot(pair.pairID, "arm")]) + if (expected.length !== slots.size || !expected.every((item) => slots.has(item))) { + ctx.addIssue({ + code: "custom", + path: ["receipt"], + message: "Intervention receipt requires every frozen outcome", + }) + } + const observedAt = Math.max(...Object.values(value.outcomes).map((outcome) => outcome.evaluatedAt)) + const recordedAt = Math.max(...Object.values(value.outcomes).map((outcome) => outcome.recordedAt)) + if (value.receipt.observedAt !== observedAt || value.receipt.assessedAt < recordedAt) { + ctx.addIssue({ code: "custom", path: ["receipt"], message: "Intervention receipt timing is invalid" }) + } + const derived = derive(value.plan, value.outcomes) + if (!same(value.receipt.families, derived.families) || value.receipt.status !== derived.status) { + ctx.addIssue({ code: "custom", path: ["receipt"], message: "Intervention assessment derivation drifted" }) + } + }) + export type State = z.infer + + const root = path.join(Global.Path.data, "harness", "interventions") + const file = (sessionID: string, candidateID: string) => + path.join(root, encodeURIComponent(sessionID), `${Hash.parse(candidateID)}.json`) + + function changed(left: Target, right: Target) { + const fields = ["artifact", "model", "context", "evaluator", "split", "environment", "budget", "seed"] as const + return fields.filter((field) => { + if (field === "artifact") return !same(left.artifact, right.artifact) + if (field === "seed") return left.condition.seed !== right.condition.seed + return !same(left.condition[field], right.condition[field]) + }) + } + + function validatePair(subject: z.infer, input: z.infer) { + const payload = structuredClone(input) as Record + delete payload.pairID + const pair = PairInput.parse(payload) + const changes = changed(pair.control, pair.arm) + const subjectArtifact = subject.artifact + if (pair.family === "replay") { + if (changes.length) throw new Error(`Replay pairs must repeat the exact same target and condition`) + if (!same(pair.arm.artifact, subjectArtifact)) throw new Error(`Replay pairs must evaluate the study subject`) + return + } + if (["retune", "ablation", "repair"].includes(pair.family)) { + if (!same(changes, ["artifact"])) { + throw new Error(`${pair.family} pairs may change only the exact subject artifact`) + } + if (!same(pair.arm.artifact, subjectArtifact)) { + throw new Error(`${pair.family} pair arms must be the study subject`) + } + return + } + const expected = `${pair.family.replace("_transfer", "")}` as (typeof changes)[number] + if (!same(changes, [expected])) throw new Error(`${pair.family} pairs may change only ${expected}`) + if (!same(pair.control.artifact, subjectArtifact) || !same(pair.arm.artifact, subjectArtifact)) { + throw new Error(`${pair.family} pairs must evaluate the same study subject`) + } + } + + function validatePlan(plan: Plan) { + if (plan.validator.scriptSHA256 !== plan.protocol.validatorSHA256) { + throw new Error(`Intervention validator does not match its frozen protocol`) + } + if (new Set(plan.pairs.map((pair) => pair.pairID)).size !== plan.pairs.length) { + throw new Error(`Intervention pairs must be unique`) + } + const families = [...new Set(plan.pairs.map((pair) => pair.family))].toSorted() + if (!same(families, plan.protocol.required)) { + throw new Error(`Intervention pairs must cover exactly the required families`) + } + for (const family of plan.protocol.required) { + const pairs = plan.pairs.filter((pair) => pair.family === family) + if (pairs.length < plan.protocol.minPairs || pairs.length > plan.protocol.maxPairs) { + throw new Error(`Intervention family ${family} violates its frozen pair bounds`) + } + if ( + !same( + pairs.map((pair) => pair.index), + pairs.map((_, index) => index), + ) + ) { + throw new Error(`Intervention family ${family} pair indexes must be contiguous from zero`) + } + } + if (plan.pairs.length > plan.protocol.maxTotalPairs) + throw new Error(`Intervention plan exceeds its total pair limit`) + } + + function match(plan: Plan, contract: HarnessContract.Info, candidateID: string) { + const expected = { + name: contract.benchmark.name, + version: contract.benchmark.version, + taskID: contract.benchmark.taskID, + metric: contract.benchmark.metric, + direction: contract.benchmark.direction, + } + if ( + plan.runID !== contract.runID || + plan.sessionID !== contract.sessionID || + plan.contractFingerprint !== HarnessContract.fingerprint(contract) || + !same(plan.protocol, contract.interventions) || + !same(plan.benchmark, expected) + ) { + throw new Error(`Intervention plan does not match the immutable harness contract`) + } + if (plan.subject.id !== candidateID) throw new Error(`Intervention plan does not match the candidate`) + } + + function critical(pairs: number) { + const values = [ + 0, 0, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, + 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042, 2.04, + ] + return values[Math.min(31, pairs - 1)]! + } + + function statistics(plan: Plan, family: HarnessContract.InterventionFamily, outcomes: Record) { + const pairs = plan.pairs.filter((pair) => pair.family === family) + const effects = pairs.flatMap((pair) => { + const control = find(outcomes, pair.pairID, "control") + const arm = find(outcomes, pair.pairID, "arm") + if (!control || !arm || control.status !== "passed" || arm.status !== "passed") return [] + if (control.score === undefined || arm.score === undefined) return [] + return [plan.benchmark.direction === "maximize" ? arm.score - control.score : control.score - arm.score] + }) + const mean = effects.length ? effects.reduce((sum, effect) => sum + effect, 0) / effects.length : undefined + const variance = + mean === undefined || effects.length < 2 + ? undefined + : effects.reduce((sum, effect) => sum + (effect - mean) ** 2, 0) / (effects.length - 1) + const deviation = variance === undefined ? undefined : Math.sqrt(variance) + const error = deviation === undefined ? undefined : deviation / Math.sqrt(effects.length) + const interval = + mean === undefined || error === undefined + ? undefined + : ([mean - critical(effects.length) * error, mean + critical(effects.length) * error] as const) + const maximum = effects.length ? Math.max(...effects.map(Math.abs)) : undefined + const regressions = effects.filter((effect) => effect < 0).length + const rule = plan.protocol.rules.find((item) => item.family === family)! + const complete = effects.length === pairs.length + const verdict = (() => { + if (!complete) return "failed" as const + if (rule.mode === "max_absolute_effect") + return maximum! <= rule.threshold ? ("passed" as const) : ("failed" as const) + if (rule.mode === "min_effect") { + if (!regressions && mean! > rule.threshold && interval![0] > rule.threshold) return "passed" as const + if (regressions || interval![1] <= rule.threshold) return "failed" as const + return "inconclusive" as const + } + if (effects.some((effect) => effect < -rule.threshold)) return "failed" as const + if (interval![0] >= -rule.threshold) return "passed" as const + return "inconclusive" as const + })() + return Family.parse({ + family, + mode: rule.mode, + threshold: rule.threshold, + pairs: pairs.length, + validPairs: effects.length, + meanEffect: mean, + standardDeviation: deviation, + standardError: error, + confidence95: interval, + maxAbsoluteEffect: maximum, + regressions, + verdict, + }) + } + + function derive(plan: Plan, outcomes: Record) { + const families = plan.protocol.required.map((family) => statistics(plan, family, outcomes)) + const status = families.some((family) => family.verdict === "failed") + ? ("failed" as const) + : families.some((family) => family.verdict === "inconclusive") + ? ("inconclusive" as const) + : ("passed" as const) + return { families, status } + } + + export async function initialize(input: Initialize, contract: HarnessContract.Info) { + const value = Initialize.parse(input) + const bound = HarnessContract.Info.parse(contract) + const protocol = bound.interventions + if (!protocol) throw new Error(`No intervention protocol is bound to session ${value.sessionID}`) + if (bound.sessionID !== value.sessionID || bound.runID !== value.runID) { + throw new Error(`Intervention plan does not match the bound harness run`) + } + if (value.validator.scriptSHA256 !== protocol.validatorSHA256) { + throw new Error(`Intervention validator does not match the immutable harness contract`) + } + const frozenAt = Date.now() + const trace = await HarnessEvolution.assert({ + contract: bound, + receiptID: value.evolutionReceiptID, + candidateID: value.subject.id, + evaluatedAt: frozenAt, + recordedAt: frozenAt, + }) + if (!same(trace.subject.artifact, value.subject.artifact)) { + throw new Error(`Intervention subject artifact does not match its evolution receipt`) + } + const search = await import("./search").then((module) => module.HarnessSearch.read(value.sessionID)) + const candidate = search.candidates[value.subject.id] + if (!candidate || !same(candidate.artifact, value.subject.artifact)) { + throw new Error(`Intervention subject does not match a candidate in the bound search`) + } + const evaluations = await import("./evaluation").then((module) => module.HarnessEvaluation.list(value.sessionID)) + if ( + evaluations.some( + (evaluation) => + evaluation.subject?.type === "candidate" && + evaluation.subject.id === value.subject.id && + evaluation.fidelity?.final !== false, + ) + ) { + throw new Error(`Intervention plan must be frozen before the candidate's final evaluation`) + } + const pairs = value.pairs + .map((pair) => { + validatePair(value.subject, pair) + return Pair.parse({ ...pair, pairID: digest(pair) }) + }) + .toSorted((left, right) => left.family.localeCompare(right.family) || left.index - right.index) + const payload = { + schemaVersion: 1 as const, + runID: value.runID, + sessionID: value.sessionID, + contractFingerprint: HarnessContract.fingerprint(bound), + protocol, + benchmark: { + name: bound.benchmark.name, + version: bound.benchmark.version, + taskID: bound.benchmark.taskID, + metric: bound.benchmark.metric!, + direction: bound.benchmark.direction as "maximize" | "minimize", + }, + subject: value.subject, + evolutionReceiptID: value.evolutionReceiptID, + validator: value.validator, + pairs, + createdAt: frozenAt, + } + const plan = Plan.parse({ ...payload, planID: digest(payload) }) + validatePlan(plan) + const expected = State.parse({ schemaVersion: 1, plan, outcomes: {}, order: [] }) + await JsonStore.update(file(value.sessionID, value.subject.id), (data) => { + if (!Object.keys(data).length) return expected + const current = State.parse(data) + if (current.plan.planID === plan.planID) return current + throw new Error(`Intervention plan is immutable once initialized`) + }) + const current = await read(value.sessionID, value.subject.id) + if (!current) throw new Error(`Intervention plan was corrupt after initialization`) + match(current.plan, bound, value.subject.id) + return current + } + + export async function read(sessionID: string, candidateID: string) { + const data = await JsonStore.read(file(sessionID, candidateID)) + const parsed = State.safeParse(data) + return parsed.success ? parsed.data : null + } + + export async function status(sessionID: string, candidateID: string, contract: HarnessContract.Info) { + const state = await read(sessionID, candidateID) + if (!state) return null + match(state.plan, contract, candidateID) + return state + } + + export async function observe(candidateID: string, input: Observe, contract: HarnessContract.Info) { + const value = Observe.parse(input) + const bound = HarnessContract.Info.parse(contract) + if (bound.sessionID !== value.sessionID) throw new Error(`Intervention observation belongs to another session`) + const target = file(value.sessionID, candidateID) + const output: { value?: Outcome } = {} + await JsonStore.update(target, (data) => { + const current = State.parse(data) + match(current.plan, bound, candidateID) + if (current.receipt) throw new Error(`Intervention study is closed after assessment`) + const pair = current.plan.pairs.find((item) => item.pairID === value.pairID) + if (!pair) throw new Error(`Intervention pair ${value.pairID} is not in the frozen plan`) + if (digest(pair[value.role]) !== value.targetSHA256) { + throw new Error(`Intervention observation target does not match the frozen pair`) + } + if (value.evaluatedAt < current.plan.createdAt) { + throw new Error(`Intervention observation predates its frozen plan`) + } + if (value.evaluatedAt > Date.now()) throw new Error(`Intervention observation is future-dated`) + const evidence = value.evidence.toSorted() + const submissionID = submission(value.sessionID, candidateID, { + pairID: value.pairID, + role: value.role, + targetSHA256: value.targetSHA256, + status: value.status, + score: value.score, + evidence, + evaluatedAt: value.evaluatedAt, + }) + const key = slot(value.pairID, value.role) + const prior = Object.values(current.outcomes).find((outcome) => slot(outcome.pairID, outcome.role) === key) + if (prior) { + if (prior.submissionID !== submissionID) + throw new Error(`Intervention outcome ${key} is immutable once recorded`) + output.value = prior + return current + } + const recordedAt = Date.now() + const payload = { + schemaVersion: 1 as const, + submissionID, + pairID: value.pairID, + role: value.role, + targetSHA256: value.targetSHA256, + status: value.status, + score: value.score, + evidence, + evaluatedAt: value.evaluatedAt, + recordedAt, + } + const outcome = Outcome.parse({ ...payload, outcomeID: digest(payload) }) + output.value = outcome + return State.parse({ + ...current, + outcomes: { ...current.outcomes, [outcome.outcomeID]: outcome }, + order: [...current.order, outcome.outcomeID], + }) + }) + if (!output.value) throw new Error(`Intervention outcome was not durable after recording`) + return output.value + } + + export async function assess(sessionID: string, candidateID: string, contract: HarnessContract.Info) { + const bound = HarnessContract.Info.parse(contract) + if (bound.sessionID !== sessionID) throw new Error(`Intervention assessment belongs to another session`) + const target = file(sessionID, candidateID) + const output: { value?: Receipt } = {} + await JsonStore.update(target, (data) => { + const current = State.parse(data) + match(current.plan, bound, candidateID) + if (current.receipt) { + output.value = current.receipt + return current + } + const expected = current.plan.pairs.flatMap((pair) => [slot(pair.pairID, "control"), slot(pair.pairID, "arm")]) + const observed = Object.values(current.outcomes).map((outcome) => slot(outcome.pairID, outcome.role)) + if (!expected.every((item) => observed.includes(item)) || observed.length !== expected.length) { + throw new Error(`Intervention assessment requires every frozen pair outcome`) + } + const derived = derive(current.plan, current.outcomes) + const observedAt = Math.max(...Object.values(current.outcomes).map((outcome) => outcome.evaluatedAt)) + const assessedAt = Date.now() + if (assessedAt < Math.max(...Object.values(current.outcomes).map((outcome) => outcome.recordedAt))) { + throw new Error(`Intervention assessment predates one of its observations`) + } + const payload = { + schemaVersion: 1 as const, + planID: current.plan.planID, + runID: current.plan.runID, + sessionID: current.plan.sessionID, + contractFingerprint: current.plan.contractFingerprint, + subject: current.plan.subject, + evolutionReceiptID: current.plan.evolutionReceiptID, + families: derived.families, + status: derived.status, + observedAt, + assessedAt, + } + const receipt = Receipt.parse({ ...payload, receiptID: digest(payload) }) + output.value = receipt + return State.parse({ ...current, receipt }) + }) + if (!output.value) throw new Error(`Intervention receipt was not durable after assessment`) + return output.value + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + candidateID: string + evolutionReceiptID: string + requirePassed: boolean + evaluatedAt: number + recordedAt: number + }) { + const state = await read(input.contract.sessionID, input.candidateID) + const receipt = state?.receipt + if (!receipt || receipt.receiptID !== Hash.parse(input.receiptID)) { + throw new Error(`Unknown or corrupt controlled intervention receipt ${input.receiptID}`) + } + if (receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract)) { + throw new Error(`Intervention receipt does not match the immutable harness contract`) + } + if (receipt.subject.id !== input.candidateID) { + throw new Error(`Intervention receipt does not match the evaluated candidate`) + } + match(state.plan, input.contract, input.candidateID) + if (receipt.evolutionReceiptID !== Hash.parse(input.evolutionReceiptID)) { + throw new Error(`Intervention receipt does not match the evaluation's evolution receipt`) + } + if (receipt.observedAt > input.evaluatedAt) { + throw new Error(`Final benchmark evaluation predates its controlled intervention observations`) + } + if (receipt.assessedAt > input.recordedAt) { + throw new Error(`Final benchmark evaluation was recorded before its intervention assessment`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A passing final evaluation requires a passing controlled intervention receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/judge.ts b/backend/cli/src/session/harness/judge.ts new file mode 100644 index 00000000..641b5087 --- /dev/null +++ b/backend/cli/src/session/harness/judge.ts @@ -0,0 +1,265 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessJudge { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const Decision = z.enum(["accept", "reject", "abstain"]) + export type Decision = z.infer + + export const Case = z + .object({ + id: z.string().min(1).max(240), + commitment: Hash, + kind: z.enum(["clean", "fault"]), + fault: HarnessContract.EvaluatorFault.optional(), + decision: Decision, + failureProbability: z.number().min(0).max(1), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + .superRefine((value, ctx) => { + if (value.kind === "fault" && !value.fault) { + ctx.addIssue({ code: "custom", path: ["fault"], message: "A fault case must name its hidden fault class" }) + } + if (value.kind === "clean" && value.fault) { + ctx.addIssue({ code: "custom", path: ["fault"], message: "A clean case cannot name a fault class" }) + } + }) + export type Case = z.infer + + export function commitment(input: Array>) { + const manifest = input + .map((item) => ({ id: item.id, commitment: item.commitment, kind: item.kind, fault: item.fault })) + .toSorted((left, right) => left.id.localeCompare(right.id)) + return digest(manifest) + } + + export const Submit = z + .object({ + sessionID: z.string().min(1).max(240), + auditorToken: Token, + cases: z.array(Case).min(3).max(2_048), + }) + .strict() + export type Submit = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + auditorToken: Token, + }) + .strict() + export type Access = z.infer + + const Rate = z + .object({ + cases: z.number().int().positive(), + detected: z.number().int().nonnegative(), + recall: z.number().min(0).max(1), + }) + .strict() + + export const Receipt = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("evaluator-audit-receipt-v1"), + receiptID: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1), + evaluator: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + source: z.enum(["benchmark", "gate", "human", "external"]), + }) + .strict(), + auditor: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + source: z.enum(["benchmark", "gate", "human", "external"]), + }) + .strict(), + suite: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + commitmentSHA256: Hash, + }) + .strict(), + cases: z.array(Case).min(3).max(2_048), + metrics: z + .object({ + cases: z.number().int().min(3), + cleanCases: z.number().int().positive(), + faultCases: z.number().int().positive(), + truePositive: z.number().int().nonnegative(), + falseNegative: z.number().int().nonnegative(), + trueNegative: z.number().int().nonnegative(), + falsePositive: z.number().int().nonnegative(), + sensitivity: z.number().min(0).max(1), + specificity: z.number().min(0).max(1), + balancedAccuracy: z.number().min(0).max(1), + brierScore: z.number().min(0).max(1), + perFault: z.partialRecord(HarnessContract.EvaluatorFault, Rate), + }) + .strict(), + status: z.enum(["passed", "failed"]), + failures: z.array(z.string().min(1).max(500)).max(64), + recordedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + delete stable.recordedAt + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Evaluator audit receipt content hash is invalid" }) + }) + export type Receipt = z.infer + + const root = path.join(Global.Path.data, "harness", "judges") + const file = (receiptID: string) => path.join(root, `${receiptID}.json`) + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const protocol = contract.evaluatorAudit + if (!protocol) throw new Error(`Harness contract does not require evaluator qualification`) + if (value.sessionID !== contract.sessionID) throw new Error(`Evaluator audit session does not match its contract`) + const cases = value.cases.toSorted((left, right) => left.id.localeCompare(right.id)) + if (new Set(cases.map((item) => item.id)).size !== cases.length) { + throw new Error(`Evaluator audit case identities must be unique`) + } + if (new Set(cases.map((item) => item.commitment)).size !== cases.length) { + throw new Error(`Evaluator audit case commitments must be unique`) + } + if (commitment(cases) !== protocol.suite.commitmentSHA256) { + throw new Error(`Evaluator audit cases do not match the precommitted hidden suite`) + } + const clean = cases.filter((item) => item.kind === "clean") + const faults = cases.filter((item) => item.kind === "fault") + if (clean.length < protocol.minCleanCases) { + throw new Error(`Evaluator audit requires at least ${protocol.minCleanCases} clean cases`) + } + for (const fault of protocol.requiredFaults) { + const count = faults.filter((item) => item.fault === fault).length + if (count >= protocol.minCasesPerFault) continue + throw new Error(`Evaluator audit requires at least ${protocol.minCasesPerFault} ${fault} cases`) + } + const truePositive = faults.filter((item) => item.decision === "reject").length + const falseNegative = faults.length - truePositive + const trueNegative = clean.filter((item) => item.decision === "accept").length + const falsePositive = clean.length - trueNegative + const sensitivity = truePositive / faults.length + const specificity = trueNegative / clean.length + const balancedAccuracy = (sensitivity + specificity) / 2 + const brierScore = + cases.reduce((sum, item) => sum + (item.failureProbability - (item.kind === "fault" ? 1 : 0)) ** 2, 0) / + cases.length + const perFault = Object.fromEntries( + protocol.requiredFaults.map((fault) => { + const items = faults.filter((item) => item.fault === fault) + const detected = items.filter((item) => item.decision === "reject").length + return [fault, { cases: items.length, detected, recall: detected / items.length }] + }), + ) + const failures = [ + ...(sensitivity < protocol.minSensitivity + ? [`sensitivity ${sensitivity} is below ${protocol.minSensitivity}`] + : []), + ...(specificity < protocol.minSpecificity + ? [`specificity ${specificity} is below ${protocol.minSpecificity}`] + : []), + ...(balancedAccuracy < protocol.minBalancedAccuracy + ? [`balanced accuracy ${balancedAccuracy} is below ${protocol.minBalancedAccuracy}`] + : []), + ...(brierScore > protocol.maxBrierScore ? [`Brier score ${brierScore} exceeds ${protocol.maxBrierScore}`] : []), + ...Object.entries(perFault).flatMap(([fault, rate]) => + rate.recall < protocol.minFaultRecall + ? [`${fault} recall ${rate.recall} is below ${protocol.minFaultRecall}`] + : [], + ), + ] + const stable = { + schemaVersion: 1 as const, + protocolVersion: "evaluator-audit-receipt-v1" as const, + protocolSHA256: digest(protocol), + sourceSessionID: contract.sessionID, + evaluator: { + name: contract.benchmark.evaluator, + version: contract.benchmark.evaluatorVersion!, + source: contract.benchmark.evaluatorSource!, + }, + auditor: protocol.auditor, + suite: protocol.suite, + cases, + metrics: { + cases: cases.length, + cleanCases: clean.length, + faultCases: faults.length, + truePositive, + falseNegative, + trueNegative, + falsePositive, + sensitivity, + specificity, + balancedAccuracy, + brierScore, + perFault, + }, + status: failures.length ? ("failed" as const) : ("passed" as const), + failures, + } + const receipt = Receipt.parse({ ...stable, receiptID: digest(stable), recordedAt: Date.now() }) + await JsonStore.update(file(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Evaluator audit receipt is immutable once recorded`) + }) + const stored = await read(receipt.receiptID) + if (!stored) throw new Error(`Evaluator audit receipt was not durable after recording`) + return stored + } + + export async function read(receiptID: string) { + const data = await JsonStore.read(file(Hash.parse(receiptID))) + const parsed = Receipt.safeParse(data) + return parsed.success ? parsed.data : null + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + recordedAt: number + requirePassed: boolean + }) { + const receipt = await read(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt evaluator audit receipt ${input.receiptID}`) + const protocol = input.contract.evaluatorAudit + if (!protocol) throw new Error(`Evaluation cites an auditor receipt without a bound evaluator audit protocol`) + if (receipt.protocolSHA256 !== digest(protocol)) { + throw new Error(`Evaluator audit receipt does not match the bound qualification protocol`) + } + if ( + receipt.evaluator.name !== input.contract.benchmark.evaluator || + receipt.evaluator.version !== input.contract.benchmark.evaluatorVersion || + receipt.evaluator.source !== input.contract.benchmark.evaluatorSource + ) { + throw new Error(`Evaluator audit receipt qualifies a different evaluator`) + } + if (receipt.recordedAt > input.recordedAt) { + throw new Error(`Evaluation predates its evaluator audit receipt`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A passing final evaluation requires a passing evaluator audit receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/memory.ts b/backend/cli/src/session/harness/memory.ts new file mode 100644 index 00000000..361e451f --- /dev/null +++ b/backend/cli/src/session/harness/memory.ts @@ -0,0 +1,296 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" +import { HarnessSearch } from "./search" + +export namespace HarnessMemory { + export const Stage = z.enum(["planning", "implementation", "evaluation", "debugging", "verification"]) + export type Stage = z.infer + + export const Entry = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + benchmark: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + taskID: z.string().min(1), + metric: z.string().optional(), + }) + .strict(), + source: z + .object({ + runID: z.string().min(1), + candidateID: z.string().regex(/^[a-f0-9]{64}$/), + evaluator: z.string().min(1), + }) + .strict(), + stage: Stage, + outcome: z.enum(["passed", "failed", "inconclusive"]), + objective: z.string().min(1).max(1_000), + proposal: z.string().min(1).max(1_000), + feedback: z.string().max(1_000).optional(), + score: z.number().finite().optional(), + metrics: z.record(z.string(), z.number().finite()), + evidence: z.array(z.string().min(1).max(500)).max(12), + usage: z + .object({ wallTimeMs: z.number().nonnegative().optional(), costUSD: z.number().nonnegative().optional() }) + .strict() + .optional(), + branch: z.string().min(1).max(120), + generation: z.number().int().nonnegative(), + artifact: HarnessSearch.Artifact, + createdAt: z.number().int().positive(), + }) + .strict() + export type Entry = z.infer + + const State = z + .object({ + schemaVersion: z.literal(1), + scope: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + taskID: z.string().min(1), + evaluator: z.string().min(1), + semanticAuditSHA256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + synthesisSHA256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + autonomySHA256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + formalProofSHA256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + replicationSHA256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + metaHarnessSHA256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + confirmationSHA256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }) + .strict(), + entries: z.record(z.string(), Entry), + revision: z.number().int().nonnegative(), + }) + .strict() + export type State = z.infer + + export type Hit = { entry: Entry; relevance: number; matched: string[] } + + const root = path.join(Global.Path.data, "harness", "retrospectives") + const digest = (input: string) => new Bun.CryptoHasher("sha256").update(input).digest("hex") + const semantic = (contract: HarnessContract.Info) => + contract.semanticAudit ? digest(JSON.stringify(contract.semanticAudit)) : undefined + const replication = (contract: HarnessContract.Info) => + contract.replication ? digest(JSON.stringify(contract.replication)) : undefined + const synthesis = (contract: HarnessContract.Info) => + contract.synthesis ? digest(JSON.stringify(contract.synthesis)) : undefined + const autonomy = (contract: HarnessContract.Info) => + contract.autonomy ? digest(JSON.stringify(contract.autonomy)) : undefined + const formal = (contract: HarnessContract.Info) => + contract.formalProof ? digest(JSON.stringify(contract.formalProof)) : undefined + const meta = (contract: HarnessContract.Info) => + contract.metaHarness ? digest(JSON.stringify(contract.metaHarness)) : undefined + const confirmation = (contract: HarnessContract.Info) => + contract.confirmation ? digest(JSON.stringify(contract.confirmation)) : undefined + const key = (contract: HarnessContract.Info) => { + const base = `${contract.benchmark.name}\0${contract.benchmark.version}\0${contract.benchmark.taskID}\0${contract.benchmark.evaluator}` + const scope = semantic(contract) + const repeat = replication(contract) + const prior = repeat ? `${base}\0${scope ?? "no-semantic-audit"}\0${repeat}` : scope ? `${base}\0${scope}` : base + const factuality = synthesis(contract) + const scientific = factuality ? `${prior}\0${factuality}` : prior + const provenance = autonomy(contract) + const accountable = provenance ? `${scientific}\0${provenance}` : scientific + const proof = formal(contract) + const verified = proof ? `${accountable}\0${proof}` : accountable + const harness = meta(contract) + const qualified = harness ? `${verified}\0${harness}` : verified + const sealed = confirmation(contract) + return digest(sealed ? `${qualified}\0${sealed}` : qualified) + } + const file = (contract: HarnessContract.Info) => path.join(root, `${key(contract)}.json`) + const clip = (value: string, max = 1_000) => + value + .replace(/[\u0000-\u001f\u007f\u200b-\u200f\u202a-\u202e\u2060-\u2064\ufeff]/g, " ") + .trim() + .slice(0, max) + const stopwords = new Set(["and", "are", "for", "from", "into", "that", "the", "their", "this", "use", "with"]) + const terms = (value: string) => + new Set( + (value.toLowerCase().match(/[a-z0-9][a-z0-9_-]{1,}/g) ?? []).filter( + (word) => word.length > 2 && !stopwords.has(word), + ), + ) + const escape = (value: string) => + value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """) + const safe = (value: string, max = 800) => escape(value).slice(0, max) + + function empty(contract: HarnessContract.Info): State { + const scope = semantic(contract) + const factuality = synthesis(contract) + const provenance = autonomy(contract) + const proof = formal(contract) + const repeat = replication(contract) + const harness = meta(contract) + const sealed = confirmation(contract) + return { + schemaVersion: 1, + scope: { + name: contract.benchmark.name, + version: contract.benchmark.version, + taskID: contract.benchmark.taskID, + evaluator: contract.benchmark.evaluator, + ...(scope ? { semanticAuditSHA256: scope } : {}), + ...(factuality ? { synthesisSHA256: factuality } : {}), + ...(provenance ? { autonomySHA256: provenance } : {}), + ...(proof ? { formalProofSHA256: proof } : {}), + ...(repeat ? { replicationSHA256: repeat } : {}), + ...(harness ? { metaHarnessSHA256: harness } : {}), + ...(sealed ? { confirmationSHA256: sealed } : {}), + }, + entries: {}, + revision: 0, + } + } + + async function state(contract: HarnessContract.Info) { + const data = await JsonStore.read(file(contract)) + const parsed = State.safeParse(data) + return parsed.success ? parsed.data : empty(contract) + } + + export async function capture(input: { sessionID: string; candidateID: string; stage: Stage }) { + const contract = await HarnessContract.read(input.sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${input.sessionID}`) + const search = await HarnessSearch.read(input.sessionID) + if (search.runID !== contract.runID) throw new Error(`Candidate search belongs to a different harness run`) + const candidate = search.candidates[input.candidateID] + if (!candidate) throw new Error(`Unknown candidate ${input.candidateID}`) + if (candidate.result?.source !== "verified") { + throw new Error(`Only externally evaluated candidates may enter retrospective memory`) + } + const id = digest(`${contract.runID}\0${candidate.id}`) + const entry = Entry.parse({ + id, + benchmark: { + name: contract.benchmark.name, + version: contract.benchmark.version, + taskID: contract.benchmark.taskID, + metric: contract.benchmark.metric, + }, + source: { + runID: contract.runID, + candidateID: candidate.id, + evaluator: candidate.result.evaluator, + }, + stage: input.stage, + outcome: candidate.result.status, + objective: clip(contract.objective), + proposal: clip(candidate.proposal), + feedback: candidate.result.feedback ? clip(candidate.result.feedback) : undefined, + score: candidate.result.score, + metrics: Object.fromEntries(Object.entries(candidate.result.metrics).slice(0, 32)), + evidence: candidate.result.evidence.slice(0, 12).map((item) => clip(item, 500)), + usage: candidate.result.usage, + branch: candidate.branch, + generation: candidate.generation, + artifact: candidate.artifact, + createdAt: candidate.result.evaluatedAt, + }) + await JsonStore.update(file(contract), (data) => { + const current = Object.keys(data).length ? State.parse(data) : empty(contract) + if (current.entries[id]) return current + return { + ...current, + entries: { ...current.entries, [id]: entry }, + revision: current.revision + 1, + } + }) + return (await state(contract)).entries[id]! + } + + export async function retrieve(input: { sessionID: string; query: string; stage?: Stage; limit?: number }) { + const contract = await HarnessContract.read(input.sessionID) + if (!contract) return [] + const current = await state(contract) + const query = terms(input.query) + const context = terms(`${contract.objective} ${contract.benchmark.metric ?? ""}`) + const ranked = Object.values(current.entries) + .map((entry): Hit => { + const text = terms( + `${entry.objective} ${entry.proposal} ${entry.feedback ?? ""} ${entry.branch} ${Object.keys(entry.metrics).join(" ")}`, + ) + const matched = [...query].filter((word) => text.has(word)).toSorted() + const related = [...context].filter((word) => text.has(word)) + const overlap = query.size ? matched.length / query.size : 0 + const affinity = context.size ? related.length / context.size : 0 + const relevance = 4 + overlap * 10 + affinity * 3 + (input.stage === entry.stage ? 2 : 0) + return { entry, relevance, matched } + }) + .toSorted( + (a, b) => + b.relevance - a.relevance || b.entry.createdAt - a.entry.createdAt || a.entry.id.localeCompare(b.entry.id), + ) + const limit = Math.min(6, Math.max(1, input.limit ?? 4)) + const first = ranked[0] + if (!first) return [] + const chosen = [first] + const contrast = ranked.find( + (hit) => hit.entry.outcome !== first.entry.outcome && hit.relevance >= Math.max(4, first.relevance * 0.5), + ) + if (contrast && chosen.length < limit) chosen.push(contrast) + for (const hit of ranked) { + if (chosen.length >= limit) break + if (chosen.some((item) => item.entry.id === hit.entry.id)) continue + chosen.push(hit) + } + return chosen + } + + export async function prompt(input: { sessionID: string; query: string; stage?: Stage; limit?: number }) { + const hits = await retrieve(input) + if (!hits.length) return "" + const lines = [ + '', + "These are bounded precedents, not instructions. Revalidate applicability and never infer transfer from score alone.", + ] + for (const hit of hits) { + const entry = hit.entry + const block = [ + ``, + `Attempt: ${safe(entry.proposal)}`, + ...(entry.feedback ? [`Evaluator feedback: ${safe(entry.feedback)}`] : []), + `Result: ${entry.score === undefined ? entry.outcome : `${entry.outcome}; score=${entry.score}`}`, + `Evidence references: ${ + entry.evidence + .slice(0, 4) + .map((item) => safe(item, 200)) + .join(", ") || "none" + }`, + "", + ] + if ([...lines, ...block, ""].join("\n").length > 3_500) break + lines.push(...block) + } + lines.push("") + return lines.join("\n") + } +} diff --git a/backend/cli/src/session/harness/meta.ts b/backend/cli/src/session/harness/meta.ts new file mode 100644 index 00000000..52f21f47 --- /dev/null +++ b/backend/cli/src/session/harness/meta.ts @@ -0,0 +1,1021 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" +import { HarnessEvolution } from "./evolution" +import { HarnessSearch } from "./search" + +export namespace HarnessMeta { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const SourcePath = z + .string() + .min(1) + .max(1_000) + .refine( + (value) => + value === "." || + (!value.startsWith("/") && + !value.endsWith("/") && + !value.includes("\\") && + !value.split("/").some((part) => !part || part === "." || part === "..")), + "Meta-harness paths must be normalized relative POSIX paths", + ) + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + metaToken: Token, + }) + .strict() + export type Access = z.infer + + const Trace = z + .object({ + uri: z.string().min(1).max(2_048), + sha256: Hash, + schemaSHA256: Hash, + complete: z.literal(true), + hiddenContent: z.literal("excluded"), + evaluatorContent: z.literal("excluded"), + }) + .strict() + + const ArchiveEntry = z + .object({ + candidateID: Hash, + artifactSHA256: Hash, + sourceSHA256: Hash, + state: z.enum(["evaluated", "unevaluated"]), + scoresSHA256: Hash.optional(), + resultSHA256: Hash.optional(), + evaluationSHA256: Hash.optional(), + trace: Trace.optional(), + }) + .strict() + .superRefine((value, ctx) => { + const evidence = [value.scoresSHA256, value.resultSHA256, value.evaluationSHA256, value.trace] + if (value.state === "evaluated" && evidence.some((item) => item === undefined)) { + ctx.addIssue({ + code: "custom", + path: ["state"], + message: "Evaluated archive entries require scores and trace evidence", + }) + } + if (value.state === "unevaluated" && evidence.some((item) => item !== undefined)) { + ctx.addIssue({ + code: "custom", + path: ["state"], + message: "Unevaluated archive entries cannot claim result evidence", + }) + } + }) + + const Archive = z + .object({ + uri: z.string().min(1).max(2_048), + sha256: Hash, + schemaSHA256: Hash, + indexSHA256: Hash, + contents: z.literal("full-source-scores-traces"), + query: z.literal("filesystem"), + complete: z.literal(true), + hiddenContent: z.literal("excluded"), + evaluatorContent: z.literal("excluded"), + entries: z + .array(ArchiveEntry) + .min(1) + .max(10_000) + .refine( + (items) => new Set(items.map((item) => item.candidateID)).size === items.length, + "Archive candidates must be unique", + ) + .refine( + (items) => items.every((item, index) => !index || items[index - 1]!.candidateID < item.candidateID), + "Archive candidates must be sorted", + ), + }) + .strict() + .superRefine((value, ctx) => { + if (digest(value.entries) !== value.indexSHA256) { + ctx.addIssue({ code: "custom", path: ["indexSHA256"], message: "Archive index hash is invalid" }) + } + const stable = structuredClone(value) as Record + delete stable.sha256 + if (digest(stable) === value.sha256) return + ctx.addIssue({ code: "custom", path: ["sha256"], message: "Archive content hash is invalid" }) + }) + + const Change = z + .object({ + action: z.enum(["create", "update", "delete", "rollback"]), + component: HarnessContract.MetaComponent, + path: SourcePath, + beforeSHA256: Hash.optional(), + afterSHA256: Hash.optional(), + reason: z.string().min(1).max(4_000), + }) + .strict() + .superRefine((value, ctx) => { + if (value.action === "create" && (value.beforeSHA256 || !value.afterSHA256)) { + ctx.addIssue({ code: "custom", path: ["action"], message: "Create changes require only an after hash" }) + } + if (value.action === "delete" && (!value.beforeSHA256 || value.afterSHA256)) { + ctx.addIssue({ code: "custom", path: ["action"], message: "Delete changes require only a before hash" }) + } + if (["update", "rollback"].includes(value.action) && (!value.beforeSHA256 || !value.afterSHA256)) { + ctx.addIssue({ + code: "custom", + path: ["action"], + message: "Update and rollback changes require before and after hashes", + }) + } + if (value.beforeSHA256 && value.afterSHA256 && value.beforeSHA256 === value.afterSHA256) { + ctx.addIssue({ code: "custom", path: ["afterSHA256"], message: "A refinement change must alter content" }) + } + }) + + const Citation = z + .object({ + candidateID: Hash, + traceSHA256: Hash, + messageIndex: z.number().int().nonnegative(), + excerptSHA256: Hash, + }) + .strict() + + const Prediction = z + .object({ + modelID: z.string().min(1).max(240), + taskID: z.string().min(1).max(200), + expected: z.enum(["fail_to_pass", "remain_pass"]), + }) + .strict() + + const Refinement = z + .object({ + revision: z.number().int().positive(), + scope: z.literal("session"), + parentSnapshotSHA256: Hash, + snapshotSHA256: Hash, + trigger: z.string().min(1).max(4_000), + diagnosis: z + .object({ + kind: z.enum(["implementation", "fundamental", "inconclusive"]), + rationale: z.string().min(1).max(8_000), + }) + .strict(), + rootCause: z.string().min(1).max(8_000), + expectedOutcome: z.string().min(1).max(8_000), + changes: z + .array(Change) + .min(1) + .max(128) + .refine( + (items) => new Set(items.map((item) => item.path)).size === items.length, + "Refinement paths must be unique", + ) + .refine( + (items) => items.every((item, index) => !index || items[index - 1]!.path < item.path), + "Refinement paths must be sorted", + ), + evidence: z + .array(Citation) + .min(1) + .max(256) + .refine( + (items) => + new Set( + items.map( + (item) => `${item.candidateID}\0${item.traceSHA256}\0${item.messageIndex}\0${item.excerptSHA256}`, + ), + ).size === items.length, + "Refinement citations must be unique", + ) + .refine( + (items) => + items.every( + (item, index) => + !index || + `${items[index - 1]!.candidateID}\0${items[index - 1]!.traceSHA256}\0${items[index - 1]!.messageIndex}\0${items[index - 1]!.excerptSHA256}` < + `${item.candidateID}\0${item.traceSHA256}\0${item.messageIndex}\0${item.excerptSHA256}`, + ), + "Refinement citations must be sorted", + ), + predictions: z + .array(Prediction) + .min(1) + .max(256) + .refine( + (items) => new Set(items.map((item) => `${item.modelID}\0${item.taskID}`)).size === items.length, + "Refinement predictions must be unique", + ) + .refine( + (items) => + items.every( + (item, index) => + !index || + `${items[index - 1]!.modelID}\0${items[index - 1]!.taskID}` < `${item.modelID}\0${item.taskID}`, + ), + "Refinement predictions must be sorted", + ), + }) + .strict() + + export const PhaseID = z.enum(["loaded", "midpoint", "pre_final", "final_validation"]) + export type PhaseID = z.infer + + const Counts = z + .object({ + followed: z.number().int().nonnegative(), + violatedCommission: z.number().int().nonnegative(), + violatedOmission: z.number().int().nonnegative(), + requiredUnobserved: z.number().int().nonnegative(), + notApplicable: z.number().int().nonnegative(), + insufficientEvidence: z.number().int().nonnegative(), + }) + .strict() + + const Phase = Counts.extend({ phase: PhaseID }).strict() + + const CellBase = z + .object({ + split: z.enum(["search", "held_out"]), + modelID: z.string().min(1).max(240), + modelCommitment: Hash, + taskID: z.string().min(1).max(200), + taskCommitment: Hash, + outcome: z.enum(["completed", "failed", "inconclusive"]), + score: z.number().finite().optional(), + passed: z.boolean().optional(), + contextTokens: z.number().int().nonnegative(), + outputSHA256: Hash, + trace: Trace, + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(64), + }) + .strict() + + const BaselineCell = CellBase.extend({ role: z.literal("baseline") }) + .strict() + .superRefine((value, ctx) => validateOutcome(value, ctx)) + + const CandidateCell = CellBase.extend({ + role: z.literal("candidate"), + loaded: z.boolean(), + phases: z + .array(Phase) + .max(PhaseID.options.length) + .refine( + (items) => new Set(items.map((item) => item.phase)).size === items.length, + "Adherence phases must be unique", + ) + .refine( + (items) => + items.every( + (item, index) => + !index || PhaseID.options.indexOf(items[index - 1]!.phase) < PhaseID.options.indexOf(item.phase), + ), + "Adherence phases must use canonical order", + ), + }) + .strict() + .superRefine((value, ctx) => validateOutcome(value, ctx)) + + function validateOutcome(value: z.infer, ctx: z.RefinementCtx) { + if (value.outcome === "completed" && (value.score === undefined || value.passed === undefined)) { + ctx.addIssue({ code: "custom", path: ["outcome"], message: "Completed cells require a score and pass verdict" }) + } + if (value.outcome !== "completed" && (value.score !== undefined || value.passed !== undefined)) { + ctx.addIssue({ code: "custom", path: ["outcome"], message: "Incomplete cells cannot publish partial scores" }) + } + } + + export const Cell = z.discriminatedUnion("role", [BaselineCell, CandidateCell]) + export type Cell = z.infer + + export const Submit = z + .object({ + schemaVersion: z.literal(1), + sessionID: z.string().min(1).max(240), + metaToken: Token, + selectionID: Hash, + candidateArtifactSHA256: Hash, + candidateManifestSHA256: Hash, + protectedManifestSHA256: Hash, + validatorSHA256: Hash, + archive: Archive, + refinements: z.array(Refinement).min(1).max(128), + cells: z + .array(Cell) + .min(4) + .max(65_536) + .refine( + (items) => + new Set(items.map((item) => `${item.split}\0${item.modelID}\0${item.taskID}\0${item.role}`)).size === + items.length, + "Qualification cells must be unique", + ) + .refine( + (items) => + items.every( + (item, index) => + !index || + `${items[index - 1]!.split}\0${items[index - 1]!.modelID}\0${items[index - 1]!.taskID}\0${items[index - 1]!.role}` < + `${item.split}\0${item.modelID}\0${item.taskID}\0${item.role}`, + ), + "Qualification cells must be sorted", + ), + evaluatedAt: z.number().int().positive(), + }) + .strict() + export type Submit = z.input + + const SelectionBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("meta-harness-selection-v1"), + selectionID: Hash, + contractSHA256: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1).max(240), + runID: z.string().min(1).max(240), + searchRevision: z.number().int().nonnegative(), + stopReason: HarnessSearch.Stop, + candidateID: Hash, + candidateArtifact: z.object({ uri: z.string().min(1).max(2_048), sha256: Hash }).strict(), + optimizationResultSHA256: Hash, + optimizationEvaluationSHA256: Hash, + selectedAt: z.number().int().positive(), + }) + .strict() + + export const Selection = SelectionBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.selectionID + if (digest(stable) === value.selectionID) return + ctx.addIssue({ code: "custom", path: ["selectionID"], message: "Meta-harness selection hash is invalid" }) + }) + export type Selection = z.infer + + export const Diagnostics = z + .object({ + updaterGain: z.number().finite().optional(), + beneficiaryGain: z.number().finite().optional(), + worstHeldoutModelGain: z.number().finite().optional(), + activationRate: z.number().finite().min(0).max(1), + requiredAdherence: z.number().finite().min(0).max(1).optional(), + finalAdherence: z.number().finite().min(0).max(1).optional(), + maxPhaseDrift: z.number().finite().min(0).max(1).optional(), + predictionPrecision: z.number().finite().min(0).max(1), + riskRegressions: z.number().int().nonnegative(), + maxContextTokens: z.number().int().nonnegative(), + meanContextIncrease: z.number().finite(), + loadedBenefit: z.number().finite().optional(), + searchPairs: z.number().int().positive(), + heldoutPairs: z.number().int().positive(), + }) + .strict() + export type Diagnostics = z.infer + + const ReceiptBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("meta-harness-receipt-v1"), + receiptID: Hash, + contractSHA256: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1).max(240), + runID: z.string().min(1).max(240), + selection: Selection, + candidateManifestSHA256: Hash, + protectedManifestSHA256: Hash, + validatorSHA256: Hash, + archive: Archive, + refinements: z.array(Refinement).min(1).max(128), + cells: z.array(Cell).min(4).max(65_536), + diagnostics: Diagnostics, + status: HarnessEvaluation.Status, + failures: z.array(z.string().min(1).max(1_000)).max(1_024), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + + export const Receipt = ReceiptBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Meta-harness receipt hash is invalid" }) + }) + export type Receipt = z.infer + + const Claim = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + contractSHA256: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1), + selectionID: Hash, + }) + .strict() + + const root = path.join(Global.Path.data, "harness", "meta") + const file = (receiptID: string) => path.join(root, `${receiptID}.json`) + const claimfile = (sessionID: string) => path.join(root, "sessions", `${digest(sessionID)}.json`) + const covers = (root: string, target: string) => root === "." || target === root || target.startsWith(`${root}/`) + const key = (cell: Cell) => `${cell.split}\0${cell.modelID}\0${cell.taskID}\0${cell.role}` + const pairKey = (cell: Cell) => `${cell.split}\0${cell.modelID}\0${cell.taskID}` + const rate = (phase: z.infer) => { + const total = phase.followed + phase.violatedCommission + phase.violatedOmission + phase.requiredUnobserved + return total ? phase.followed / total : undefined + } + + async function claim(sessionID: string) { + const data = await JsonStore.read(claimfile(sessionID)) + if (!Object.keys(data).length) return null + return Claim.parse(data) + } + + export async function select(contract: HarnessContract.Info) { + const protocol = contract.metaHarness + if (!protocol) throw new Error(`Harness contract does not require meta-harness qualification`) + const state = await HarnessSearch.read(contract.sessionID) + if (state.runID !== contract.runID) throw new Error(`Search state does not match the bound harness run`) + if (state.status !== "completed" || !state.stopReason || !state.bestID) { + throw new Error(`Meta-harness qualification requires a terminal search with one verified winner`) + } + if (Object.values(state.reservations).some((item) => item.status === "open")) { + throw new Error(`Meta-harness qualification cannot start while candidate reservations remain open`) + } + const candidate = state.candidates[state.bestID] + if (!candidate || candidate.result?.source !== "verified" || candidate.result.status !== "passed") { + throw new Error(`The server-selected meta-harness subject is not a verified passing candidate`) + } + const evaluation = (await HarnessEvaluation.list(contract.sessionID)).findLast( + (item) => + item.subject?.type === "candidate" && item.subject.id === candidate.id && HarnessEvaluation.verified(item), + ) + if (!evaluation) throw new Error(`The terminal winner has no durable verified optimization evaluation`) + const stable = { + schemaVersion: 1 as const, + protocolVersion: "meta-harness-selection-v1" as const, + contractSHA256: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + sourceSessionID: contract.sessionID, + runID: contract.runID, + searchRevision: state.revision, + stopReason: state.stopReason, + candidateID: candidate.id, + candidateArtifact: candidate.artifact, + optimizationResultSHA256: digest(candidate.result), + optimizationEvaluationSHA256: HarnessEvaluation.fingerprint(evaluation), + selectedAt: state.updatedAt, + } + return Selection.parse({ ...stable, selectionID: digest(stable) }) + } + + function expected(protocol: HarnessContract.MetaHarness) { + return [ + ...protocol.search.models.flatMap((modelID) => + protocol.search.tasks.flatMap((task) => + (["baseline", "candidate"] as const).map((role) => `search\0${modelID.id}\0${task.id}\0${role}`), + ), + ), + ...protocol.heldout.models.flatMap((modelID) => + protocol.heldout.tasks.flatMap((task) => + (["baseline", "candidate"] as const).map((role) => `held_out\0${modelID.id}\0${task.id}\0${role}`), + ), + ), + ].toSorted() + } + + function task(protocol: HarnessContract.MetaHarness, split: Cell["split"], id: string) { + return (split === "search" ? protocol.search.tasks : protocol.heldout.tasks).find((item) => item.id === id)! + } + + function model(protocol: HarnessContract.MetaHarness, split: Cell["split"], id: string) { + return (split === "search" ? protocol.search.models : protocol.heldout.models).find((item) => item.id === id)! + } + + async function validateArchive( + protocol: HarnessContract.MetaHarness, + archive: z.infer, + state: HarnessSearch.State, + evaluations: HarnessEvaluation.Info[], + ) { + if ( + archive.schemaSHA256 !== protocol.archiveSchemaSHA256 || + archive.contents !== protocol.archive.contents || + archive.query !== protocol.archive.query || + archive.hiddenContent !== protocol.archive.hiddenContent || + archive.evaluatorContent !== protocol.archive.evaluatorContent + ) { + throw new Error(`Full-history archive does not match the frozen meta-harness protocol`) + } + const candidates = Object.values(state.candidates).toSorted((left, right) => left.id.localeCompare(right.id)) + if ( + JSON.stringify(archive.entries.map((item) => item.candidateID)) !== + JSON.stringify(candidates.map((item) => item.id)) + ) { + throw new Error(`Full-history archive must contain every search candidate exactly once`) + } + for (const candidate of candidates) { + const entry = archive.entries.find((item) => item.candidateID === candidate.id)! + if (entry.artifactSHA256 !== candidate.artifact.sha256) { + throw new Error(`Archive candidate ${candidate.id} changed its artifact`) + } + if (!candidate.result && entry.state !== "unevaluated") { + throw new Error(`Archive candidate ${candidate.id} fabricates an evaluation`) + } + if (!candidate.result) continue + if (entry.state !== "evaluated" || entry.resultSHA256 !== digest(candidate.result)) { + throw new Error(`Archive candidate ${candidate.id} changed its recorded result`) + } + if (entry.scoresSHA256 !== digest(candidate.result.metrics)) { + throw new Error(`Archive candidate ${candidate.id} changed its recorded scores`) + } + if (!candidate.result.evolution) throw new Error(`Archive candidate ${candidate.id} has no exact source lineage`) + const evolution = await HarnessEvolution.read(state.sessionID, candidate.result.evolution.receiptID) + if (!evolution || entry.sourceSHA256 !== evolution.snapshot.artifact.sha256) { + throw new Error(`Archive candidate ${candidate.id} changed its exact source snapshot`) + } + if (entry.artifactSHA256 !== entry.sourceSHA256) { + throw new Error(`Meta-harness candidates must use the exact source snapshot as their search artifact`) + } + const evaluation = + evaluations.findLast( + (item) => + item.subject?.type === "candidate" && + item.subject.id === candidate.id && + item.fidelity?.stage === candidate.result?.fidelity?.stage, + ) ?? evaluations.findLast((item) => item.subject?.type === "candidate" && item.subject.id === candidate.id) + if (!evaluation || entry.evaluationSHA256 !== HarnessEvaluation.fingerprint(evaluation)) { + throw new Error(`Archive candidate ${candidate.id} changed its evaluation`) + } + if (entry.trace?.schemaSHA256 !== protocol.traceSchemaSHA256) { + throw new Error(`Archive candidate ${candidate.id} changed the trace schema`) + } + } + } + + async function validateManifests( + protocol: HarnessContract.MetaHarness, + state: HarnessSearch.State, + candidateID: string, + candidateManifestSHA256: string, + protectedManifestSHA256: string, + ) { + const candidate = state.candidates[candidateID] + if (!candidate?.result?.evolution) throw new Error(`Selected meta-harness candidate has no exact source lineage`) + const receipt = await HarnessEvolution.read(state.sessionID, candidate.result.evolution.receiptID) + if (!receipt) throw new Error(`Selected meta-harness source lineage is unavailable`) + const files = receipt.snapshot.files.map((file) => ({ path: file.path, sha256: file.sha256 })) + if (digest(files) !== candidateManifestSHA256) { + throw new Error(`Meta-harness candidate manifest does not match its exact source snapshot`) + } + const protectedFiles = files.filter((file) => protocol.protected.roots.some((root) => covers(root, file.path))) + if ( + digest(protectedFiles) !== protocol.protected.manifestSHA256 || + protectedManifestSHA256 !== protocol.protected.manifestSHA256 + ) { + throw new Error(`Meta-harness candidate changed the protected source manifest`) + } + } + + function validateRefinements( + protocol: HarnessContract.MetaHarness, + refinements: z.infer[], + archive: z.infer, + artifactSHA256: string, + ) { + const searchModels = new Set(protocol.search.models.map((item) => item.id)) + const searchTasks = new Set(protocol.search.tasks.map((item) => item.id)) + const entries = new Map(archive.entries.map((item) => [item.candidateID, item])) + const predictions = new Set() + for (const [index, refinement] of refinements.entries()) { + if (refinement.revision !== index + 1) throw new Error(`Refinement revisions must be contiguous from one`) + const parent = index ? refinements[index - 1]!.snapshotSHA256 : protocol.baseline.artifactSHA256 + if (refinement.parentSnapshotSHA256 !== parent) + throw new Error(`Refinement snapshot lineage is stale or discontinuous`) + for (const change of refinement.changes) { + const roots = protocol.mutable.filter((item) => covers(item.root, change.path)) + if (roots.length !== 1 || roots[0]!.component !== change.component) { + throw new Error(`Refinement path ${change.path} is outside its declared mutable component`) + } + if (protocol.protected.roots.some((root) => covers(root, change.path))) { + throw new Error(`Refinement attempted to modify protected path ${change.path}`) + } + } + for (const citation of refinement.evidence) { + const entry = entries.get(citation.candidateID) + if (entry?.state !== "evaluated" || entry.trace?.sha256 !== citation.traceSHA256) { + throw new Error(`Refinement evidence is not bound to an archived search trace`) + } + } + if (refinement.predictions.some((item) => !searchModels.has(item.modelID) || !searchTasks.has(item.taskID))) { + throw new Error(`Refinement predictions may cite only frozen search cells`) + } + for (const prediction of refinement.predictions) { + const id = `${prediction.modelID}\0${prediction.taskID}` + if (predictions.has(id)) throw new Error(`Refinement predictions must be unique across the full lineage`) + predictions.add(id) + } + } + if (refinements.at(-1)?.snapshotSHA256 !== artifactSHA256) { + throw new Error(`Final refinement snapshot does not match the selected candidate artifact`) + } + } + + function diagnose( + protocol: HarnessContract.MetaHarness, + direction: "maximize" | "minimize", + cells: Cell[], + refinements: z.infer[], + ) { + if (JSON.stringify(cells.map(key)) !== JSON.stringify(expected(protocol))) { + throw new Error(`Qualification matrix must contain every frozen model-task baseline and candidate cell`) + } + for (const cell of cells) { + if (cell.trace.schemaSHA256 !== protocol.traceSchemaSHA256) { + throw new Error(`Qualification cell changed the frozen trace schema`) + } + if ( + cell.modelCommitment !== model(protocol, cell.split, cell.modelID).commitment || + cell.taskCommitment !== task(protocol, cell.split, cell.taskID).commitment + ) { + throw new Error(`Qualification cell changed a frozen model or task commitment`) + } + if (cell.role !== "candidate") continue + const activation = task(protocol, cell.split, cell.taskID).activationRequired + if (!activation && cell.phases.length) throw new Error(`Non-activation tasks cannot publish adherence phases`) + if ( + activation && + cell.loaded && + cell.outcome === "completed" && + JSON.stringify(cell.phases.map((item) => item.phase)) !== JSON.stringify(PhaseID.options) + ) { + throw new Error(`Loaded activation tasks require every canonical adherence phase`) + } + if (activation && !cell.loaded && cell.phases.length) { + throw new Error(`A non-loaded harness cannot claim adherence observations`) + } + } + const cellsByPair = new Map() + for (const cell of cells) { + const pair = cellsByPair.get(pairKey(cell)) ?? {} + pair[cell.role] = cell + cellsByPair.set(pairKey(cell), pair) + } + const pairs = [...cellsByPair.values()].map((pair) => ({ + baseline: pair.baseline!, + candidate: pair.candidate!, + })) + const completed = pairs.filter( + (pair) => pair.baseline.outcome === "completed" && pair.candidate.outcome === "completed", + ) + const adjusted = (pair: (typeof completed)[number]) => + direction === "maximize" + ? pair.candidate.score! - pair.baseline.score! + : pair.baseline.score! - pair.candidate.score! + const search = completed.filter((pair) => pair.candidate.split === "search") + const heldout = completed.filter((pair) => pair.candidate.split === "held_out") + const mean = (items: number[]) => + items.length ? items.reduce((sum, value) => sum + value, 0) / items.length : undefined + const updaterGain = mean(search.map(adjusted)) + const beneficiaryGain = mean(heldout.map(adjusted)) + const heldoutModels = protocol.heldout.models.map((model) => ({ + modelID: model.id, + gain: mean(heldout.filter((pair) => pair.candidate.modelID === model.id).map(adjusted)), + })) + const worstHeldoutModelGain = heldoutModels.every((item) => item.gain !== undefined) + ? Math.min(...heldoutModels.map((item) => item.gain!)) + : undefined + const activationCells = cells.filter( + (cell): cell is z.infer => + cell.role === "candidate" && task(protocol, cell.split, cell.taskID).activationRequired, + ) + const activationRate = activationCells.filter((cell) => cell.loaded).length / activationCells.length + const phases = activationCells.filter((cell) => cell.loaded).flatMap((cell) => cell.phases) + const relevant = phases.reduce( + (sum, phase) => + sum + phase.followed + phase.violatedCommission + phase.violatedOmission + phase.requiredUnobserved, + 0, + ) + const requiredAdherence = relevant ? phases.reduce((sum, phase) => sum + phase.followed, 0) / relevant : undefined + const finals = phases.filter((phase) => phase.phase === "final_validation") + const finalRelevant = finals.reduce( + (sum, phase) => + sum + phase.followed + phase.violatedCommission + phase.violatedOmission + phase.requiredUnobserved, + 0, + ) + const finalAdherence = finalRelevant + ? finals.reduce((sum, phase) => sum + phase.followed, 0) / finalRelevant + : undefined + const drift = activationCells + .filter((cell) => cell.loaded) + .map((cell) => { + const first = cell.phases.find((phase) => phase.phase === "loaded") + const last = cell.phases.find((phase) => phase.phase === "final_validation") + if (!first || !last) return undefined + const start = rate(first) + const end = rate(last) + return start === undefined || end === undefined ? undefined : Math.max(0, start - end) + }) + const maxPhaseDrift = drift.every((item) => item !== undefined) + ? Math.max(0, ...drift.map((item) => item!)) + : undefined + const lookup = new Map(pairs.map((pair) => [pairKey(pair.candidate), pair])) + const predictions = refinements.flatMap((item) => item.predictions) + const correct = predictions.filter((prediction) => { + const pair = lookup.get(`search\0${prediction.modelID}\0${prediction.taskID}`) + if (!pair || pair.baseline.outcome !== "completed" || pair.candidate.outcome !== "completed") return false + if (prediction.expected === "fail_to_pass") + return pair.baseline.passed === false && pair.candidate.passed === true + return pair.baseline.passed === true && pair.candidate.passed === true + }).length + const predictionPrecision = correct / predictions.length + const riskRegressions = completed.filter( + (pair) => pair.baseline.passed === true && pair.candidate.passed === false, + ).length + const candidateCells = cells.filter((cell) => cell.role === "candidate") + const maxContextTokens = Math.max(0, ...candidateCells.map((cell) => cell.contextTokens)) + const meanContextIncrease = + mean(pairs.map((pair) => pair.candidate.contextTokens - pair.baseline.contextTokens)) ?? 0 + const loaded = completed.filter((pair) => pair.candidate.role === "candidate" && pair.candidate.loaded) + const unloaded = completed.filter((pair) => pair.candidate.role === "candidate" && !pair.candidate.loaded) + const loadedMean = mean(loaded.map(adjusted)) + const unloadedMean = mean(unloaded.map(adjusted)) + const loadedBenefit = loadedMean === undefined || unloadedMean === undefined ? undefined : loadedMean - unloadedMean + const diagnostics = Diagnostics.parse({ + updaterGain, + beneficiaryGain, + worstHeldoutModelGain, + activationRate, + requiredAdherence, + finalAdherence, + maxPhaseDrift, + predictionPrecision, + riskRegressions, + maxContextTokens, + meanContextIncrease, + loadedBenefit, + searchPairs: protocol.search.models.length * protocol.search.tasks.length, + heldoutPairs: protocol.heldout.models.length * protocol.heldout.tasks.length, + }) + const incomplete = [ + ...cells + .filter((cell) => cell.outcome !== "completed") + .map((cell) => `cell:${cell.split}:${cell.modelID}:${cell.taskID}:${cell.role}:${cell.outcome}`), + ...(phases.some((phase) => phase.insufficientEvidence > 0) ? ["adherence:insufficient-evidence"] : []), + ...(updaterGain === undefined ? ["updater-gain:unavailable"] : []), + ...(beneficiaryGain === undefined ? ["beneficiary-gain:unavailable"] : []), + ...(worstHeldoutModelGain === undefined ? ["heldout-model-regression:unavailable"] : []), + ...(requiredAdherence === undefined ? ["required-adherence:unavailable"] : []), + ...(finalAdherence === undefined ? ["final-adherence:unavailable"] : []), + ...(maxPhaseDrift === undefined ? ["phase-drift:unavailable"] : []), + ] + const thresholds = [ + ...(updaterGain !== undefined && updaterGain < protocol.thresholds.minSearchGain + ? [`updater-gain:${updaterGain}`] + : []), + ...(beneficiaryGain !== undefined && beneficiaryGain < protocol.thresholds.minHeldoutGain + ? [`beneficiary-gain:${beneficiaryGain}`] + : []), + ...(worstHeldoutModelGain !== undefined && worstHeldoutModelGain < -protocol.thresholds.maxModelRegression + ? [`heldout-model-regression:${worstHeldoutModelGain}`] + : []), + ...(activationRate < protocol.thresholds.minActivationRate ? [`activation-rate:${activationRate}`] : []), + ...(requiredAdherence !== undefined && requiredAdherence < protocol.thresholds.minRequiredAdherence + ? [`required-adherence:${requiredAdherence}`] + : []), + ...(finalAdherence !== undefined && finalAdherence < protocol.thresholds.minFinalAdherence + ? [`final-adherence:${finalAdherence}`] + : []), + ...(maxPhaseDrift !== undefined && maxPhaseDrift > protocol.thresholds.maxPhaseDrift + ? [`phase-drift:${maxPhaseDrift}`] + : []), + ...(predictionPrecision < protocol.thresholds.minPredictionPrecision + ? [`prediction-precision:${predictionPrecision}`] + : []), + ...(riskRegressions > protocol.thresholds.maxRiskRegressions ? [`risk-regressions:${riskRegressions}`] : []), + ...(maxContextTokens > protocol.thresholds.maxContextTokens ? [`context-tokens:${maxContextTokens}`] : []), + ...(meanContextIncrease > protocol.thresholds.maxMeanContextIncrease + ? [`mean-context-increase:${meanContextIncrease}`] + : []), + ] + const failures = [...incomplete, ...thresholds] + const uncertain = cells.some((cell) => cell.outcome === "inconclusive") || incomplete.length > 0 + const hard = cells.some((cell) => cell.outcome === "failed") || thresholds.length > 0 + const status = hard ? ("failed" as const) : uncertain ? ("inconclusive" as const) : ("passed" as const) + return { diagnostics, failures, status } + } + + function comparable(receipt: Receipt, stable: Omit) { + const current = structuredClone(receipt) as Record + delete current.receiptID + delete current.recordedAt + return JSON.stringify(current) === JSON.stringify(stable) + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const protocol = contract.metaHarness + if (!protocol) throw new Error(`Harness contract does not require meta-harness qualification`) + if (value.sessionID !== contract.sessionID) throw new Error(`Meta-harness session does not match its contract`) + const selection = await select(contract) + if (value.selectionID !== selection.selectionID) + throw new Error(`Meta-harness submission changed the server selection`) + if (value.candidateArtifactSHA256 !== selection.candidateArtifact.sha256) { + throw new Error(`Meta-harness qualifier did not evaluate the server-selected candidate artifact`) + } + if (value.candidateManifestSHA256 === protocol.baseline.manifestSHA256) { + throw new Error(`Meta-harness candidate did not change the frozen baseline manifest`) + } + if (value.protectedManifestSHA256 !== protocol.protected.manifestSHA256) { + throw new Error(`Meta-harness candidate changed the protected manifest`) + } + if (value.validatorSHA256 !== protocol.validatorSHA256) { + throw new Error(`Meta-harness qualification changed the frozen validator`) + } + const now = Date.now() + if (value.evaluatedAt < selection.selectedAt || value.evaluatedAt > now) { + throw new Error(`Meta-harness qualification timestamp is outside the terminal selection interval`) + } + const [state, evaluations] = await Promise.all([ + HarnessSearch.read(contract.sessionID), + HarnessEvaluation.list(contract.sessionID), + ]) + await validateArchive(protocol, value.archive, state, evaluations) + await validateManifests( + protocol, + state, + selection.candidateID, + value.candidateManifestSHA256, + value.protectedManifestSHA256, + ) + validateRefinements(protocol, value.refinements, value.archive, selection.candidateArtifact.sha256) + const direction = contract.benchmark.direction + if (direction !== "maximize" && direction !== "minimize") { + throw new Error(`Meta-harness qualification requires a numeric benchmark direction`) + } + const result = diagnose(protocol, direction, value.cells, value.refinements) + const stable = { + schemaVersion: 1 as const, + protocolVersion: "meta-harness-receipt-v1" as const, + contractSHA256: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + sourceSessionID: contract.sessionID, + runID: contract.runID, + selection, + candidateManifestSHA256: value.candidateManifestSHA256, + protectedManifestSHA256: value.protectedManifestSHA256, + validatorSHA256: value.validatorSHA256, + archive: value.archive, + refinements: value.refinements, + cells: value.cells, + diagnostics: result.diagnostics, + status: result.status, + failures: result.failures, + evaluatedAt: value.evaluatedAt, + } + const active = await claim(contract.sessionID) + if (active) { + const receipt = await read(active.receiptID) + if (!receipt) throw new Error(`The session's frozen meta-harness receipt is corrupt`) + if (comparable(receipt, stable)) return receipt + throw new Error(`The session already has a frozen meta-harness receipt; qualification retries are forbidden`) + } + const body = { ...stable, recordedAt: now } + const receipt = Receipt.parse({ ...body, receiptID: digest(body) }) + await JsonStore.update(file(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Meta-harness receipt is immutable once recorded`) + }) + const saved = await read(receipt.receiptID) + if (!saved) throw new Error(`Meta-harness receipt was not durable after recording`) + const statement = Claim.parse({ + schemaVersion: 1, + receiptID: saved.receiptID, + contractSHA256: saved.contractSHA256, + protocolSHA256: saved.protocolSHA256, + sourceSessionID: saved.sourceSessionID, + selectionID: saved.selection.selectionID, + }) + await JsonStore.update(claimfile(contract.sessionID), async (data) => { + if (!Object.keys(data).length) return statement + const current = Claim.parse(data) + if (current.receiptID === statement.receiptID) return current + const winner = await read(current.receiptID) + if (winner && comparable(winner, stable)) return current + throw new Error(`The session already has a frozen meta-harness receipt; qualification retries are forbidden`) + }) + const frozen = await claim(contract.sessionID) + if (!frozen) throw new Error(`Meta-harness receipt was not durably frozen for its session`) + const winner = await read(frozen.receiptID) + if (winner && comparable(winner, stable)) return winner + throw new Error(`The session already has a different frozen meta-harness receipt`) + } + + export async function read(receiptID: string) { + const id = Hash.parse(receiptID) + const data = await JsonStore.read(file(id)) + const parsed = Receipt.safeParse(data) + return parsed.success && parsed.data.receiptID === id ? parsed.data : null + } + + export function bind(contract: HarnessContract.Info, input: Receipt) { + const receipt = Receipt.parse(input) + const protocol = contract.metaHarness + if (!protocol) throw new Error(`Receipt cites a meta-harness protocol that is not bound`) + if ( + receipt.contractSHA256 !== HarnessContract.fingerprint(contract) || + receipt.protocolSHA256 !== digest(protocol) || + receipt.sourceSessionID !== contract.sessionID || + receipt.runID !== contract.runID + ) { + throw new Error(`Meta-harness receipt does not match the bound contract`) + } + return receipt + } + + export async function assert(contract: HarnessContract.Info, receiptID: string) { + const stored = await read(receiptID) + if (!stored) throw new Error(`Unknown or corrupt meta-harness receipt ${receiptID}`) + const receipt = bind(contract, stored) + const active = await claim(contract.sessionID) + if (active?.receiptID !== receipt.receiptID) + throw new Error(`Receipt is not the session's canonical meta-harness qualification`) + const selection = await select(contract) + if (JSON.stringify(selection) !== JSON.stringify(receipt.selection)) { + throw new Error(`Meta-harness receipt changed the server-selected terminal winner`) + } + const [state, evaluations] = await Promise.all([ + HarnessSearch.read(contract.sessionID), + HarnessEvaluation.list(contract.sessionID), + ]) + await validateArchive(contract.metaHarness!, receipt.archive, state, evaluations) + await validateManifests( + contract.metaHarness!, + state, + selection.candidateID, + receipt.candidateManifestSHA256, + receipt.protectedManifestSHA256, + ) + validateRefinements(contract.metaHarness!, receipt.refinements, receipt.archive, selection.candidateArtifact.sha256) + const direction = contract.benchmark.direction + if (direction !== "maximize" && direction !== "minimize") { + throw new Error(`Meta-harness qualification requires a numeric benchmark direction`) + } + const result = diagnose(contract.metaHarness!, direction, receipt.cells, receipt.refinements) + if ( + receipt.status !== result.status || + JSON.stringify(receipt.diagnostics) !== JSON.stringify(result.diagnostics) || + JSON.stringify(receipt.failures) !== JSON.stringify(result.failures) + ) { + throw new Error(`Meta-harness receipt does not match the backend-derived diagnosis`) + } + return receipt + } + + export async function current(contract: HarnessContract.Info) { + if (!contract.metaHarness) return null + const active = await claim(contract.sessionID) + return active ? assert(contract, active.receiptID) : null + } + + export async function assertPromotable(contract: HarnessContract.Info) { + if (!contract.metaHarness?.promotionRequired) return null + const receipt = await current(contract) + if (!receipt) throw new Error(`Sealed confirmation is blocked until meta-harness qualification is recorded`) + if (receipt.status !== "passed") { + throw new Error(`Sealed confirmation is blocked by ${receipt.status} meta-harness qualification`) + } + return receipt + } + + export function prompt(contract: HarnessContract.Info) { + const protocol = contract.metaHarness + if (!protocol) return "" + return [ + "", + "The base harness, evaluator, hidden tasks, and protected roots are immutable. Propose only small session-scoped deltas under the declared mutable prompt, memory, skill, tool, middleware, subagent, or scaffold roots.", + "Every refinement must cite archived search-trace evidence, state a root cause and expected impact, and predeclare search-task flips or protected passing cells before evaluation.", + "Retain complete candidate source, scores, and raw execution traces in the filesystem archive. Summaries are navigation aids, never substitutes for the underlying trace bytes.", + `Qualification uses ${protocol.search.models.length} search model(s) and ${protocol.heldout.models.length} unseen model(s). Held-out task and model results never feed search, refinement, memory, or candidate selection.`, + "A harness that is not loaded, is not followed, drifts during long trajectories, regresses one held-out model, mutates protected files, or exceeds its context budget cannot reach sealed confirmation.", + "Meta-harness diagnostics are a promotion firewall, not the official benchmark score.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + return contract ? prompt(contract) : "" + } +} diff --git a/backend/cli/src/session/harness/orchestrator.ts b/backend/cli/src/session/harness/orchestrator.ts new file mode 100644 index 00000000..45ee6df9 --- /dev/null +++ b/backend/cli/src/session/harness/orchestrator.ts @@ -0,0 +1,1836 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessConfirmation } from "./confirmation" +import { HarnessContract } from "./contract" +import { HarnessMeta } from "./meta" +import { HarnessSemantic } from "./semantic" +import { HarnessReplication } from "./replication" +import { HarnessSynthesis } from "./synthesis" +import { HarnessAutonomy } from "./autonomy" +import { HarnessBlueprint } from "./blueprint" +import { HarnessFormal } from "./formal" + +export namespace HarnessOrchestrator { + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + export const WorkerAgent = z.enum(["task", "biology", "physics", "ml", "critique", "physics-critique", "reviewer"]) + const Agent = WorkerAgent + const Status = z.enum(["pending", "executed", "completed", "failed", "cancelled"]) + const Lane = z.enum(["producer-a", "producer-b"]) + const WorkerPolicy = z.enum(["claimed-v1", "task-attested-v1"]) + + const Usage = z + .object({ + steps: z.number().int().nonnegative().optional(), + tokens: z.number().int().nonnegative().optional(), + costUSD: z.number().nonnegative().optional(), + wallTimeMs: z.number().int().nonnegative().optional(), + }) + .strict() + + const Allocation = Usage + + export const WorkerReceipt = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + workID: z.string().regex(/^[a-f0-9]{64}$/), + workerSessionID: z.string().min(1), + turnID: z.string().min(1), + agent: Agent, + workPromptSHA256: z.string().regex(/^[a-f0-9]{64}$/), + taskPromptSHA256: z.string().regex(/^[a-f0-9]{64}$/), + outcome: z.enum(["completed", "failed"]), + usage: Usage, + toolCalls: z.number().int().nonnegative(), + failedToolCalls: z.number().int().nonnegative(), + startedAt: z.number().int().positive(), + completedAt: z.number().int().positive(), + provisional: z.literal(true), + }) + .strict() + .superRefine((value, ctx) => { + if (value.failedToolCalls > value.toolCalls) { + ctx.addIssue({ code: "custom", path: ["failedToolCalls"], message: "Failed tool calls exceed all tool calls" }) + } + if (value.completedAt < value.startedAt) { + ctx.addIssue({ code: "custom", path: ["completedAt"], message: "Worker receipt ends before it starts" }) + } + }) + export type WorkerReceipt = z.infer + + export const Verdict = z + .object({ + decision: z.enum(["support", "reject", "abstain"]), + severity: z.enum(["none", "minor", "critical", "unknown"]).optional(), + confidence: z.number().min(0).max(1), + checks: z + .array( + z + .object({ + id: z.string().min(1).max(200), + status: z.enum(["passed", "failed", "inconclusive"]), + evidenceRefs: z.array(z.string().min(1).max(2_048)).min(1).max(16), + }) + .strict(), + ) + .min(1) + .max(64), + }) + .strict() + .superRefine((value, ctx) => { + if (value.decision === "support" && value.checks.some((check) => check.status !== "passed")) { + ctx.addIssue({ code: "custom", path: ["checks"], message: "A supporting verdict requires every check to pass" }) + } + if (value.decision === "reject" && !value.checks.some((check) => check.status === "failed")) { + ctx.addIssue({ code: "custom", path: ["checks"], message: "A rejecting verdict requires a failed check" }) + } + if (value.decision === "abstain" && !value.checks.some((check) => check.status === "inconclusive")) { + ctx.addIssue({ + code: "custom", + path: ["checks"], + message: "An abstaining verdict requires an inconclusive check", + }) + } + if (value.severity === undefined) return + if (value.decision === "support" && value.severity !== "none") { + ctx.addIssue({ code: "custom", path: ["severity"], message: "Supporting verdict severity must be none" }) + } + if (value.decision === "reject" && !["minor", "critical"].includes(value.severity)) { + ctx.addIssue({ + code: "custom", + path: ["severity"], + message: "Rejecting verdict severity must be minor or critical", + }) + } + if (value.decision === "abstain" && value.severity !== "unknown") { + ctx.addIssue({ + code: "custom", + path: ["severity"], + message: "Abstaining verdict severity must be unknown", + }) + } + }) + export type Verdict = z.infer + + const Submission = z + .object({ + summary: z.string().min(1).max(8_000), + artifactRefs: z.array(z.string().min(1).max(2_048)).max(32).default([]), + evidenceRefs: z.array(z.string().min(1).max(2_048)).max(32).default([]), + usage: Usage.optional(), + verdict: Verdict.optional(), + }) + .strict() + + export const Result = Submission.extend({ + completedAt: z.number().int().positive(), + }).strict() + export type Result = z.infer + + export const CheckpointSubmit = z + .object({ + evaluatorToken: z.string().min(32).max(1_024), + round: z.number().int().min(1).max(8), + utility: z.number().finite().min(0).max(1), + uncertainty: z.number().finite().min(0).max(1), + evidenceRefs: z + .array(z.string().min(1).max(2_048)) + .min(1) + .max(32) + .refine((items) => new Set(items).size === items.length, "Checkpoint evidence references must be unique"), + evaluatedAt: z.number().int().positive(), + }) + .strict() + + export const Checkpoint = CheckpointSubmit.omit({ evaluatorToken: true }) + .extend({ + id: z.string().regex(/^[a-f0-9]{64}$/), + gain: z.number().finite().nullable(), + qualified: z.boolean(), + recordedAt: z.number().int().positive(), + }) + .strict() + export type Checkpoint = z.infer + + const CheckpointInput = CheckpointSubmit.omit({ evaluatorToken: true }) + .extend({ sessionID: z.string().min(1) }) + .strict() + + const checkpointID = (fingerprint: string, sessionID: string, value: Omit, "id">) => + digest({ + fingerprint, + sessionID, + round: value.round, + utility: value.utility, + uncertainty: value.uncertainty, + evidenceRefs: value.evidenceRefs, + evaluatedAt: value.evaluatedAt, + gain: value.gain, + qualified: value.qualified, + recordedAt: value.recordedAt, + }) + + function progress(config: HarnessContract.Adaptive, checkpoints: Checkpoint[], maxRounds: number) { + return checkpoints.reduce( + (state, item, index) => { + const previous = checkpoints[index - 1] + const gain = previous ? item.utility - previous.utility : null + const qualified = item.uncertainty <= config.maxUncertainty + const stalled = !qualified ? 0 : gain === null ? 0 : gain < config.minUtilityGain ? state.stalled + 1 : 0 + const target = + qualified && + item.round >= config.minRounds && + config.targetUtility !== undefined && + item.utility >= config.targetUtility + const exhausted = qualified && item.round >= config.minRounds && stalled >= config.patience + const reason = target + ? ("target_reached" as const) + : exhausted + ? ("marginal_utility_exhausted" as const) + : item.round === maxRounds + ? ("max_rounds" as const) + : undefined + return { + stalled, + expected: [...state.expected, { gain, qualified }], + reasons: [...state.reasons, reason], + } + }, + { + stalled: 0, + expected: [] as Array<{ gain: number | null; qualified: boolean }>, + reasons: [] as Array<"target_reached" | "marginal_utility_exhausted" | "max_rounds" | undefined>, + }, + ) + } + + const Adaptive = HarnessContract.Adaptive.extend({ + checkpoints: z.array(Checkpoint).max(8), + stalled: z.number().int().nonnegative().max(8), + phase: z.enum(["searching", "finalizing"]), + stopReason: z.enum(["target_reached", "marginal_utility_exhausted", "max_rounds"]).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.checkpoints.map((item) => item.id)).size !== value.checkpoints.length) { + ctx.addIssue({ + code: "custom", + path: ["checkpoints"], + message: "Adaptive checkpoint identities must be unique", + }) + } + if (value.checkpoints.some((item, index) => item.round !== index + 1)) { + ctx.addIssue({ code: "custom", path: ["checkpoints"], message: "Adaptive checkpoints must be sequential" }) + } + if (value.phase === "searching" && value.stopReason) { + ctx.addIssue({ + code: "custom", + path: ["stopReason"], + message: "Searching orchestration cannot have a stop reason", + }) + } + if (value.phase === "finalizing" && !value.stopReason) { + ctx.addIssue({ + code: "custom", + path: ["stopReason"], + message: "Finalizing orchestration requires a stop reason", + }) + } + }) + export type Adaptive = z.infer + + export const Work = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + role: HarnessContract.Role, + label: z.string().min(1).max(160), + round: z.number().int().nonnegative(), + agent: Agent, + dependencies: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(16) + .refine((items) => new Set(items).size === items.length, "Work dependencies must be unique"), + prompt: z.string().min(1).max(40_000), + allocation: Allocation, + lane: Lane.optional(), + status: Status, + workerSessionID: z.string().min(1).optional(), + workerReceipt: WorkerReceipt.optional(), + result: Result.optional(), + failure: z.string().min(1).max(4_000).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.status === "completed" && !value.result) { + ctx.addIssue({ code: "custom", path: ["result"], message: "Completed work requires a result" }) + } + if (value.status === "completed" && !value.workerSessionID) { + ctx.addIssue({ code: "custom", path: ["workerSessionID"], message: "Completed work requires a worker session" }) + } + if (value.status === "failed" && !value.failure) { + ctx.addIssue({ code: "custom", path: ["failure"], message: "Failed work requires a failure reason" }) + } + if (value.status === "failed" && !value.workerSessionID) { + ctx.addIssue({ code: "custom", path: ["workerSessionID"], message: "Failed work requires a worker session" }) + } + if (value.status === "executed" && (!value.workerSessionID || !value.workerReceipt)) { + ctx.addIssue({ code: "custom", message: "Executed work requires a worker receipt and session" }) + } + if (value.status === "executed" && (value.result || value.failure)) { + ctx.addIssue({ code: "custom", message: "Executed work cannot contain settled state" }) + } + if ( + value.status === "pending" && + (value.result || value.failure || value.workerSessionID || value.workerReceipt) + ) { + ctx.addIssue({ code: "custom", message: "Pending work cannot contain settled state" }) + } + if ( + value.status === "cancelled" && + (value.result || value.failure || value.workerSessionID || value.workerReceipt) + ) { + ctx.addIssue({ code: "custom", message: "Cancelled work cannot contain settled state" }) + } + if (value.status === "completed" && value.failure) { + ctx.addIssue({ code: "custom", path: ["failure"], message: "Completed work cannot contain a failure" }) + } + if (value.status === "failed" && value.result) { + ctx.addIssue({ code: "custom", path: ["result"], message: "Failed work cannot contain a result" }) + } + }) + export type Work = z.infer + + export const RepairRoute = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + attempt: z.number().int().min(1).max(8), + candidateID: z.string().regex(/^[a-f0-9]{64}$/), + actionID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + verifierIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .min(1) + .max(2) + .refine((items) => new Set(items).size === items.length, "Repair route verifiers must be unique"), + decision: z.enum(["accept", "revise", "restart", "investigate"]), + confidence: z.number().finite().min(0).max(1), + evidenceRefs: z + .array(z.string().min(1).max(2_048)) + .min(1) + .max(128) + .refine((items) => new Set(items).size === items.length, "Repair route evidence must be unique"), + recordedAt: z.number().int().positive(), + }) + .strict() + export type RepairRoute = z.infer + + const Repair = HarnessContract.Repair.extend({ + phase: z.enum(["producing", "verifying", "investigating", "completed"]), + candidateID: z.string().regex(/^[a-f0-9]{64}$/), + verifierIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(2) + .refine((items) => new Set(items).size === items.length, "Repair panel verifiers must be unique"), + evidenceID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + routes: z.array(RepairRoute).max(8), + stopReason: z.enum(["accepted", "attempt_limit", "work_failed"]).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.phase === "verifying" && !value.verifierIDs.length) { + ctx.addIssue({ code: "custom", path: ["verifierIDs"], message: "Verifying repair needs a verifier panel" }) + } + if (value.phase !== "verifying" && value.verifierIDs.length) { + ctx.addIssue({ code: "custom", path: ["verifierIDs"], message: "Only verification may retain a live panel" }) + } + if (value.phase === "investigating" && !value.evidenceID) { + ctx.addIssue({ code: "custom", path: ["evidenceID"], message: "Investigation needs an evidence work item" }) + } + if (value.phase !== "investigating" && value.evidenceID) { + ctx.addIssue({ code: "custom", path: ["evidenceID"], message: "Only investigation may retain evidence work" }) + } + if (value.phase === "completed" && !value.stopReason) { + ctx.addIssue({ code: "custom", path: ["stopReason"], message: "Completed repair needs a stop reason" }) + } + if (value.phase !== "completed" && value.stopReason) { + ctx.addIssue({ code: "custom", path: ["stopReason"], message: "Active repair cannot have a stop reason" }) + } + if (value.routes.some((item, index) => item.attempt !== index + 1)) { + ctx.addIssue({ code: "custom", path: ["routes"], message: "Repair routes must be sequential" }) + } + }) + export type Repair = z.infer + + function routeID(fingerprint: string, sessionID: string, route: Omit | RepairRoute) { + return digest({ + fingerprint, + sessionID, + attempt: route.attempt, + candidateID: route.candidateID, + actionID: route.actionID, + verifierIDs: route.verifierIDs, + decision: route.decision, + confidence: route.confidence, + evidenceRefs: route.evidenceRefs, + recordedAt: route.recordedAt, + }) + } + + function decision(verifiers: Work[], minConfidence: number): RepairRoute["decision"] { + const verdicts = verifiers.map((item) => item.result?.verdict) + if (verdicts.some((item) => item?.decision === "reject" && item.severity === "critical")) return "restart" + if (verdicts.some((item) => item?.decision === "abstain" || item?.severity === "unknown")) return "investigate" + if (verdicts.some((item) => item?.decision === "reject" && item.severity === "minor")) return "revise" + if (verdicts.every((item) => item?.decision === "support" && item.confidence >= minConfidence)) return "accept" + return "investigate" + } + + function lane(work: Pick) { + if (work.role === "generation" && work.label === "seed-a") return "producer-a" as const + if (work.role === "generation" && work.label === "seed-b") return "producer-b" as const + if (work.role === "evolution" && /^evolved-candidate-\d+$/.test(work.label)) return "producer-a" as const + if (work.role === "evolution" && /^divergent-candidate-\d+$/.test(work.label)) return "producer-b" as const + } + + function workID( + runID: string, + work: Pick, + policy: "legacy-v1" | "fresh-v1" | "producer-lanes-v1", + workers: z.infer, + ) { + return digest({ + runID, + role: work.role, + label: work.label, + dependencies: work.dependencies, + round: work.round, + ...(policy === "legacy-v1" ? {} : { sessionPolicy: policy }), + ...(workers === "claimed-v1" ? {} : { workerPolicy: workers }), + ...(work.lane ? { lane: work.lane } : {}), + }) + } + + function receiptID(fingerprint: string, runID: string, receipt: Omit) { + return digest({ + fingerprint, + runID, + workID: receipt.workID, + workerSessionID: receipt.workerSessionID, + turnID: receipt.turnID, + agent: receipt.agent, + workPromptSHA256: receipt.workPromptSHA256, + taskPromptSHA256: receipt.taskPromptSHA256, + outcome: receipt.outcome, + usage: receipt.usage, + toolCalls: receipt.toolCalls, + failedToolCalls: receipt.failedToolCalls, + startedAt: receipt.startedAt, + completedAt: receipt.completedAt, + provisional: receipt.provisional, + }) + } + + export const Selection = z + .object({ + topology: HarnessContract.Topology.exclude(["auto"]), + source: z.enum(["contract", "policy"]), + reasons: z.array(z.string().min(1)).min(1), + traits: HarnessContract.Traits, + }) + .strict() + export type Selection = z.infer + + export const Consensus = z + .object({ + status: z.enum(["supported", "rejected", "disputed", "insufficient"]), + verifierCount: z.number().int().nonnegative(), + support: z.number().int().nonnegative(), + reject: z.number().int().nonnegative(), + abstain: z.number().int().nonnegative(), + confidence: z.number().min(0).max(1), + evidenceRefs: z.array(z.string().min(1).max(2_048)).max(128), + provisional: z.literal(true), + derivedAt: z.number().int().positive(), + }) + .strict() + export type Consensus = z.infer + + function summarize(verifiers: Work[], derivedAt: number) { + const verdicts = verifiers.flatMap((item) => (item.result?.verdict ? [item.result.verdict] : [])) + const support = verdicts.filter((verdict) => verdict.decision === "support").length + const reject = verdicts.filter((verdict) => verdict.decision === "reject").length + const abstain = verdicts.filter((verdict) => verdict.decision === "abstain").length + const status = + verdicts.length < 2 + ? "insufficient" + : support === verdicts.length + ? "supported" + : reject === verdicts.length + ? "rejected" + : "disputed" + const evidenceRefs = [ + ...new Set( + verifiers.flatMap((item) => [ + ...(item.result?.evidenceRefs ?? []), + ...(item.result?.verdict?.checks.flatMap((check) => check.evidenceRefs) ?? []), + ]), + ), + ] + return Consensus.parse({ + status, + verifierCount: verdicts.length, + support, + reject, + abstain, + confidence: verdicts.length + ? verdicts.reduce((sum, verdict) => sum + verdict.confidence, 0) / verdicts.length + : 0, + evidenceRefs, + provisional: true, + derivedAt, + }) + } + + export const State = z + .object({ + schemaVersion: z.literal(3), + protocolVersion: z.enum(["coalition-v1", "coalition-v2", "coalition-v3"]), + sessionPolicy: z.enum(["legacy-v1", "fresh-v1", "producer-lanes-v1"]), + workerPolicy: WorkerPolicy, + runID: z.string().min(1), + sessionID: z.string().min(1), + contractFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + objective: z.string().min(1), + selection: Selection, + maxWorkers: z.number().int().min(1).max(2), + maxRounds: z.number().int().min(1).max(8), + minIndependentVerifiers: z.number().int().min(1).max(2), + status: z.enum(["active", "awaiting_checkpoint", "completed"]), + adaptive: Adaptive.optional(), + repair: Repair.optional(), + consensus: Consensus.optional(), + work: z.record(z.string(), Work), + order: z.array(z.string().regex(/^[a-f0-9]{64}$/)), + revision: z.number().int().nonnegative(), + createdAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.order).size !== value.order.length) { + ctx.addIssue({ code: "custom", path: ["order"], message: "Orchestration order must be unique" }) + } + if (Object.keys(value.work).length !== value.order.length) { + ctx.addIssue({ code: "custom", path: ["work"], message: "Orchestration work must exactly match its order" }) + } + const items = Object.values(value.work) + if (value.sessionPolicy === "producer-lanes-v1" && value.selection.topology !== "evolution") { + ctx.addIssue({ code: "custom", path: ["sessionPolicy"], message: "Producer lanes require evolution topology" }) + } + for (const work of items) { + const expected = value.sessionPolicy === "producer-lanes-v1" ? lane(work) : undefined + if (work.lane !== expected) { + ctx.addIssue({ code: "custom", path: ["work", work.id, "lane"], message: "Producer lane assignment drifted" }) + } + if (work.id !== workID(value.runID, work, value.sessionPolicy, value.workerPolicy)) { + ctx.addIssue({ + code: "custom", + path: ["work", work.id, "id"], + message: "Orchestration work identity drifted", + }) + } + } + const turns = new Set() + for (const work of items) { + const receipt = work.workerReceipt + if (value.workerPolicy === "task-attested-v1" && ["executed", "completed", "failed"].includes(work.status)) { + if (!receipt) { + ctx.addIssue({ code: "custom", path: ["work", work.id], message: "Settled work lacks a Task receipt" }) + continue + } + } + if (!receipt) continue + if (value.workerPolicy !== "task-attested-v1") { + ctx.addIssue({ + code: "custom", + path: ["work", work.id], + message: "Claimed worker state cannot contain receipts", + }) + } + if ( + receipt.workID !== work.id || + receipt.workerSessionID !== work.workerSessionID || + receipt.agent !== work.agent || + receipt.workPromptSHA256 !== digest(work.prompt) || + receipt.id !== receiptID(value.contractFingerprint, value.runID, receipt) + ) { + ctx.addIssue({ code: "custom", path: ["work", work.id, "workerReceipt"], message: "Worker receipt drifted" }) + } + if (receipt.startedAt < value.createdAt || receipt.completedAt > value.updatedAt) { + ctx.addIssue({ + code: "custom", + path: ["work", work.id, "workerReceipt"], + message: "Worker receipt falls outside the persisted orchestration window", + }) + } + if (turns.has(receipt.turnID)) { + ctx.addIssue({ code: "custom", path: ["work"], message: "A Task turn cannot attest multiple work items" }) + } + turns.add(receipt.turnID) + if (work.status === "completed" && receipt.outcome !== "completed") { + ctx.addIssue({ code: "custom", path: ["work", work.id], message: "Completed work has a failed Task receipt" }) + } + } + const sessions = new Map() + for (const work of items) { + if (!work.workerSessionID) continue + sessions.set(work.workerSessionID, [...(sessions.get(work.workerSessionID) ?? []), work]) + } + for (const works of sessions.values()) { + if (works.length === 1) continue + const lanes = new Set(works.map((work) => work.lane)) + if ( + value.sessionPolicy === "producer-lanes-v1" && + lanes.size === 1 && + !lanes.has(undefined) && + works.every((work) => ["generation", "evolution"].includes(work.role)) + ) { + continue + } + ctx.addIssue({ + code: "custom", + path: ["work"], + message: "Coalition worker session crossed isolation boundaries", + }) + } + if (value.sessionPolicy === "producer-lanes-v1") { + for (const name of Lane.options) { + const works = items.filter((work) => work.lane === name && work.workerSessionID) + if (new Set(works.map((work) => work.workerSessionID)).size <= 1) continue + ctx.addIssue({ code: "custom", path: ["work"], message: `Producer lane ${name} changed worker session` }) + } + } + if (value.consensus && value.status !== "completed") { + ctx.addIssue({ code: "custom", path: ["consensus"], message: "Consensus requires settled orchestration" }) + } + if (value.adaptive && value.protocolVersion !== "coalition-v2") { + ctx.addIssue({ + code: "custom", + path: ["protocolVersion"], + message: "Adaptive orchestration requires coalition-v2", + }) + } + if (value.repair && value.protocolVersion !== "coalition-v3") { + ctx.addIssue({ + code: "custom", + path: ["protocolVersion"], + message: "Verifier-routed repair requires coalition-v3", + }) + } + if (value.repair && value.selection.topology !== "verifier_loop") { + ctx.addIssue({ code: "custom", path: ["repair"], message: "Repair state requires verifier_loop topology" }) + } + if (value.status === "awaiting_checkpoint" && !value.adaptive) { + ctx.addIssue({ code: "custom", path: ["status"], message: "Only adaptive orchestration awaits checkpoints" }) + } + if (value.adaptive?.checkpoints.some((item) => item.round > value.maxRounds)) { + ctx.addIssue({ code: "custom", path: ["adaptive", "checkpoints"], message: "Checkpoint exceeds max rounds" }) + } + if ( + value.adaptive?.checkpoints.some( + (item) => item.id !== checkpointID(value.contractFingerprint, value.sessionID, item), + ) + ) { + ctx.addIssue({ code: "custom", path: ["adaptive", "checkpoints"], message: "Checkpoint content hash drifted" }) + } + if (value.adaptive) { + const derived = progress(value.adaptive, value.adaptive.checkpoints, value.maxRounds) + if ( + value.adaptive.checkpoints.some((item, index) => { + const expected = derived.expected[index]! + return item.gain !== expected.gain || item.qualified !== expected.qualified + }) + ) { + ctx.addIssue({ code: "custom", path: ["adaptive", "checkpoints"], message: "Checkpoint derivation drifted" }) + } + if (derived.reasons.slice(0, -1).some((reason) => reason !== undefined)) { + ctx.addIssue({ + code: "custom", + path: ["adaptive", "checkpoints"], + message: "Checkpoints continued after a stop", + }) + } + const reason = derived.reasons.at(-1) + const phase = reason ? "finalizing" : "searching" + if ( + value.adaptive.stalled !== derived.stalled || + value.adaptive.phase !== phase || + value.adaptive.stopReason !== reason + ) { + ctx.addIssue({ code: "custom", path: ["adaptive"], message: "Adaptive control state drifted" }) + } + } + if (value.repair) { + const items = Object.values(value.work) + if (value.repair.routes.length > value.maxRounds) { + ctx.addIssue({ code: "custom", path: ["repair", "routes"], message: "Repair exceeded attempt limit" }) + } + if (!value.work[value.repair.candidateID]) { + ctx.addIssue({ code: "custom", path: ["repair", "candidateID"], message: "Repair candidate is missing" }) + } + if (value.repair.verifierIDs.some((id) => value.work[id]?.role !== "verification")) { + ctx.addIssue({ code: "custom", path: ["repair", "verifierIDs"], message: "Repair panel drifted" }) + } + if (value.repair.evidenceID && value.work[value.repair.evidenceID]?.role !== "investigation") { + ctx.addIssue({ code: "custom", path: ["repair", "evidenceID"], message: "Repair evidence work drifted" }) + } + for (const route of value.repair.routes) { + const verifiers = route.verifierIDs.map((id) => value.work[id]).filter((item): item is Work => !!item) + const action = route.actionID ? value.work[route.actionID] : undefined + const terminal = route.decision === "accept" || route.attempt === value.maxRounds + const expectedRole = + route.decision === "revise" + ? "revision" + : route.decision === "restart" + ? "generation" + : route.decision === "investigate" + ? "investigation" + : undefined + const expectedDependencies = + route.decision === "restart" ? route.verifierIDs : [route.candidateID, ...route.verifierIDs] + const expectedLabel = + route.decision === "revise" + ? `targeted-revision-${route.attempt}` + : route.decision === "restart" + ? `clean-restart-${route.attempt}` + : `evidence-investigation-${route.attempt}` + if ( + route.id !== routeID(value.contractFingerprint, value.sessionID, route) || + verifiers.length !== value.minIndependentVerifiers || + verifiers.some((item) => item.status !== "completed" || item.role !== "verification") || + route.candidateID !== verifiers[0]?.dependencies[0] || + route.verifierIDs.some((id) => value.work[id]?.dependencies[0] !== route.candidateID) || + route.decision !== decision(verifiers, value.repair.minConfidence) || + terminal === !!action || + (!!action && + (action.role !== expectedRole || + action.label !== expectedLabel || + action.round !== route.attempt || + JSON.stringify(action.dependencies) !== JSON.stringify(expectedDependencies))) + ) { + ctx.addIssue({ code: "custom", path: ["repair", "routes"], message: "Repair route derivation drifted" }) + break + } + const evidenceRefs = [ + ...new Set( + verifiers.flatMap((item) => [ + ...(item.result?.evidenceRefs ?? []), + ...(item.result?.verdict?.checks.flatMap((check) => check.evidenceRefs) ?? []), + ]), + ), + ] + const confidence = + verifiers.reduce((sum, item) => sum + (item.result?.verdict?.confidence ?? 0), 0) / verifiers.length + if (JSON.stringify(route.evidenceRefs) !== JSON.stringify(evidenceRefs) || route.confidence !== confidence) { + ctx.addIssue({ code: "custom", path: ["repair", "routes"], message: "Repair route evidence drifted" }) + break + } + } + const final = value.repair.routes.at(-1) + if ( + value.repair.stopReason === "accepted" && + (final?.decision !== "accept" || value.repair.routes.length > value.maxRounds) + ) { + ctx.addIssue({ code: "custom", path: ["repair", "stopReason"], message: "Repair acceptance drifted" }) + } + if (value.repair.stopReason === "attempt_limit" && value.repair.routes.length !== value.maxRounds) { + ctx.addIssue({ code: "custom", path: ["repair", "stopReason"], message: "Repair attempt stop drifted" }) + } + if (value.repair.stopReason === "attempt_limit" && final?.decision === "accept") { + ctx.addIssue({ code: "custom", path: ["repair", "stopReason"], message: "Accepted repair cannot exhaust" }) + } + if (value.repair.stopReason === "work_failed" && !items.some((item) => item.status === "failed")) { + ctx.addIssue({ code: "custom", path: ["repair", "stopReason"], message: "Repair failure stop drifted" }) + } + if (value.repair.phase === "completed" && value.status !== "completed") { + ctx.addIssue({ code: "custom", path: ["status"], message: "Completed repair must settle orchestration" }) + } + if (value.repair.phase !== "completed" && value.status === "completed") { + ctx.addIssue({ code: "custom", path: ["status"], message: "Active repair cannot settle orchestration" }) + } + } + if ( + value.consensus && + value.consensus.support + value.consensus.reject + value.consensus.abstain !== value.consensus.verifierCount + ) { + ctx.addIssue({ code: "custom", path: ["consensus"], message: "Consensus verdict counts do not reconcile" }) + } + if (value.consensus) { + const final = value.repair?.stopReason === "work_failed" ? undefined : value.repair?.routes.at(-1) + const verifiers = final + ? final.verifierIDs.map((id) => value.work[id]!).filter(Boolean) + : value.repair + ? [] + : value.order.map((id) => value.work[id]!).filter((item) => item.role === "verification") + if (JSON.stringify(value.consensus) !== JSON.stringify(summarize(verifiers, value.consensus.derivedAt))) { + ctx.addIssue({ code: "custom", path: ["consensus"], message: "Consensus derivation drifted" }) + } + } + const seen = new Set() + for (const id of value.order) { + const work = value.work[id] + if (!work) { + ctx.addIssue({ code: "custom", path: ["work", id], message: "Orchestration work is missing" }) + continue + } + if (work.id !== id) { + ctx.addIssue({ code: "custom", path: ["work", id, "id"], message: "Orchestration work id drifted" }) + } + for (const dependency of work.dependencies) { + if (seen.has(dependency)) continue + ctx.addIssue({ + code: "custom", + path: ["work", id, "dependencies"], + message: `Dependency ${dependency} must precede ${id}`, + }) + } + seen.add(id) + } + }) + export type State = z.infer + + export type Ready = Work & { + resumeSessionID?: string + context: Array<{ + id: string + role: HarnessContract.Role + summary: string + artifactRefs: string[] + evidenceRefs: string[] + }> + } + + const root = path.join(Global.Path.data, "harness", "orchestration") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const clamp = (value: number) => Math.max(0, Math.min(1, value)) + const evolving = (role: HarnessContract.Role) => ["proximity", "reflection", "ranking", "evolution"].includes(role) + + function parse(data: Record) { + const migrated = + data.schemaVersion === 1 + ? { ...data, schemaVersion: 3, sessionPolicy: "legacy-v1" as const, workerPolicy: "claimed-v1" as const } + : data.schemaVersion === 2 + ? { ...data, schemaVersion: 3, workerPolicy: "claimed-v1" as const } + : data + return State.parse(migrated) + } + + const required: Record, HarnessContract.Role[]> = { + solo: ["generation"], + centralized: ["generation", "reflection", "verification"], + fork_join: ["generation", "simulation", "synthesis", "verification"], + tournament: ["generation", "proximity", "reflection", "ranking", "verification"], + evolution: ["generation", "proximity", "reflection", "ranking", "evolution", "investigation", "verification"], + verifier_loop: ["generation", "revision", "verification", "investigation"], + } + + function supports(topology: Exclude, roles?: HarnessContract.Role[]) { + if (!roles) return true + return required[topology].every((role) => roles.includes(role)) + } + + export function infer(contract: HarnessContract.Info): HarnessContract.Traits { + const text = `${contract.objective} ${contract.benchmark.task}`.toLowerCase() + const profile = contract.profile + const packs = contract.packs ?? [] + const decomposability = clamp( + (contract.benchmark.family === "generalist" ? 0.78 : 0.48) + + (packs.length > 1 ? 0.12 : 0) + + (/multi[- ]?step|end[- ]?to[- ]?end|workflow|survey|portfolio/.test(text) ? 0.12 : 0), + ) + const sequentiality = clamp( + (profile === "theory" ? 0.82 : profile === "reproduce" ? 0.66 : 0.38) + + (/proof|derive|derivation|replay|protocol order/.test(text) ? 0.12 : 0), + ) + const toolIntensity = clamp( + Math.max(contract.tools.length / 12, ["numerical", "training", "forecast"].includes(profile) ? 0.72 : 0.35), + ) + const uncertainty = clamp( + (["optimize", "reproduce"].includes(profile) ? 0.68 : 0.42) + + (/discover|novel|unknown|open[- ]ended|hypothesis/.test(text) ? 0.2 : 0), + ) + const verificationRisk = clamp( + (["held_out", "release"].includes(contract.benchmark.split) ? 0.72 : 0.48) + + (packs.length ? 0.12 : 0) + + (/sota|state.of.the.art|causal|mechanis|safety/.test(text) ? 0.14 : 0), + ) + const novelty = clamp( + (profile === "optimize" ? 0.68 : 0.38) + (/discover|novel|evolve|improve|new method/.test(text) ? 0.22 : 0), + ) + const crossDomain = clamp(packs.length / 3 + (contract.benchmark.family === "generalist" ? 0.3 : 0)) + return HarnessContract.Traits.parse({ + decomposability, + sequentiality, + toolIntensity, + uncertainty, + verificationRisk, + novelty, + crossDomain, + }) + } + + export function select(contract: HarnessContract.Info): Selection { + const config = contract.orchestration + const traits = config?.traits ?? infer(contract) + const roles = config?.roles + if (config?.topology && config.topology !== "auto") { + if (!supports(config.topology, roles)) { + throw new Error(`Orchestration roles do not permit the contract topology ${config.topology}`) + } + return Selection.parse({ topology: config.topology, source: "contract", reasons: ["contract-topology"], traits }) + } + const workers = config?.maxWorkers ?? 2 + const steps = contract.budget.steps ?? Number.POSITIVE_INFINITY + const tokens = contract.budget.tokens ?? Number.POSITIVE_INFINITY + const candidates = contract.budget.candidates ?? Number.POSITIVE_INFINITY + const small = workers === 1 || steps < 24 || tokens < 12_000 || candidates < 2 + const choose = (topology: Exclude, reasons: string[]): Selection | undefined => + supports(topology, roles) && (topology !== "verifier_loop" || !!config?.repair) + ? Selection.parse({ topology, source: "policy", reasons, traits }) + : undefined + if (small) return choose("solo", ["bounded-coordination-budget"]) ?? selectSolo(traits, roles) + if (traits.sequentiality >= 0.72 && traits.decomposability < 0.58) { + return ( + (traits.verificationRisk >= 0.62 + ? (choose("verifier_loop", ["sequential-task", "verifier-routed-repair"]) ?? + choose("centralized", ["sequential-task", "verification-gate"])) + : choose("solo", ["sequential-task"])) ?? selectSolo(traits, roles) + ) + } + if (traits.toolIntensity >= 0.76 && traits.decomposability < 0.64) { + return choose("centralized", ["tool-coordination-overhead", "central-control"]) ?? selectSolo(traits, roles) + } + if (traits.novelty >= 0.7 && traits.uncertainty >= 0.66) { + return choose("evolution", ["open-ended-search", "high-uncertainty"]) ?? selectSolo(traits, roles) + } + if (traits.uncertainty >= 0.62 && traits.verificationRisk >= 0.66) { + return choose("tournament", ["independent-hypotheses", "pairwise-critique"]) ?? selectSolo(traits, roles) + } + if (traits.decomposability >= 0.66 || traits.crossDomain >= 0.64) { + return choose("fork_join", ["decomposable-work", "bounded-parallelism"]) ?? selectSolo(traits, roles) + } + if (traits.verificationRisk >= 0.62) { + return ( + choose("verifier_loop", ["verifier-routed-repair"]) ?? + choose("centralized", ["verification-gate"]) ?? + selectSolo(traits, roles) + ) + } + return selectSolo(traits, roles) + } + + function verify(state: State, contract: HarnessContract.Info) { + if (state.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Orchestration state belongs to a different contract`) + } + const selection = select(contract) + const config = contract.orchestration + const adaptive = state.adaptive + ? { + protocolVersion: state.adaptive.protocolVersion, + minRounds: state.adaptive.minRounds, + patience: state.adaptive.patience, + minUtilityGain: state.adaptive.minUtilityGain, + maxUncertainty: state.adaptive.maxUncertainty, + targetUtility: state.adaptive.targetUtility, + } + : undefined + const repair = state.repair + ? { protocolVersion: state.repair.protocolVersion, minConfidence: state.repair.minConfidence } + : undefined + const protocol = config?.repair ? "coalition-v3" : config?.adaptive ? "coalition-v2" : "coalition-v1" + if ( + state.objective !== contract.objective || + JSON.stringify(state.selection) !== JSON.stringify(selection) || + state.maxWorkers !== (config?.maxWorkers ?? 2) || + state.maxRounds !== (config?.maxRounds ?? 2) || + state.minIndependentVerifiers !== (config?.minIndependentVerifiers ?? 1) || + JSON.stringify(adaptive) !== JSON.stringify(config?.adaptive) || + JSON.stringify(repair) !== JSON.stringify(config?.repair) || + (state.workerPolicy === "task-attested-v1" && state.protocolVersion !== protocol) + ) { + throw new Error(`Orchestration control state drifted from its bound contract`) + } + return state + } + + function selectSolo(traits: HarnessContract.Traits, roles?: HarnessContract.Role[]): Selection { + if (!supports("solo", roles)) throw new Error(`Orchestration roles must permit generation`) + return Selection.parse({ topology: "solo", source: "policy", reasons: ["coordination-not-justified"], traits }) + } + + function agent(role: HarnessContract.Role, contract: HarnessContract.Info) { + if (role === "reflection" || role === "investigation") { + return contract.profile === "theory" || contract.profile === "numerical" ? "physics-critique" : "critique" + } + if (role === "verification") { + return contract.profile === "theory" || contract.profile === "numerical" ? "physics-critique" : "reviewer" + } + if (["proximity", "ranking", "synthesis"].includes(role)) return "task" + const family = contract.benchmark.family + if (family === "biology") return "biology" + if (family === "physics") return "physics" + if (family === "ml" || contract.profile === "training" || contract.profile === "forecast") return "ml" + return "task" + } + + function instruction(role: HarnessContract.Role) { + const lines: Record = { + generation: + "Develop one independent, executable solution or hypothesis. State assumptions and produce artifact references.", + proximity: + "Cluster upstream proposals by mechanism and failure mode. Preserve distinct ideas; do not select a winner.", + reflection: + "Adversarially test upstream work for correctness, novelty, leakage, and missing controls. Return actionable falsifiers.", + ranking: + "Run pairwise comparisons using the declared objective and evidence. Emit a provisional ranking with uncertainty.", + evolution: + "Create a new candidate by combining verified strengths while explicitly avoiding documented failure modes.", + revision: + "Repair only the evidence-backed failed checks in the upstream candidate. Preserve supported components, expose every change, and return a complete replacement artifact rather than a patch to hidden reasoning.", + verification: + "Verify from a fresh session using artifacts and observable evidence only. Do not trust producer conclusions or hidden reasoning. Return support, reject, or abstain with confidence and evidence-backed checks; classify severity as none, minor, critical, or unknown; do not inspect another verifier's verdict.", + investigation: + "Acquire observable evidence for inconclusive checks and search for counterexamples, edge cases, sabotage, reward hacking, and distribution-shift failures. Do not revise the candidate.", + simulation: + "Execute the appropriate simulator or numerical check and report configuration, invariants, convergence, and artifacts.", + synthesis: + "Join upstream outputs without erasing disagreement. Separate supported results, unresolved conflicts, and next tests.", + } + return lines[role] + } + + function unit( + contract: HarnessContract.Info, + selection: Selection, + role: HarnessContract.Role, + label: string, + dependencies: string[] = [], + round = 0, + lane?: z.infer, + ): Omit { + const policy = selection.topology === "evolution" ? "producer-lanes-v1" : "fresh-v1" + const id = workID(contract.runID, { role, label, dependencies, round, lane }, policy, "task-attested-v1") + return { + id, + role, + label, + round, + agent: agent(role, contract), + dependencies, + ...(lane ? { lane } : {}), + prompt: [ + ``, + `Objective: ${contract.objective}`, + ...(contract.semanticAudit ? [HarnessSemantic.prompt(contract)] : []), + ...(contract.synthesis ? [HarnessSynthesis.prompt(contract)] : []), + ...(contract.autonomy ? [HarnessAutonomy.prompt(contract)] : []), + ...(contract.formalProof ? [HarnessFormal.prompt(contract)] : []), + ...(contract.formalProof?.blueprint ? [HarnessBlueprint.prompt(contract)] : []), + ...(contract.replication ? [HarnessReplication.prompt(contract)] : []), + ...(contract.metaHarness ? [HarnessMeta.prompt(contract)] : []), + ...(contract.confirmation ? [HarnessConfirmation.prompt(contract)] : []), + instruction(role), + ...(lane + ? [ + "This is a persistent producer lane. On resumed rounds, inspect the new dependency artifacts and feedback, retain only useful tested context, and use tools to propose, test, repair, and critique the edit before returning one candidate.", + "Lane memory is search context only. It cannot certify a benchmark result, replace observable evidence, or influence verifier authority.", + ] + : []), + ...(selection.topology === "verifier_loop" && role === "generation" && label.startsWith("clean-restart-") + ? [ + "Start from a blank solution. Use upstream verifier summaries only as failure constraints; do not reconstruct, quote, or minimally edit the rejected candidate.", + ] + : []), + ...(selection.topology === "verifier_loop" && role === "verification" + ? [ + "Severity is controller input: none means all checks pass; minor means a localized correction can preserve the candidate premise; critical means the premise or global reasoning is invalid; unknown means evidence is insufficient.", + "The backend, not this worker, chooses accept, revise, restart, or investigate after every independent verifier settles.", + ] + : []), + "Return a concise result, artifact references, evidence references, and actual resource usage.", + "Your output is provisional orchestration state, never benchmark evidence or a final scientific claim.", + "", + ].join("\n"), + status: "pending", + } + } + + function plan(contract: HarnessContract.Info, selection: Selection) { + const items: Array> = [] + const add = ( + role: HarnessContract.Role, + label: string, + dependencies: string[] = [], + round = 0, + lane?: z.infer, + ) => { + const item = unit(contract, selection, role, label, dependencies, round, lane) + items.push(item) + return item.id + } + const verify = (dependencies: string[], round: number) => + Array.from({ length: contract.orchestration?.minIndependentVerifiers ?? 1 }, (_, index) => + add("verification", `independent-verification-${index + 1}`, dependencies, round), + ) + if (selection.topology === "solo") add("generation", "direct-solution") + if (selection.topology === "centralized") { + const generated = add("generation", "central-proposal") + const reflected = add("reflection", "central-critique", [generated]) + verify([generated, reflected], 1) + } + if (selection.topology === "fork_join") { + const generated = add("generation", "analytical-branch") + const simulated = add("simulation", "computational-branch") + const synthesis = add("synthesis", "evidence-join", [generated, simulated], 1) + verify([synthesis], 1) + } + if (selection.topology === "tournament") { + const first = add("generation", "independent-proposal-a") + const second = add("generation", "independent-proposal-b") + const proximity = add("proximity", "proposal-map", [first, second]) + const reflection = add("reflection", "tournament-critique", [first, second, proximity]) + const ranking = add("ranking", "pairwise-ranking", [proximity, reflection], 1) + verify([first, second, ranking], 1) + } + if (selection.topology === "evolution") { + const first = add("generation", "seed-a", [], 0, "producer-a") + const second = add("generation", "seed-b", [], 0, "producer-b") + const evolved = Array.from({ length: contract.orchestration?.maxRounds ?? 2 }).reduce( + (parents: string[], _, index) => { + const round = index + 1 + const proximity = add("proximity", `mechanism-map-${round}`, parents, round) + const reflection = add("reflection", `adversarial-reflection-${round}`, [...parents, proximity], round) + const ranking = add("ranking", `pairwise-tournament-${round}`, [proximity, reflection], round) + return [ + add("evolution", `evolved-candidate-${round}`, [ranking, reflection], round, "producer-a"), + add("evolution", `divergent-candidate-${round}`, [ranking, reflection], round, "producer-b"), + ] + }, + [first, second], + ) + const investigation = add("investigation", "failure-discovery", evolved, contract.orchestration?.maxRounds ?? 2) + verify([...evolved, investigation], (contract.orchestration?.maxRounds ?? 2) + 1) + } + if (selection.topology === "verifier_loop") add("generation", "initial-candidate") + return items + } + + function allocation(contract: HarnessContract.Info, count: number): z.infer { + const share = (value: number | undefined) => (value === undefined ? undefined : Math.floor(value / count)) + return Allocation.parse({ + steps: share(contract.budget.steps), + tokens: share(contract.budget.tokens), + costUSD: contract.budget.costUSD === undefined ? undefined : contract.budget.costUSD / count, + wallTimeMs: share(contract.budget.wallTimeMs), + }) + } + + export async function initialize(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${sessionID}`) + const selection = select(contract) + const planned = plan(contract, selection) + if (!planned.length) throw new Error(`Orchestration policy produced no work`) + const capacity = contract.orchestration?.repair + ? contract.orchestration.maxRounds * (contract.orchestration.minIndependentVerifiers + 1) + : planned.length + if (contract.budget.steps !== undefined && contract.budget.steps < capacity) { + throw new Error(`Contract step budget cannot allocate one step to every orchestration unit`) + } + const budget = allocation(contract, capacity) + const work = Object.fromEntries(planned.map((item) => [item.id, Work.parse({ ...item, allocation: budget })])) + const now = Date.now() + const state = State.parse({ + schemaVersion: 3, + protocolVersion: contract.orchestration?.repair + ? "coalition-v3" + : contract.orchestration?.adaptive + ? "coalition-v2" + : "coalition-v1", + sessionPolicy: selection.topology === "evolution" ? "producer-lanes-v1" : "fresh-v1", + workerPolicy: "task-attested-v1", + runID: contract.runID, + sessionID, + contractFingerprint: HarnessContract.fingerprint(contract), + objective: contract.objective, + selection, + maxWorkers: contract.orchestration?.maxWorkers ?? 2, + maxRounds: contract.orchestration?.maxRounds ?? 2, + minIndependentVerifiers: contract.orchestration?.minIndependentVerifiers ?? 1, + status: "active", + adaptive: contract.orchestration?.adaptive + ? { + ...contract.orchestration.adaptive, + checkpoints: [], + stalled: 0, + phase: "searching", + } + : undefined, + repair: contract.orchestration?.repair + ? { + ...contract.orchestration.repair, + phase: "producing", + candidateID: planned[0]!.id, + verifierIDs: [], + routes: [], + } + : undefined, + work, + order: planned.map((item) => item.id), + revision: 0, + createdAt: now, + updatedAt: now, + }) + await JsonStore.update(file(sessionID), (data) => { + if (!Object.keys(data).length) return state + const current = parse(data) + if (current.contractFingerprint === state.contractFingerprint) return current + throw new Error(`Orchestration state belongs to a different contract`) + }) + return read(sessionID) + } + + export async function read(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${sessionID}`) + return verify(parse(await JsonStore.read(file(sessionID))), contract) + } + + function resume(state: State, work: Work) { + if (state.sessionPolicy !== "producer-lanes-v1" || !work.lane) return + const index = state.order.indexOf(work.id) + return state.order + .slice(0, index) + .map((id) => state.work[id]!) + .findLast((item) => item.lane === work.lane && item.status === "completed")?.workerSessionID + } + + function worker(state: State, work: Work, sessionID: string) { + const expected = resume(state, work) + if (expected) { + if (sessionID !== expected) { + throw new Error(`Producer lane ${work.lane} must resume its prior worker session`) + } + return + } + if (Object.values(state.work).some((item) => item.workerSessionID === sessionID)) { + throw new Error(`Each fresh coalition role requires a distinct worker session`) + } + } + + const WorkerAttestation = z + .object({ + sessionID: z.string().min(1), + workID: z.string().regex(/^[a-f0-9]{64}$/), + workerSessionID: z.string().min(1), + turnID: z.string().min(1), + agent: Agent, + prompt: z.string().min(1).max(128_000), + outcome: z.enum(["completed", "failed"]), + usage: Usage, + toolCalls: z.number().int().nonnegative(), + failedToolCalls: z.number().int().nonnegative(), + startedAt: z.number().int().positive(), + completedAt: z.number().int().positive(), + }) + .strict() + + export async function attest(input: z.input) { + const value = WorkerAttestation.parse(input) + const contract = await HarnessContract.read(value.sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${value.sessionID}`) + await JsonStore.update(file(value.sessionID), (data) => { + const state = verify(parse(data), contract) + if (state.workerPolicy !== "task-attested-v1") { + throw new Error(`Legacy orchestration does not accept Task execution receipts`) + } + const work = state.work[value.workID] + if (!work) throw new Error(`Unknown orchestration work ${value.workID}`) + const base = { + workID: work.id, + workerSessionID: value.workerSessionID, + turnID: value.turnID, + agent: value.agent, + workPromptSHA256: digest(work.prompt), + taskPromptSHA256: digest(value.prompt), + outcome: value.outcome, + usage: Usage.parse(value.usage), + toolCalls: value.toolCalls, + failedToolCalls: value.failedToolCalls, + startedAt: value.startedAt, + completedAt: value.completedAt, + provisional: true as const, + } + const receipt = WorkerReceipt.parse({ + id: receiptID(state.contractFingerprint, state.runID, base), + ...base, + }) + if (work.status === "executed") { + if (JSON.stringify(work.workerReceipt) === JSON.stringify(receipt)) return state + throw new Error(`Task execution receipt is immutable`) + } + if (work.status !== "pending") throw new Error(`Orchestration work ${work.id} cannot accept a Task receipt`) + if (!ready(state).some((item) => item.id === work.id)) { + throw new Error(`Orchestration work is not ready for Task execution`) + } + if (value.agent !== work.agent) throw new Error(`Task execution used the wrong coalition agent`) + if (!value.prompt.includes(work.prompt)) throw new Error(`Task execution omitted the canonical coalition prompt`) + const now = Date.now() + if (value.startedAt < state.createdAt || value.completedAt > now) { + throw new Error(`Task execution timestamps fall outside the orchestration window`) + } + if (Object.values(state.work).some((item) => item.workerReceipt?.turnID === value.turnID)) { + throw new Error(`Task turn already attests another orchestration work item`) + } + worker(state, work, value.workerSessionID) + return State.parse({ + ...state, + work: { + ...state.work, + [work.id]: { + ...work, + status: "executed", + workerSessionID: value.workerSessionID, + workerReceipt: receipt, + }, + }, + revision: state.revision + 1, + updatedAt: now, + }) + }) + return read(value.sessionID) + } + + export function ready(state: State): Ready[] { + const current = State.parse(state) + return current.order.flatMap((id) => { + const work = current.work[id]! + if (work.status !== "pending") return [] + if (!work.dependencies.every((dependency) => current.work[dependency]?.status === "completed")) return [] + if ( + current.adaptive && + evolving(work.role) && + work.round > 1 && + !current.adaptive.checkpoints.some((item) => item.round === work.round - 1) + ) { + return [] + } + if ( + current.adaptive && + work.role === "investigation" && + !current.adaptive.checkpoints.some((item) => item.round === work.round) + ) { + return [] + } + const context = work.dependencies.map((dependency) => { + const parent = current.work[dependency]! + return { + id: parent.id, + role: parent.role, + summary: parent.result!.summary, + artifactRefs: parent.result!.artifactRefs, + evidenceRefs: parent.result!.evidenceRefs, + } + }) + const sessionID = resume(current, work) + return [{ ...work, ...(sessionID ? { resumeSessionID: sessionID } : {}), context }] + }) + } + + function within(usage: z.infer | undefined, budget: z.infer) { + if (!usage) return + const checks: Array<[keyof z.infer, number | undefined, number | undefined]> = [ + ["steps", usage.steps, budget.steps], + ["tokens", usage.tokens, budget.tokens], + ["costUSD", usage.costUSD, budget.costUSD], + ["wallTimeMs", usage.wallTimeMs, budget.wallTimeMs], + ] + const exceeded = checks.find(([, value, limit]) => value !== undefined && limit !== undefined && value > limit) + if (exceeded) throw new Error(`Work exceeded its ${exceeded[0]} allocation`) + } + + function due(state: Pick) { + if (!state.adaptive || state.adaptive.phase !== "searching") return + const round = state.adaptive.checkpoints.length + 1 + const items = state.order.map((id) => state.work[id]!).filter((item) => item.round === round && evolving(item.role)) + if (!items.length || items.some((item) => item.status !== "completed")) return + return round + } + + function append(state: State, items: Array>) { + const budget = state.work[state.order[0]!]!.allocation + const added = items.map((item) => Work.parse({ ...item, allocation: budget })) + return { + ...state, + work: Object.fromEntries([...Object.entries(state.work), ...added.map((item) => [item.id, item] as const)]), + order: [...state.order, ...added.map((item) => item.id)], + } + } + + function panel(state: State, contract: HarnessContract.Info, dependencies: string[], attempt: number) { + const verifiers = Array.from({ length: state.minIndependentVerifiers }, (_, index) => + unit( + contract, + state.selection, + "verification", + `repair-verification-${attempt}-${index + 1}`, + dependencies, + attempt, + ), + ) + const next = append(state, verifiers) + return State.parse({ + ...next, + repair: { + ...state.repair!, + phase: "verifying", + verifierIDs: verifiers.map((item) => item.id), + evidenceID: undefined, + }, + }) + } + + function advance(state: State, contract: HarnessContract.Info, now: number): State { + if (!state.repair) return state + if (state.repair.phase === "completed") return state + if (state.repair.phase === "producing") { + const candidate = state.work[state.repair.candidateID]! + if (["failed", "cancelled"].includes(candidate.status)) { + return State.parse({ + ...state, + status: "completed", + repair: { ...state.repair, phase: "completed", verifierIDs: [], stopReason: "work_failed" }, + }) + } + if (candidate.status !== "completed") return state + return panel(state, contract, [candidate.id], state.repair.routes.length + 1) + } + if (state.repair.phase === "investigating") { + const evidence = state.work[state.repair.evidenceID!]! + if (["failed", "cancelled"].includes(evidence.status)) { + return State.parse({ + ...state, + status: "completed", + repair: { + ...state.repair, + phase: "completed", + verifierIDs: [], + evidenceID: undefined, + stopReason: "work_failed", + }, + }) + } + if (evidence.status !== "completed") return state + return panel( + { ...state, repair: { ...state.repair, verifierIDs: [], evidenceID: undefined } }, + contract, + [state.repair.candidateID, evidence.id], + state.repair.routes.length + 1, + ) + } + const verifiers = state.repair.verifierIDs.map((id) => state.work[id]!) + if (verifiers.some((item) => !["completed", "failed", "cancelled"].includes(item.status))) return state + if (verifiers.some((item) => item.status !== "completed")) { + return State.parse({ + ...state, + status: "completed", + repair: { ...state.repair, phase: "completed", verifierIDs: [], stopReason: "work_failed" }, + }) + } + const route = decision(verifiers, state.repair.minConfidence) + const attempt = state.repair.routes.length + 1 + const stopping = route === "accept" || attempt === state.maxRounds + const dependencies = + route === "restart" ? state.repair.verifierIDs : [state.repair.candidateID, ...state.repair.verifierIDs] + const role = + route === "revise" + ? ("revision" as const) + : route === "restart" + ? ("generation" as const) + : ("investigation" as const) + const label = + route === "revise" + ? `targeted-revision-${attempt}` + : route === "restart" + ? `clean-restart-${attempt}` + : `evidence-investigation-${attempt}` + const action = stopping ? undefined : unit(contract, state.selection, role, label, dependencies, attempt) + const evidenceRefs = [ + ...new Set( + verifiers.flatMap((item) => [ + ...item.result!.evidenceRefs, + ...item.result!.verdict!.checks.flatMap((check) => check.evidenceRefs), + ]), + ), + ] + const record = { + attempt, + candidateID: state.repair.candidateID, + actionID: action?.id, + verifierIDs: state.repair.verifierIDs, + decision: route, + confidence: + verifiers.reduce((sum, item) => sum + item.result!.verdict!.confidence, 0) / state.minIndependentVerifiers, + evidenceRefs, + recordedAt: now, + } + const recorded = RepairRoute.parse({ id: routeID(state.contractFingerprint, state.sessionID, record), ...record }) + const routes = [...state.repair.routes, recorded] + if (stopping) { + return State.parse({ + ...state, + status: "completed", + repair: { + ...state.repair, + phase: "completed", + verifierIDs: [], + routes, + stopReason: route === "accept" ? "accepted" : "attempt_limit", + }, + }) + } + if (!action) throw new Error(`Repair route did not produce its required action`) + const next = append(state, [action]) + if (route === "investigate") { + return State.parse({ + ...next, + repair: { + ...state.repair, + phase: "investigating", + verifierIDs: [], + evidenceID: action.id, + routes, + }, + }) + } + return State.parse({ + ...next, + repair: { + ...state.repair, + phase: "producing", + candidateID: action.id, + verifierIDs: [], + routes, + }, + }) + } + + function settle(state: State, now: number): State { + const work = Object.fromEntries( + state.order.map((id) => { + const item = state.work[id]! + const blocked = item.dependencies.some((dependency) => + ["failed", "cancelled"].includes(state.work[dependency]?.status ?? "pending"), + ) + return [id, blocked && item.status === "pending" ? { ...item, status: "cancelled" as const } : item] + }), + ) + const done = Object.values(work).every((item) => ["completed", "failed", "cancelled"].includes(item.status)) + const checkpoint = due({ ...state, work }) + const final = state.repair?.stopReason === "work_failed" ? undefined : state.repair?.routes.at(-1) + const verifiers = final + ? final.verifierIDs.map((id) => work[id]!) + : state.repair + ? [] + : state.order.map((id) => work[id]!).filter((item) => item.role === "verification") + const consensus = done ? summarize(verifiers, now) : undefined + const status = state.repair + ? state.repair.phase === "completed" && done + ? "completed" + : "active" + : done + ? "completed" + : checkpoint + ? "awaiting_checkpoint" + : "active" + return State.parse({ ...state, work, status, consensus, updatedAt: now }) + } + + function finale(state: State, contract: HarnessContract.Info, round: number) { + const parents = state.order.filter((id) => { + const item = state.work[id]! + return item.role === "evolution" && item.round === round && item.status === "completed" + }) + if (parents.length < 2) + throw new Error(`Adaptive finalization requires two completed candidates from round ${round}`) + const probeBudget = state.order + .map((id) => state.work[id]!) + .find((item) => item.role === "investigation")?.allocation + const verifyBudget = state.order + .map((id) => state.work[id]!) + .find((item) => item.role === "verification")?.allocation + if (!probeBudget || !verifyBudget) + throw new Error(`Adaptive finalization could not reserve investigation and verification`) + const cancelled = Object.fromEntries( + state.order.map((id) => { + const item = state.work[id]! + const stop = + item.status === "pending" && (item.round > round || ["investigation", "verification"].includes(item.role)) + return [id, stop ? { ...item, status: "cancelled" as const } : item] + }), + ) + const probe = Work.parse({ + ...unit(contract, state.selection, "investigation", `adaptive-failure-discovery-${round}`, parents, round), + allocation: probeBudget, + }) + const verifiers = Array.from({ length: state.minIndependentVerifiers }, (_, index) => + Work.parse({ + ...unit( + contract, + state.selection, + "verification", + `adaptive-independent-verification-${index + 1}-${round}`, + [...parents, probe.id], + round + 1, + ), + allocation: verifyBudget, + }), + ) + return State.parse({ + ...state, + work: Object.fromEntries([ + ...Object.entries(cancelled), + [probe.id, probe], + ...verifiers.map((item) => [item.id, item] as const), + ]), + order: [...state.order, probe.id, ...verifiers.map((item) => item.id)], + }) + } + + export async function checkpoint(input: z.input, contract: HarnessContract.Info) { + const value = CheckpointInput.parse(input) + const bound = HarnessContract.Info.parse(contract) + if (bound.sessionID !== value.sessionID) throw new Error(`Utility checkpoint does not match the harness contract`) + await JsonStore.update(file(value.sessionID), (data) => { + const state = verify(parse(data), bound) + if (!state.adaptive) throw new Error(`Orchestration does not declare adaptive marginal-utility control`) + const existing = state.adaptive.checkpoints.find((item) => item.round === value.round) + if (existing) { + const previous = { + round: existing.round, + utility: existing.utility, + uncertainty: existing.uncertainty, + evidenceRefs: existing.evidenceRefs, + evaluatedAt: existing.evaluatedAt, + } + const submitted = { + round: value.round, + utility: value.utility, + uncertainty: value.uncertainty, + evidenceRefs: value.evidenceRefs, + evaluatedAt: value.evaluatedAt, + } + if (JSON.stringify(previous) === JSON.stringify(submitted)) return state + throw new Error(`Adaptive checkpoint for round ${value.round} is immutable`) + } + if (state.adaptive.phase !== "searching") throw new Error(`Adaptive search is already finalizing`) + const expected = state.adaptive.checkpoints.length + 1 + if (value.round !== expected) throw new Error(`Expected adaptive checkpoint for round ${expected}`) + if (due(state) !== value.round) throw new Error(`Adaptive round ${value.round} has not completed`) + const completed = state.order + .map((id) => state.work[id]!) + .filter((item) => item.round === value.round && evolving(item.role)) + .map((item) => item.result!.completedAt) + const now = Date.now() + if (value.evaluatedAt < Math.max(bound.createdAt, ...completed)) { + throw new Error(`Adaptive checkpoint predates the completed round`) + } + if (value.evaluatedAt > now + 300_000) throw new Error(`Adaptive checkpoint timestamp is implausibly far ahead`) + const previous = state.adaptive.checkpoints.at(-1) + const gain = previous ? value.utility - previous.utility : null + const qualified = value.uncertainty <= state.adaptive.maxUncertainty + const stalled = !qualified + ? 0 + : gain === null + ? 0 + : gain < state.adaptive.minUtilityGain + ? state.adaptive.stalled + 1 + : 0 + const target = + qualified && + value.round >= state.adaptive.minRounds && + state.adaptive.targetUtility !== undefined && + value.utility >= state.adaptive.targetUtility + const exhausted = qualified && value.round >= state.adaptive.minRounds && stalled >= state.adaptive.patience + const reason = target + ? "target_reached" + : exhausted + ? "marginal_utility_exhausted" + : value.round === state.maxRounds + ? "max_rounds" + : undefined + const record = { + round: value.round, + utility: value.utility, + uncertainty: value.uncertainty, + evidenceRefs: value.evidenceRefs, + evaluatedAt: value.evaluatedAt, + gain, + qualified, + recordedAt: now, + } + const checkpoint = Checkpoint.parse({ + id: checkpointID(state.contractFingerprint, state.sessionID, record), + ...record, + }) + const adaptive: Adaptive = { + ...state.adaptive, + checkpoints: [...state.adaptive.checkpoints, checkpoint], + stalled, + phase: reason ? "finalizing" : "searching", + stopReason: reason, + } + const updated = State.parse({ + ...state, + adaptive, + status: "active", + revision: state.revision + 1, + updatedAt: now, + }) + const next = reason && value.round < state.maxRounds ? finale(updated, bound, value.round) : updated + return settle(next, now) + }) + return read(value.sessionID) + } + + export async function complete(input: { + sessionID: string + workID: string + workerSessionID: string + result: z.input + }) { + const submission = Submission.parse(input.result) + const contract = await HarnessContract.read(input.sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${input.sessionID}`) + await JsonStore.update(file(input.sessionID), (data) => { + const state = verify(parse(data), contract) + const work = state.work[input.workID] + if (!work) throw new Error(`Unknown orchestration work ${input.workID}`) + const usage = work.workerReceipt?.usage ?? submission.usage + const submitted = { ...submission, ...(usage ? { usage } : {}) } + if (work.status === "completed") { + const previous = { + summary: work.result!.summary, + artifactRefs: work.result!.artifactRefs, + evidenceRefs: work.result!.evidenceRefs, + usage: work.result!.usage, + verdict: work.result!.verdict, + } + if (work.workerSessionID === input.workerSessionID && JSON.stringify(previous) === JSON.stringify(submitted)) { + return state + } + throw new Error(`Completed orchestration work is immutable`) + } + if (state.workerPolicy === "task-attested-v1" && work.status !== "executed") { + throw new Error(`Orchestration work must be executed by the Task tool before completion`) + } + if (state.workerPolicy === "claimed-v1" && work.status !== "pending") { + throw new Error(`Orchestration work ${input.workID} is not pending`) + } + if (!work.dependencies.every((dependency) => state.work[dependency]?.status === "completed")) { + throw new Error(`Orchestration work cannot complete before its dependencies`) + } + if (state.workerPolicy === "claimed-v1") worker(state, work, input.workerSessionID) + if (state.workerPolicy === "task-attested-v1" && work.workerSessionID !== input.workerSessionID) { + throw new Error(`Coalition completion does not match the Task execution receipt`) + } + if (work.workerReceipt?.outcome === "failed") { + throw new Error(`A failed Task execution cannot complete coalition work`) + } + if ( + work.workerReceipt && + submission.usage && + JSON.stringify(submission.usage) !== JSON.stringify(work.workerReceipt.usage) + ) { + throw new Error(`Coalition usage does not match the Task execution receipt`) + } + if (work.role === "verification" && !submission.verdict) { + throw new Error(`Verification work requires a structured verdict`) + } + if (work.role === "verification" && !submission.evidenceRefs.length) { + throw new Error(`Verification work requires observable evidence references`) + } + if (state.repair && work.role === "verification" && !submission.verdict?.severity) { + throw new Error(`Verifier-routed repair requires a severity classification`) + } + if (work.role !== "verification" && submission.verdict) { + throw new Error(`Only verification work may submit a verdict`) + } + const now = Date.now() + const result = Result.parse({ ...submitted, completedAt: now }) + within(result.usage, work.allocation) + const next: State = { + ...state, + work: { + ...state.work, + [work.id]: { ...work, status: "completed", workerSessionID: input.workerSessionID, result }, + }, + revision: state.revision + 1, + updatedAt: now, + } + return settle(advance(next, contract, now), now) + }) + return read(input.sessionID) + } + + export async function fail(input: { sessionID: string; workID: string; workerSessionID: string; failure: string }) { + const contract = await HarnessContract.read(input.sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${input.sessionID}`) + await JsonStore.update(file(input.sessionID), (data) => { + const state = verify(parse(data), contract) + const work = state.work[input.workID] + if (!work) throw new Error(`Unknown orchestration work ${input.workID}`) + if (work.status === "failed") { + if (work.workerSessionID === input.workerSessionID && work.failure === input.failure) return state + throw new Error(`Failed orchestration work is immutable`) + } + if (state.workerPolicy === "task-attested-v1" && work.status !== "executed") { + throw new Error(`Orchestration work must be executed by the Task tool before failure settlement`) + } + if (state.workerPolicy === "claimed-v1" && work.status !== "pending") { + throw new Error(`Orchestration work ${input.workID} is not pending`) + } + if (!work.dependencies.every((dependency) => state.work[dependency]?.status === "completed")) { + throw new Error(`Orchestration work cannot fail before its dependencies`) + } + if (state.workerPolicy === "claimed-v1") worker(state, work, input.workerSessionID) + if (state.workerPolicy === "task-attested-v1" && work.workerSessionID !== input.workerSessionID) { + throw new Error(`Coalition failure does not match the Task execution receipt`) + } + const now = Date.now() + const next: State = { + ...state, + work: { + ...state.work, + [work.id]: { + ...work, + status: "failed", + workerSessionID: input.workerSessionID, + failure: z.string().min(1).max(4_000).parse(input.failure), + }, + }, + revision: state.revision + 1, + updatedAt: now, + } + return settle(advance(next, contract, now), now) + }) + return read(input.sessionID) + } +} diff --git a/backend/cli/src/session/harness/pack.ts b/backend/cli/src/session/harness/pack.ts new file mode 100644 index 00000000..3738eb9c --- /dev/null +++ b/backend/cli/src/session/harness/pack.ts @@ -0,0 +1,6 @@ +import z from "zod" + +export namespace HarnessPack { + export const Id = z.enum(["statistics", "biology", "physics", "pde", "chemistry", "ml", "forecast", "formal"]) + export type Id = z.infer +} diff --git a/backend/cli/src/session/harness/profile.ts b/backend/cli/src/session/harness/profile.ts new file mode 100644 index 00000000..2a5050d7 --- /dev/null +++ b/backend/cli/src/session/harness/profile.ts @@ -0,0 +1,111 @@ +import { HarnessContract } from "./contract" + +export namespace HarnessProfile { + export type Selection = { + id: HarnessContract.Profile + source: "contract" | "heuristic" | "control" + confidence: number + reasons: string[] + prompt: string + } + + const instructions: Record = { + react: [ + "Use the direct ReAct control: take the smallest reliable path to the requested result.", + "Do not create candidate populations, review panels, or process artifacts unless the task itself requires them.", + ], + optimize: [ + "Treat this as evaluator-driven optimization. Pin the metric, direction, budget, and baseline before changing the candidate.", + "Preserve candidate lineage and failed approaches; explore distinct branches before exploiting the strongest measured branch.", + "Never call a candidate better without running the declared evaluator, and stop when the budget or terminal criterion is reached.", + ], + reproduce: [ + "Treat this as research reproduction. Extract the target claims, protocol, inputs, outputs, and success criteria before execution.", + "Keep claims linked to observable evidence and finish with an independent clean replay or an explicit reproducibility blocker.", + ], + theory: [ + "Treat this as theoretical physics. Track assumptions, units, sign conventions, limiting cases, and the exact quantity to derive.", + "Use an independent derivation or adversarial check before accepting the final result; disagreement remains visible until resolved.", + ], + numerical: [ + "Treat this as numerical science. Pin equations, domains, boundary and initial conditions, discretization, tolerances, and invariants.", + "Require a convergence, stability, conservation, manufactured-solution, or known-limit check appropriate to the claimed result.", + ], + training: [ + "Treat this as a bounded training experiment. Validate data splits, model identity, chat template, decoding, seed, and baseline first.", + "Checkpoint recoverably, compare methods through the declared evaluator, and audit leakage or benchmark-targeted data before claiming progress.", + ], + forecast: [ + "Treat this as forecast-model evaluation. Pin dataset, initialization, variables, region, resolution, lead times, and deterministic or probabilistic mode.", + "Report the full metric portfolio and compute budget; do not collapse incompatible forecast settings into one SOTA claim.", + ], + } + + const prompt = (selection: Omit) => + [ + ``, + ...instructions[selection.id].map((line) => `- ${line}`), + "This profile refines the normal OpenScience contract; safety, user instructions, and actual tool evidence remain authoritative.", + "", + ].join("\n") + + const select = (selection: Omit): Selection => ({ ...selection, prompt: prompt(selection) }) + + export function classify(input: { agent?: string; text: string; contract?: HarnessContract.Info | null }): Selection { + if (input.contract) { + return select({ + id: input.contract.profile, + source: "contract", + confidence: 1, + reasons: [`contract:${input.contract.runID}`], + }) + } + + const text = input.text.toLowerCase() + const agent = input.agent ?? "research" + const has = (pattern: RegExp) => pattern.test(text) + const reproduction = + has(/\b(reproduce|replicate|rediscover|re-create)\b/) && has(/\b(paper|study|result|experiment)\b/) + if (reproduction) { + return select({ id: "reproduce", source: "heuristic", confidence: 0.94, reasons: ["reproduction-language"] }) + } + + const weather = + has(/\b(weather|forecast|forecasting|nowcast)\b/) && has(/\b(model|train|evaluate|benchmark|skill)\b/) + if ((agent === "ml" || agent === "research") && weather) { + return select({ id: "forecast", source: "heuristic", confidence: 0.92, reasons: ["forecast-contract"] }) + } + + const training = has(/\b(post[- ]?train|fine[- ]?tun|rlhf|grpo|dpo|sft|preference training)\b/) + if ((agent === "ml" || agent === "research") && training) { + return select({ id: "training", source: "heuristic", confidence: 0.9, reasons: ["training-language"] }) + } + + const numerical = has(/\b(pde|finite element|finite volume|spectral method|numerical simulation|cfd|solver)\b/) + if ((agent === "physics" || agent === "research") && numerical) { + return select({ id: "numerical", source: "heuristic", confidence: 0.9, reasons: ["numerical-physics-language"] }) + } + + const theory = + has(/\b(derive|derivation|prove|proof|theoretical)\b/) && + has(/\b(physics|hamiltonian|lagrangian|field theory|quantum|relativ|thermodynamic)\b/) + if ((agent === "physics" || agent === "research") && theory) { + return select({ id: "theory", source: "heuristic", confidence: 0.86, reasons: ["theory-language"] }) + } + + const objective = + has(/\b(leaderboard|kaggle|submission)\b/) || + (has(/\bbenchmark\b/) && has(/\b(metric|score|objective|medal|accuracy|loss|reward)\b/)) + const optimize = has(/\b(optimi[sz]e|improve|iterate|evolve|search)\b/) && objective + if (optimize) { + return select({ id: "optimize", source: "heuristic", confidence: 0.88, reasons: ["measurable-optimization"] }) + } + + return select({ id: "react", source: "control", confidence: 1, reasons: ["conservative-default"] }) + } + + export async function resolve(input: { sessionID: string; agent?: string; text: string }) { + const contract = await HarnessContract.read(input.sessionID) + return classify({ ...input, contract }) + } +} diff --git a/backend/cli/src/session/harness/replication.ts b/backend/cli/src/session/harness/replication.ts new file mode 100644 index 00000000..6d6380fe --- /dev/null +++ b/backend/cli/src/session/harness/replication.ts @@ -0,0 +1,516 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessReplication { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(240), + }) + .strict() + export type Subject = z.infer + + export const Observation = z + .object({ + stratumID: z.string().min(1).max(120), + clusterID: z.string().min(1).max(120), + stratumSHA256: Hash, + clusterSHA256: Hash, + status: z.enum(["passed", "failed", "inconclusive"]), + score: z.number().finite().optional(), + outputSHA256: Hash, + environmentSHA256: Hash, + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + evaluatedAt: z.number().int().positive(), + }) + .strict() + export type Observation = z.infer + + export const Submit = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + subject: Subject, + observations: z.array(Observation).min(3).max(512), + }) + .strict() + export type Submit = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + export type Access = z.infer + + const Statistics = z + .object({ + units: z.number().int().positive(), + passed: z.number().int().nonnegative(), + failed: z.number().int().nonnegative(), + inconclusive: z.number().int().nonnegative(), + estimator: HarnessContract.ReplicationEstimator, + estimate: z.number().finite().optional(), + confidence: z.literal(0.95), + interval: z.tuple([z.number().finite(), z.number().finite()]).optional(), + intervalWidth: z.number().finite().nonnegative().optional(), + conservativeBound: z.number().finite().optional(), + method: z.enum(["stratified-bootstrap-percentile-v1", "wilson-score-v1"]), + resamples: z.number().int().positive().optional(), + }) + .strict() + + const ReceiptBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("replicated-evaluation-receipt-v1"), + receiptID: Hash, + protocolSHA256: Hash, + contractSHA256: Hash, + sourceSessionID: z.string().min(1).max(240), + subject: Subject, + metric: z.string().min(1).max(200), + protocol: HarnessContract.Replication, + observations: z.array(Observation).min(3).max(512), + statistics: Statistics, + status: z.enum(["passed", "failed", "inconclusive"]), + failures: z.array(z.string().min(1).max(500)).max(1_024), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(16_384), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + + export const Receipt = ReceiptBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + if (digest(stable) === value.receiptID) return + ctx.addIssue({ + code: "custom", + path: ["receiptID"], + message: "Replicated evaluation receipt content hash is invalid", + }) + }) + export type Receipt = z.infer + + const Claim = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + protocolSHA256: Hash, + contractSHA256: Hash, + sourceSessionID: z.string().min(1).max(240), + subject: Subject, + }) + .strict() + + const root = path.join(Global.Path.data, "harness", "replications") + const file = (receiptID: string) => path.join(root, `${receiptID}.json`) + const claimfile = (contract: HarnessContract.Info, subject: Subject) => + path.join(root, "subjects", digest(contract.sessionID), `${digest(subject)}.json`) + const key = (input: Pick) => `${input.stratumID}\0${input.clusterID}` + const sorted = (input: Observation[]) => + input.toSorted( + (left, right) => left.stratumID.localeCompare(right.stratumID) || left.clusterID.localeCompare(right.clusterID), + ) + + function average(input: Array<{ value: number; weight: number }>) { + const scale = Math.max(...input.map((item) => Math.abs(item.value))) + if (scale === 0) return 0 + const total = input.reduce((sum, item) => sum + item.weight, 0) + const state = { sum: 0, correction: 0 } + for (const item of input) { + const value = (item.value / scale) * item.weight + const next = state.sum + value + state.correction += Math.abs(state.sum) >= Math.abs(value) ? state.sum - next + value : value - next + state.sum + state.sum = next + } + return ((state.sum + state.correction) / total) * scale + } + + function estimate(kind: Exclude, input: number[]) { + const values = input.toSorted((left, right) => left - right) + if (kind === "mean") return average(values.map((value) => ({ value, weight: 1 }))) + if (kind === "median") { + const middle = Math.floor(values.length / 2) + return values.length % 2 + ? values[middle]! + : average([ + { value: values[middle - 1]!, weight: 1 }, + { value: values[middle]!, weight: 1 }, + ]) + } + return average( + values.flatMap((value, index) => { + const lower = Math.max(index / values.length, 0.25) + const upper = Math.min((index + 1) / values.length, 0.75) + const weight = Math.max(0, upper - lower) + return weight === 0 ? [] : [{ value, weight }] + }), + ) + } + + function quantile(input: number[], probability: number) { + const values = input.toSorted((left, right) => left - right) + const position = (values.length - 1) * probability + const lower = Math.floor(position) + const upper = Math.ceil(position) + if (lower === upper) return values[lower]! + const weight = position - lower + return average([ + { value: values[lower]!, weight: 1 - weight }, + { value: values[upper]!, weight }, + ]) + } + + function bootstrap(protocol: HarnessContract.Replication, observations: Observation[]) { + if (protocol.interval.method !== "stratified-bootstrap-percentile-v1") { + throw new Error(`Numeric replicated evaluation requires a stratified bootstrap interval`) + } + if (protocol.estimator === "pass_rate") throw new Error(`Pass-rate evaluation cannot use numeric bootstrap`) + const estimator = protocol.estimator + const scores = new Map(observations.map((item) => [key(item), item.score!])) + const state = { value: protocol.interval.seed >>> 0 } + const random = () => { + state.value = (state.value + 0x6d2b79f5) >>> 0 + const first = Math.imul(state.value ^ (state.value >>> 15), 1 | state.value) + const second = first + Math.imul(first ^ (first >>> 7), 61 | first) + return ((second ^ (second >>> 14)) >>> 0) / 4294967296 + } + const draws = Array.from({ length: protocol.interval.resamples }, () => { + const strata = Array.from( + { length: protocol.sampling.strata.length }, + () => protocol.sampling.strata[Math.floor(random() * protocol.sampling.strata.length)]!, + ) + return estimate( + estimator, + strata.flatMap((stratum) => + Array.from({ length: protocol.sampling.clusters.length }, () => { + const cluster = protocol.sampling.clusters[Math.floor(random() * protocol.sampling.clusters.length)]! + return scores.get(`${stratum.id}\0${cluster.id}`)! + }), + ), + ) + }) + const alpha = 1 - protocol.interval.confidence + return [quantile(draws, alpha / 2), quantile(draws, 1 - alpha / 2)] as [number, number] + } + + function wilson(passed: number, total: number) { + const probability = passed / total + const z = 1.959963984540054 + const scale = 1 + (z * z) / total + const center = (probability + (z * z) / (2 * total)) / scale + const radius = (z * Math.sqrt((probability * (1 - probability)) / total + (z * z) / (4 * total * total))) / scale + return [Math.max(0, center - radius), Math.min(1, center + radius)] as [number, number] + } + + function evidence(observations: Observation[]) { + return [...new Set(observations.flatMap((item) => item.evidence))].toSorted() + } + + function inspect(protocol: HarnessContract.Replication, input: Observation[]) { + const observations = sorted(input.map((item) => Observation.parse(item))) + if (new Set(observations.map(key)).size !== observations.length) { + throw new Error(`Replicated evaluation units must be unique`) + } + const expected = protocol.sampling.strata + .flatMap((stratum) => protocol.sampling.clusters.map((cluster) => `${stratum.id}\0${cluster.id}`)) + .toSorted() + if (JSON.stringify(observations.map(key)) !== JSON.stringify(expected)) { + throw new Error(`Replicated evaluation must contain the complete frozen stratum-cluster grid`) + } + for (const observation of observations) { + const stratum = protocol.sampling.strata.find((item) => item.id === observation.stratumID)! + const cluster = protocol.sampling.clusters.find((item) => item.id === observation.clusterID)! + if ( + observation.stratumSHA256 !== stratum.commitmentSHA256 || + observation.clusterSHA256 !== cluster.commitmentSHA256 + ) { + throw new Error(`Replicated evaluation unit ${key(observation)} changed a frozen axis commitment`) + } + if (observation.environmentSHA256 !== protocol.environmentSHA256) { + throw new Error(`Replicated evaluation unit ${key(observation)} changed the frozen environment`) + } + if (protocol.estimator === "pass_rate" && observation.score !== undefined) { + throw new Error(`Pass-rate observations cannot submit agent-selected numeric scores`) + } + if (protocol.estimator !== "pass_rate" && observation.status === "passed" && observation.score === undefined) { + throw new Error(`Passing numeric replicate ${key(observation)} must report a score`) + } + if (protocol.estimator !== "pass_rate" && observation.status !== "passed" && observation.score !== undefined) { + throw new Error(`Non-passing numeric replicate ${key(observation)} cannot contribute a score`) + } + } + const passed = observations.filter((item) => item.status === "passed").length + const failed = observations.filter((item) => item.status === "failed").length + const inconclusive = observations.length - passed - failed + const base = { + units: observations.length, + passed, + failed, + inconclusive, + estimator: protocol.estimator, + confidence: 0.95 as const, + method: protocol.interval.method, + ...(protocol.interval.method === "stratified-bootstrap-percentile-v1" + ? { resamples: protocol.interval.resamples } + : {}), + } + if (protocol.estimator !== "pass_rate" && (failed || inconclusive)) { + const status = failed ? ("failed" as const) : ("inconclusive" as const) + const failures = observations.flatMap((item) => + item.status === "passed" ? [] : [`unit:${item.stratumID}/${item.clusterID}:${item.status}`], + ) + return { observations, statistics: Statistics.parse(base), status, failures } + } + const values = observations.flatMap((item) => (item.score === undefined ? [] : [item.score])) + const point = + protocol.estimator === "pass_rate" ? passed / observations.length : estimate(protocol.estimator, values) + const interval = + protocol.estimator === "pass_rate" ? wilson(passed, observations.length) : bootstrap(protocol, observations) + const width = interval[1] - interval[0] + const bound = protocol.decision.direction === "minimize" ? interval[1] : interval[0] + const target = + protocol.decision.direction === "minimize" ? bound <= protocol.decision.target : bound >= protocol.decision.target + const precise = protocol.decision.maxIntervalWidth === undefined || width <= protocol.decision.maxIntervalWidth + const uncertain = inconclusive > 0 + const failures = [ + ...(uncertain ? [`${inconclusive} replicated units were inconclusive`] : []), + ...(!target + ? [ + `conservative bound ${bound} does not satisfy ${protocol.decision.direction} target ${protocol.decision.target}`, + ] + : []), + ...(!precise ? [`interval width ${width} exceeds ${protocol.decision.maxIntervalWidth}`] : []), + ] + const status = uncertain ? ("inconclusive" as const) : target && precise ? ("passed" as const) : ("failed" as const) + return { + observations, + statistics: Statistics.parse({ + ...base, + estimate: point, + interval, + intervalWidth: width, + conservativeBound: bound, + }), + status, + failures, + } + } + + async function born(contract: HarnessContract.Info, subject: Subject) { + if (subject.type === "run") { + if (subject.id !== contract.runID) + throw new Error(`Replicated evaluation run subject does not match its contract`) + return contract.createdAt + } + const state = await import("./search") + .then((module) => module.HarnessSearch.read(contract.sessionID)) + .catch(() => null) + const candidate = state?.runID === contract.runID ? state.candidates[subject.id] : undefined + if (!candidate) throw new Error(`Replicated evaluation candidate does not exist in the bound search`) + return candidate.createdAt + } + + function sameSubject(subject: Subject, input?: { type: "run" | "candidate"; id: string }) { + if (subject.type === "candidate") return input?.type === "candidate" && input.id === subject.id + return input === undefined || (input.type === "run" && input.id === subject.id) + } + + async function claim(contract: HarnessContract.Info, subject: Subject) { + const data = await JsonStore.read(claimfile(contract, subject)) + if (!Object.keys(data).length) return null + return Claim.parse(data) + } + + export function prompt(contract: HarnessContract.Info) { + const protocol = contract.replication + if (!protocol) return "" + const units = protocol.sampling.strata.length * protocol.sampling.clusters.length + return [ + "", + "The evaluator froze this uncertainty protocol before execution. Treat it as evaluation policy, not evidence that a result passes.", + `Evaluate the complete ${protocol.sampling.strata.length} ${protocol.sampling.stratumKind} × ${protocol.sampling.clusters.length} ${protocol.sampling.clusterKind} clusters (${units} units) using its exact axis commitments; no subset, duplicate, substitution, or post-hoc unit is admissible.`, + `Estimator: ${protocol.estimator}; interval: ${protocol.interval.method} at 95% confidence.`, + `All units must use the frozen environment commitment ${protocol.environmentSHA256}.`, + `Promotion rule: ${protocol.decision.rule}; direction=${protocol.decision.direction}; target=${protocol.decision.target}${protocol.decision.maxIntervalWidth === undefined ? "" : `; max_interval_width=${protocol.decision.maxIntervalWidth}`}.`, + "A final score must equal the backend-derived aggregate. A passing result must satisfy the conservative confidence bound, not the best replicate or point estimate.", + "Do not split repeated measurements into fake independent clusters, tune on frozen units, omit failures, or claim replication compliance yourself; produce immutable outputs and evidence for the evaluator.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + return contract ? prompt(contract) : "" + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const protocol = contract.replication + if (!protocol) throw new Error(`Harness contract does not require replicated evaluation`) + if (value.sessionID !== contract.sessionID) + throw new Error(`Replicated evaluation session does not match its contract`) + const createdAt = await born(contract, value.subject) + const existing = await import("./evaluation").then((module) => module.HarnessEvaluation.list(contract.sessionID)) + if (existing.some((item) => moduleFinal(item) && sameSubject(value.subject, item.subject))) { + throw new Error(`Replicated evaluation receipt must be recorded before the subject's final evaluation`) + } + const now = Date.now() + for (const observation of value.observations) { + if (observation.evaluatedAt < createdAt || observation.evaluatedAt > now) { + throw new Error(`Replicated observation timestamp is outside the bound subject interval`) + } + } + const audit = inspect(protocol, value.observations) + const metric = contract.benchmark.metric + if (!metric) throw new Error(`Replicated evaluation contract has no bound metric`) + const claimed = await claim(contract, value.subject) + if (claimed) { + const current = await read(claimed.receiptID) + if (!current) throw new Error(`The subject's frozen replicated evaluation receipt is corrupt`) + if ( + current.protocolSHA256 !== digest(protocol) || + current.contractSHA256 !== HarnessContract.fingerprint(contract) || + current.sourceSessionID !== contract.sessionID || + !sameSubject(value.subject, current.subject) + ) { + throw new Error(`The subject's frozen replicated evaluation claim does not match its bound contract`) + } + if (JSON.stringify(current.observations) === JSON.stringify(audit.observations)) return current + throw new Error(`The subject already has a frozen replicated evaluation receipt; selective retries are forbidden`) + } + const stable = { + schemaVersion: 1 as const, + protocolVersion: "replicated-evaluation-receipt-v1" as const, + protocolSHA256: digest(protocol), + contractSHA256: HarnessContract.fingerprint(contract), + sourceSessionID: contract.sessionID, + subject: value.subject, + metric, + protocol, + observations: audit.observations, + statistics: audit.statistics, + status: audit.status, + failures: audit.failures, + evidence: evidence(audit.observations), + evaluatedAt: Math.max(...audit.observations.map((item) => item.evaluatedAt)), + recordedAt: now, + } + const receipt = Receipt.parse({ ...stable, receiptID: digest(stable) }) + await JsonStore.update(file(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Replicated evaluation receipt is immutable once recorded`) + }) + const saved = await read(receipt.receiptID) + if (!saved) throw new Error(`Replicated evaluation receipt was not durable after recording`) + const statement = Claim.parse({ + schemaVersion: 1, + receiptID: saved.receiptID, + protocolSHA256: saved.protocolSHA256, + contractSHA256: saved.contractSHA256, + sourceSessionID: saved.sourceSessionID, + subject: saved.subject, + }) + await JsonStore.update(claimfile(contract, value.subject), async (data) => { + if (!Object.keys(data).length) return statement + const current = Claim.parse(data) + if (current.receiptID === statement.receiptID) return current + const winner = await read(current.receiptID) + if (winner && JSON.stringify(winner.observations) === JSON.stringify(audit.observations)) return current + throw new Error(`The subject already has a frozen replicated evaluation receipt; selective retries are forbidden`) + }) + const active = await claim(contract, value.subject) + if (active?.receiptID !== saved.receiptID) { + const winner = active ? await read(active.receiptID) : null + if (winner && JSON.stringify(winner.observations) === JSON.stringify(audit.observations)) return winner + throw new Error(`Replicated evaluation receipt was not durably frozen for its subject`) + } + return saved + } + + const moduleFinal = (input: { fidelity?: { final: boolean } }) => input.fidelity?.final !== false + + export async function read(receiptID: string) { + const id = Hash.parse(receiptID) + const data = await JsonStore.read(file(id)) + const parsed = Receipt.safeParse(data) + return parsed.success && parsed.data.receiptID === id ? parsed.data : null + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: Subject + score?: number + evaluatedAt: number + recordedAt: number + requirePassed: boolean + }) { + const receipt = await read(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt replicated evaluation receipt ${input.receiptID}`) + const protocol = input.contract.replication + if (!protocol) throw new Error(`Evaluation cites a replication receipt without a bound replication protocol`) + if (receipt.protocolSHA256 !== digest(protocol) || JSON.stringify(receipt.protocol) !== JSON.stringify(protocol)) { + throw new Error(`Replicated evaluation receipt does not match the bound protocol`) + } + if (receipt.sourceSessionID !== input.contract.sessionID) { + throw new Error(`Replicated evaluation receipt belongs to a different harness session`) + } + if (receipt.contractSHA256 !== HarnessContract.fingerprint(input.contract)) { + throw new Error(`Replicated evaluation receipt does not match the bound contract`) + } + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Replicated evaluation receipt belongs to a different evaluation subject`) + } + if (receipt.metric !== input.contract.benchmark.metric) { + throw new Error(`Replicated evaluation receipt changed the bound metric`) + } + const active = await claim(input.contract, input.subject) + if (active?.receiptID !== receipt.receiptID) { + throw new Error(`Evaluation cites a non-canonical receipt instead of the subject's frozen receipt`) + } + const createdAt = await born(input.contract, input.subject) + const audit = inspect(protocol, receipt.observations) + if ( + receipt.status !== audit.status || + JSON.stringify(receipt.failures) !== JSON.stringify(audit.failures) || + JSON.stringify(receipt.statistics) !== JSON.stringify(audit.statistics) || + JSON.stringify(receipt.evidence) !== JSON.stringify(evidence(audit.observations)) || + receipt.evaluatedAt !== Math.max(...audit.observations.map((item) => item.evaluatedAt)) + ) { + throw new Error(`Replicated evaluation receipt does not match backend-derived uncertainty state`) + } + if ( + audit.observations.some( + (observation) => observation.evaluatedAt < createdAt || observation.evaluatedAt > receipt.recordedAt, + ) + ) { + throw new Error(`Replicated evaluation receipt contains an observation outside the bound subject interval`) + } + if (receipt.evaluatedAt > input.evaluatedAt || receipt.recordedAt > input.recordedAt) { + throw new Error(`Evaluation predates its replicated evaluation receipt`) + } + if (input.score !== undefined && receipt.statistics.estimate !== input.score) { + throw new Error(`Final evaluation score does not match the backend-derived replicated estimate`) + } + if (input.requirePassed && input.score === undefined) { + throw new Error(`A passing replicated evaluation must report the backend-derived aggregate score`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A passing final evaluation requires a passing conservative replication receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/report.ts b/backend/cli/src/session/harness/report.ts new file mode 100644 index 00000000..ad28128d --- /dev/null +++ b/backend/cli/src/session/harness/report.ts @@ -0,0 +1,548 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { HarnessAdaptation } from "./adaptation" +import { HarnessAutonomy } from "./autonomy" +import { HarnessBlueprint } from "./blueprint" +import { HarnessConfirmation } from "./confirmation" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" +import { HarnessFormal } from "./formal" +import { HarnessMeta } from "./meta" +import { HarnessSearch } from "./search" +import { SessionTrace } from "../trace" + +export namespace HarnessReport { + const Tokens = z + .object({ + input: z.number().nonnegative(), + output: z.number().nonnegative(), + reasoning: z.number().nonnegative(), + cacheRead: z.number().nonnegative(), + cacheWrite: z.number().nonnegative(), + total: z.number().nonnegative(), + }) + .strict() + + export const Info = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1), + sessionID: z.string().min(1), + contractFingerprint: z.string().regex(/^[a-f0-9]{64}$/), + comparisonKey: z.string().regex(/^[a-f0-9]{64}$/), + benchmark: z + .object({ + id: z.string().min(1), + title: z.string().min(1), + family: HarnessContract.Family, + version: z.string().min(1), + taskID: z.string().min(1), + split: HarnessContract.Split, + }) + .strict(), + execution: z + .object({ + profile: HarnessContract.Profile, + packs: z.array(z.string()), + provider: z.string().min(1), + model: z.string().min(1), + effort: z.string().optional(), + intervention: z.enum(["autonomous", "human_reprompted"]), + autonomy: z + .object({ + claimedLevel: HarnessContract.AutonomyLevel, + derivedLevel: HarnessContract.AutonomyLevel.optional(), + status: z.enum(["passed", "failed", "inconclusive"]).optional(), + }) + .strict() + .optional(), + formal: z + .object({ + tier: HarnessContract.FormalTier, + relation: HarnessContract.FormalRelation, + status: z.enum(["passed", "failed"]).optional(), + blueprint: HarnessBlueprint.Summary.optional(), + }) + .strict() + .optional(), + seed: z.number().int(), + }) + .strict(), + quality: z + .object({ + source: z.enum(["optimization", "sealed_confirmation"]), + provisional: z.boolean(), + status: HarnessEvaluation.Status.optional(), + metric: z.string().optional(), + direction: z.enum(["maximize", "minimize", "pass"]), + score: z.number().finite().optional(), + target: z.number().finite().optional(), + targetReached: z.boolean(), + evaluator: z.string().min(1), + evaluatorVersion: z.string().optional(), + simulationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + integrityReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evolutionReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + interventionReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evaluatorAuditReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + semanticReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + replicationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + auditReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + failureDiscoveryReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + synthesisReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + autonomyReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + proofReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + metaReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + confirmationReceiptID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + evaluations: z.number().int().nonnegative(), + }) + .strict(), + efficiency: z + .object({ + costUSD: z.number().nonnegative().optional(), + evaluatorCostUSD: z.number().nonnegative().optional(), + tokens: Tokens.optional(), + wallTimeMs: z.number().nonnegative().optional(), + evaluatorWallTimeMs: z.number().nonnegative().optional(), + toolCalls: z.number().int().nonnegative().optional(), + searches: z.number().int().nonnegative().optional(), + dedupeHits: z.number().int().nonnegative().optional(), + retries: z.number().int().nonnegative().optional(), + failures: z.number().int().nonnegative().optional(), + candidates: z.number().int().nonnegative().optional(), + }) + .strict(), + search: z + .object({ + status: z.enum(["active", "completed"]), + stopReason: HarnessSearch.Stop.optional(), + bestID: z.string().optional(), + candidates: z.number().int().nonnegative(), + verified: z.number().int().nonnegative(), + generations: z.number().int().nonnegative(), + stalled: z.number().int().nonnegative(), + proposalPolicy: z.enum(["advisory-v2", "leased-v3", "adaptive-v4"]), + controller: HarnessContract.Search.optional(), + adaptation: HarnessAdaptation.Summary.optional(), + objectives: HarnessContract.Objectives, + objectiveAudit: HarnessContract.ObjectiveAudit.optional(), + archive: z.number().int().nonnegative(), + }) + .strict() + .optional(), + metaHarness: z + .object({ + status: HarnessEvaluation.Status, + selectionID: z.string().regex(/^[a-f0-9]{64}$/), + diagnostics: HarnessMeta.Diagnostics, + failures: z.array(z.string()), + }) + .strict() + .optional(), + generatedAt: z.number().int().positive(), + }) + .strict() + export type Info = z.infer + + export type Trace = Pick< + SessionTrace.Info["summary"], + | "cost" + | "tokens" + | "totalCompletionTimeMs" + | "toolCalls" + | "searchCount" + | "dedupeHits" + | "retryCount" + | "failureCount" + > + + const root = path.join(Global.Path.data, "harness", "reports") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + const reached = (contract: HarnessContract.Info, result?: { status: HarnessEvaluation.Status; score?: number }) => { + if (result?.status !== "passed") return false + if (contract.benchmark.direction === "pass" || contract.benchmark.direction === undefined) return true + if (result.score === undefined || contract.benchmark.target === undefined) return false + if (contract.benchmark.direction === "maximize") return result.score >= contract.benchmark.target + return result.score <= contract.benchmark.target + } + + const selected = (evaluations: HarnessEvaluation.Info[], search?: HarnessSearch.State) => { + const best = search?.bestID + ? evaluations.findLast( + (item) => + item.subject?.type === "candidate" && item.subject.id === search.bestID && HarnessEvaluation.final(item), + ) + : undefined + return ( + best ?? + evaluations.findLast((item) => !item.subject && HarnessEvaluation.final(item)) ?? + evaluations.findLast(HarnessEvaluation.final) + ) + } + + export function compile(input: { + contract: HarnessContract.Info + evaluations: HarnessEvaluation.Info[] + trace?: Trace + search?: HarnessSearch.State + meta?: HarnessMeta.Receipt + confirmation?: HarnessConfirmation.Receipt + autonomy?: HarnessAutonomy.Receipt + formal?: HarnessFormal.Receipt + blueprint?: HarnessBlueprint.Summary + generatedAt?: number + }) { + const contract = HarnessContract.Info.parse(input.contract) + const evaluations = input.evaluations.map((item) => HarnessEvaluation.Info.parse(item)) + const evaluation = selected(evaluations, input.search) + const autonomy = input.autonomy ? HarnessAutonomy.Receipt.parse(input.autonomy) : undefined + if (autonomy && autonomy.receiptID !== evaluation?.autonomyReceiptID) { + throw new Error(`Human-AI autonomy receipt does not match the selected evaluation`) + } + const formal = input.formal ? HarnessFormal.Receipt.parse(input.formal) : undefined + if (formal && formal.receiptID !== evaluation?.proofReceiptID) { + throw new Error(`Formal proof receipt does not match the selected evaluation`) + } + const blueprint = input.blueprint ? HarnessBlueprint.bind(contract, input.blueprint) : undefined + const meta = input.meta ? HarnessMeta.bind(contract, input.meta) : undefined + if (meta && meta.selection.candidateID !== input.search?.bestID) { + throw new Error(`Meta-harness qualification does not match the selected search winner`) + } + const confirmation = input.confirmation ? HarnessConfirmation.binds(contract, input.confirmation) : undefined + if (confirmation && !contract.confirmation) { + throw new Error(`A legacy harness report cannot cite sealed confirmation evidence`) + } + if (confirmation && contract.metaHarness && meta?.status !== "passed") { + throw new Error(`A sealed report requires a passing meta-harness qualification`) + } + if (confirmation && meta && confirmation.selection.candidateID !== meta.selection.candidateID) { + throw new Error(`Meta-harness and sealed confirmation receipts select different candidates`) + } + const provisional = Boolean(contract.confirmation && !confirmation) + const result = contract.confirmation ? confirmation : evaluation + const direction = contract.benchmark.direction ?? "pass" + const comparisonKey = digest({ + benchmark: contract.benchmark.name, + version: contract.benchmark.version, + taskID: contract.benchmark.taskID, + split: contract.benchmark.split, + evaluator: contract.benchmark.evaluator, + evaluatorVersion: contract.benchmark.evaluatorVersion, + evaluatorSource: contract.benchmark.evaluatorSource, + fidelities: contract.benchmark.fidelities, + metric: contract.benchmark.metric, + direction, + target: contract.benchmark.target, + objectives: contract.benchmark.objectives, + objectiveAudit: contract.benchmark.objectiveAudit, + packs: (contract.packs ?? []).toSorted(), + simulation: contract.simulation, + integrity: contract.integrity, + evolution: contract.evolution, + metaHarness: contract.metaHarness, + interventions: contract.interventions, + search: contract.search, + evaluatorAudit: contract.evaluatorAudit, + semanticAudit: contract.semanticAudit, + replication: contract.replication, + audit: contract.audit, + failureDiscovery: contract.failureDiscovery, + synthesis: contract.synthesis, + autonomy: contract.autonomy, + formalProof: contract.formalProof, + confirmation: contract.confirmation, + contamination: contract.contamination, + }) + const tokens = input.trace + ? { + input: input.trace.tokens.input, + output: input.trace.tokens.output, + reasoning: input.trace.tokens.reasoning, + cacheRead: input.trace.tokens.cache.read, + cacheWrite: input.trace.tokens.cache.write, + total: + input.trace.tokens.input + + input.trace.tokens.output + + input.trace.tokens.reasoning + + input.trace.tokens.cache.read + + input.trace.tokens.cache.write, + } + : undefined + const candidates = input.search ? Object.values(input.search.candidates) : [] + const costs = [ + ...evaluations.flatMap((item) => (item.usage?.costUSD === undefined ? [] : [item.usage.costUSD])), + ...(confirmation?.usage?.costUSD === undefined ? [] : [confirmation.usage.costUSD]), + ] + const walls = [ + ...evaluations.flatMap((item) => (item.usage?.wallTimeMs === undefined ? [] : [item.usage.wallTimeMs])), + ...(confirmation?.usage?.wallTimeMs === undefined ? [] : [confirmation.usage.wallTimeMs]), + ] + const evaluatorCostUSD = costs.length ? costs.reduce((sum, value) => sum + value, 0) : undefined + const evaluatorWallTimeMs = walls.length ? walls.reduce((sum, value) => sum + value, 0) : undefined + const costUSD = + input.trace?.cost === undefined && evaluatorCostUSD === undefined + ? undefined + : (input.trace?.cost ?? 0) + (evaluatorCostUSD ?? 0) + const wallTimeMs = + input.trace?.totalCompletionTimeMs === undefined && evaluatorWallTimeMs === undefined + ? undefined + : (input.trace?.totalCompletionTimeMs ?? 0) + (evaluatorWallTimeMs ?? 0) + return Info.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + contractFingerprint: HarnessContract.fingerprint(contract), + comparisonKey, + benchmark: { + id: contract.benchmark.name, + title: contract.benchmark.title, + family: contract.benchmark.family, + version: contract.benchmark.version, + taskID: contract.benchmark.taskID, + split: contract.benchmark.split, + }, + execution: { + profile: contract.profile, + packs: contract.packs ?? [], + provider: contract.model.provider, + model: contract.model.name, + effort: contract.model.effort, + intervention: contract.intervention, + autonomy: contract.autonomy + ? { + claimedLevel: contract.autonomy.claimedLevel, + derivedLevel: autonomy?.derivedLevel, + status: autonomy?.status, + } + : undefined, + formal: contract.formalProof + ? { + tier: contract.formalProof.tier, + relation: contract.formalProof.relation, + status: formal?.status, + blueprint, + } + : undefined, + seed: contract.seed, + }, + quality: { + source: contract.confirmation ? "sealed_confirmation" : "optimization", + provisional, + status: result?.status, + metric: contract.benchmark.metric, + direction, + score: result?.score, + target: contract.benchmark.target, + targetReached: reached(contract, result), + evaluator: contract.confirmation?.claim.evaluator.name ?? contract.benchmark.evaluator, + evaluatorVersion: contract.confirmation?.claim.evaluator.version ?? contract.benchmark.evaluatorVersion, + simulationReceiptID: contract.confirmation ? undefined : evaluation?.simulationReceiptID, + integrityReceiptID: contract.confirmation ? undefined : evaluation?.integrityReceiptID, + evolutionReceiptID: contract.confirmation ? undefined : evaluation?.evolutionReceiptID, + interventionReceiptID: contract.confirmation ? undefined : evaluation?.interventionReceiptID, + evaluatorAuditReceiptID: contract.confirmation ? undefined : evaluation?.evaluatorAuditReceiptID, + semanticReceiptID: contract.confirmation ? undefined : evaluation?.semanticReceiptID, + replicationReceiptID: contract.confirmation ? undefined : evaluation?.replicationReceiptID, + auditReceiptID: contract.confirmation ? undefined : evaluation?.auditReceiptID, + failureDiscoveryReceiptID: contract.confirmation ? undefined : evaluation?.failureDiscoveryReceiptID, + synthesisReceiptID: contract.confirmation ? undefined : evaluation?.synthesisReceiptID, + autonomyReceiptID: evaluation?.autonomyReceiptID, + proofReceiptID: evaluation?.proofReceiptID, + metaReceiptID: meta?.receiptID, + confirmationReceiptID: confirmation?.receiptID, + evaluations: evaluations.length, + }, + efficiency: { + costUSD, + evaluatorCostUSD, + tokens, + wallTimeMs, + evaluatorWallTimeMs, + toolCalls: input.trace?.toolCalls, + searches: input.trace?.searchCount, + dedupeHits: input.trace?.dedupeHits, + retries: input.trace?.retryCount, + failures: input.trace?.failureCount, + candidates: input.search ? candidates.length : undefined, + }, + search: input.search + ? { + status: input.search.status, + stopReason: input.search.stopReason, + bestID: input.search.bestID, + candidates: candidates.length, + verified: candidates.filter((item) => item.result?.source === "verified").length, + generations: Math.max(0, ...candidates.map((item) => item.generation)), + stalled: input.search.stalled, + proposalPolicy: input.search.proposalPolicy, + controller: input.search.controller, + adaptation: + input.search.proposalPolicy === "adaptive-v4" ? HarnessSearch.adaptation(input.search) : undefined, + objectives: input.search.objectives, + objectiveAudit: contract.benchmark.objectiveAudit, + archive: HarnessSearch.frontier(input.search).length, + } + : undefined, + metaHarness: meta + ? { + status: meta.status, + selectionID: meta.selection.selectionID, + diagnostics: meta.diagnostics, + failures: meta.failures, + } + : undefined, + generatedAt: input.generatedAt ?? Date.now(), + }) + } + + export async function build(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${sessionID}`) + const [evaluations, trace, search, meta, confirmation, blueprint] = await Promise.all([ + HarnessEvaluation.list(sessionID), + SessionTrace.build(sessionID), + HarnessSearch.read(sessionID).catch(() => undefined), + HarnessMeta.current(contract), + HarnessConfirmation.current(contract), + contract.formalProof?.blueprint + ? HarnessBlueprint.read(sessionID) + .then((value) => value.summary) + .catch(() => undefined) + : undefined, + ]) + const evaluation = selected(evaluations, search ?? undefined) + const autonomy = evaluation?.autonomyReceiptID + ? await HarnessAutonomy.read(evaluation.autonomyReceiptID, contract) + : undefined + const formal = evaluation?.proofReceiptID + ? await HarnessFormal.read(evaluation.proofReceiptID, contract) + : undefined + const report = compile({ + contract, + evaluations, + trace: trace.summary, + search, + meta: meta ?? undefined, + confirmation: confirmation ?? undefined, + autonomy, + formal, + blueprint, + }) + await Bun.write(file(sessionID), JSON.stringify(report, null, 2) + "\n") + return report + } + + function score(left: Info, right: Info) { + if (left.quality.status === "passed" && right.quality.status !== "passed") return 1 + if (left.quality.status !== "passed" && right.quality.status === "passed") return -1 + if (left.quality.direction === "pass") return 0 + if (left.quality.score === undefined || right.quality.score === undefined) return 0 + const delta = left.quality.score - right.quality.score + return left.quality.direction === "maximize" ? delta : -delta + } + + export function dominates(left: Info, right: Info) { + const a = Info.parse(left) + const b = Info.parse(right) + if (a.comparisonKey !== b.comparisonKey) return false + if (a.quality.provisional || b.quality.provisional) return false + if (a.quality.status !== "passed") return false + const quality = score(a, b) + if (quality < 0) return false + const pairs = [ + [a.efficiency.costUSD, b.efficiency.costUSD], + [a.efficiency.tokens?.total, b.efficiency.tokens?.total], + [a.efficiency.wallTimeMs, b.efficiency.wallTimeMs], + ].filter((pair): pair is [number, number] => pair[0] !== undefined && pair[1] !== undefined) + if (pairs.some(([x, y]) => x > y)) return false + return quality > 0 || pairs.some(([x, y]) => x < y) + } + + export function frontier(input: Info[]) { + const reports = input.map((item) => Info.parse(item)) + return reports.filter( + (item) => + !item.quality.provisional && !reports.some((other) => other.runID !== item.runID && dominates(other, item)), + ) + } + + export function compare(input: Info[], baselineRunID: string) { + const reports = input.map((item) => Info.parse(item)) + const baseline = reports.find((item) => item.runID === baselineRunID) + if (!baseline) throw new Error(`Unknown baseline run ${baselineRunID}`) + if (reports.some((item) => item.quality.provisional)) { + throw new Error(`Provisional optimization results cannot enter final quality-cost comparison`) + } + if (reports.some((item) => item.comparisonKey !== baseline.comparisonKey)) { + throw new Error(`Quality-cost reports are only comparable under the same benchmark contract key`) + } + const pareto = new Set(frontier(reports).map((item) => item.runID)) + return reports.map((item) => ({ + runID: item.runID, + scoreImprovement: + item.quality.score === undefined || baseline.quality.score === undefined + ? undefined + : item.quality.direction === "minimize" + ? baseline.quality.score - item.quality.score + : item.quality.score - baseline.quality.score, + costDelta: + item.efficiency.costUSD === undefined || baseline.efficiency.costUSD === undefined + ? undefined + : item.efficiency.costUSD - baseline.efficiency.costUSD, + tokenDelta: + item.efficiency.tokens === undefined || baseline.efficiency.tokens === undefined + ? undefined + : item.efficiency.tokens.total - baseline.efficiency.tokens.total, + wallTimeDelta: + item.efficiency.wallTimeMs === undefined || baseline.efficiency.wallTimeMs === undefined + ? undefined + : item.efficiency.wallTimeMs - baseline.efficiency.wallTimeMs, + pareto: pareto.has(item.runID), + })) + } +} diff --git a/backend/cli/src/session/harness/search.ts b/backend/cli/src/session/harness/search.ts new file mode 100644 index 00000000..110a95f1 --- /dev/null +++ b/backend/cli/src/session/harness/search.ts @@ -0,0 +1,1657 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessAdaptation } from "./adaptation" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" +import { HarnessEvolution } from "./evolution" + +export namespace HarnessSearch { + export const Stop = z.enum(["budget_exhausted", "objective_met", "no_improvement", "user_cancelled", "runtime_error"]) + export type Stop = z.infer + + export const Strategy = z.enum(["seed", "explore", "exploit", "fuse", "migrate", "diverge"]) + export type Strategy = z.infer + + export const Mode = z.enum(["single-pass", "stepwise", "diff"]) + export type Mode = z.infer + + export const Operator = z.enum([ + "bug-fix", + "external-dependency", + "architectural-change", + "composition", + "local-refinement", + "pruning", + "refactor", + "efficiency", + "hyperparameter-tuning", + ]) + export type Operator = z.infer + + export const Mandate = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + protocol: z.literal("agentic-variation-v1"), + operator: Operator, + instruction: z.string().min(1).max(1_000), + }) + .strict() + export type Mandate = z.infer + + export const Artifact = z + .object({ + uri: z.string().min(1).max(2_048), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict() + export type Artifact = z.infer + + export const Evolution = HarnessEvolution.Diagnostics.extend({ + receiptID: z.string().regex(/^[a-f0-9]{64}$/), + }).strict() + export type Evolution = z.infer + + export const Result = z + .object({ + source: z.enum(["observed", "screened", "verified"]), + status: HarnessEvaluation.Status, + score: z.number().finite().optional(), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "A candidate result may contain at most 128 metrics") + .default({}), + checks: z.array(HarnessEvaluation.Check).default([]), + evidence: z.array(z.string().min(1).max(1_000)).max(128).default([]), + usage: HarnessEvaluation.Usage.optional(), + fidelity: z + .object({ stage: z.string().min(1).max(100), final: z.boolean() }) + .strict() + .optional(), + feedback: z.string().max(8_000).optional(), + evaluator: z.string().max(200).optional(), + evolution: Evolution.optional(), + evaluatedAt: z.number().int().positive(), + recordedRevision: z.number().int().positive().optional(), + }) + .strict() + export type Result = z.infer + + export const Lease = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + revision: z.number().int().nonnegative(), + strategy: Strategy, + mode: Mode, + targetIsland: z.number().int().nonnegative(), + contextIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(6) + .refine((ids) => new Set(ids).size === ids.length, "Recommendation context must be unique"), + control: HarnessAdaptation.Control.optional(), + }) + .strict() + export type Lease = z.infer + + export const Candidate = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + parentIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(2) + .refine((ids) => new Set(ids).size === ids.length, "Candidate parents must be unique"), + inspirationIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(2) + .refine((ids) => new Set(ids).size === ids.length, "Candidate inspirations must be unique") + .default([]), + branch: z.string().min(1).max(120), + generation: z.number().int().nonnegative(), + island: z.number().int().nonnegative().default(0), + ordinal: z.number().int().nonnegative().optional(), + createdRevision: z.number().int().positive().optional(), + proposal: z.string().min(1).max(4_000), + artifact: Artifact, + lease: Lease.optional(), + reservationID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + result: Result.optional(), + createdAt: z.number().int().positive(), + }) + .strict() + export type Candidate = z.infer + + export const Reservation = z + .object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + ordinal: z.number().int().nonnegative(), + parentIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(2) + .refine((ids) => new Set(ids).size === ids.length, "Reservation parents must be unique"), + inspirationIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(2) + .refine((ids) => new Set(ids).size === ids.length, "Reservation inspirations must be unique"), + lease: Lease, + mandate: Mandate.optional(), + status: z.enum(["open", "consumed", "released"]), + candidateID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + createdAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + }) + .strict() + export type Reservation = z.infer + + export const Population = z + .object({ + mode: z.enum(["legacy", "islands"]), + count: z.number().int().min(1).max(4), + initial: z.number().int().min(1).max(2).optional(), + topology: z.literal("ring"), + migrationInterval: z.number().int().positive(), + }) + .strict() + export type Population = z.infer + + export const State = z + .object({ + schemaVersion: z.literal(4), + proposalPolicy: z.enum(["advisory-v2", "leased-v3", "adaptive-v4"]), + runID: z.string().min(1), + sessionID: z.string().min(1), + objective: z.string().min(1), + evaluator: z.string().min(1), + metric: z.string().min(1), + direction: z.enum(["maximize", "minimize", "pass"]), + target: z.number().finite().optional(), + objectives: HarnessContract.Objectives.default([]), + controller: HarnessContract.Search.optional(), + population: Population, + budget: z + .object({ + candidates: z.number().int().positive(), + wallTimeMs: z.number().int().positive().optional(), + stall: z.number().int().positive().default(5), + }) + .strict(), + status: z.enum(["active", "completed"]), + stopReason: Stop.optional(), + candidates: z.record(z.string(), Candidate), + reservations: z.record(z.string(), Reservation).default({}), + bestID: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + archiveIDs: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .refine((items) => new Set(items).size === items.length, "Pareto archive candidates must be unique") + .default([]), + stalled: z.number().int().nonnegative(), + revision: z.number().int().nonnegative(), + startedAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + }) + .strict() + export type State = z.infer + + export type Recommendation = Lease & { + parentIDs: string[] + inspirationIDs: string[] + reasons: string[] + } + + type Route = Omit + + const root = path.join(Global.Path.data, "harness", "search") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + const candidates = (state: State) => Object.values(state.candidates) + const reservations = (state: State) => Object.values(state.reservations) + const open = (state: State) => reservations(state).filter((item) => item.status === "open") + const verified = (candidate: Candidate) => + candidate.result?.source === "verified" && candidate.result.status === "passed" + + const identity = ( + input: Pick< + Candidate, + "parentIDs" | "inspirationIDs" | "branch" | "proposal" | "artifact" | "lease" | "reservationID" + >, + ) => + digest({ + parentIDs: input.parentIDs.toSorted(), + inspirationIDs: input.inspirationIDs.toSorted(), + branch: input.branch, + proposal: input.proposal, + artifact: input.artifact, + ...(input.lease ? { lease: input.lease } : {}), + ...(input.reservationID ? { reservationID: input.reservationID } : {}), + }) + + const leaseID = (state: Pick, input: Omit) => + digest({ + runID: state.runID, + sessionID: state.sessionID, + revision: input.revision, + strategy: input.strategy, + mode: input.mode, + parentIDs: input.parentIDs.toSorted(), + inspirationIDs: input.inspirationIDs.toSorted(), + targetIsland: input.targetIsland, + contextIDs: input.contextIDs, + ...(input.control ? { control: input.control } : {}), + }) + + const mandateID = (input: Omit) => digest(input) + + const reservationID = ( + state: Pick, + input: Pick, + ) => + digest({ + runID: state.runID, + sessionID: state.sessionID, + ordinal: input.ordinal, + parentIDs: input.parentIDs.toSorted(), + inspirationIDs: input.inspirationIDs.toSorted(), + lease: input.lease, + ...(input.mandate ? { mandate: input.mandate } : {}), + createdAt: input.createdAt, + }) + + function order(state: State, left: Candidate, right: Candidate) { + if (state.direction === "pass") return left.createdAt - right.createdAt || left.id.localeCompare(right.id) + const a = left.result?.score + const b = right.result?.score + if (a === undefined && b === undefined) return left.id.localeCompare(right.id) + if (a === undefined) return 1 + if (b === undefined) return -1 + const delta = state.direction === "maximize" ? b - a : a - b + return delta || left.createdAt - right.createdAt || left.id.localeCompare(right.id) + } + + const ranked = (state: State) => + candidates(state) + .filter(verified) + .toSorted((a, b) => order(state, a, b)) + + const values = (state: State, candidate: Candidate) => { + if (state.direction === "pass") return [] + return [ + { direction: state.direction, value: candidate.result?.score }, + ...state.objectives.map((item) => ({ ...item, value: candidate.result?.metrics[item.metric] })), + ] + } + + const dominates = (state: State, left: Candidate, right: Candidate) => { + const a = values(state, left) + const b = values(state, right) + if (!a.length || a.some((item) => item.value === undefined) || b.some((item) => item.value === undefined)) { + return false + } + const deltas = a.map((item, index) => { + const other = b[index]! + if (item.direction !== other.direction) throw new Error(`Pareto objective directions are inconsistent`) + return item.direction === "maximize" ? item.value! - other.value! : other.value! - item.value! + }) + return deltas.every((delta) => delta >= 0) && deltas.some((delta) => delta > 0) + } + + const archive = (state: State) => { + const pool = ranked(state) + return pool.filter( + (candidate) => !pool.some((other) => other.id !== candidate.id && dominates(state, other, candidate)), + ) + } + + export const frontier = (input: State) => archive(State.parse(input)) + + const reached = (state: State, candidate: Candidate) => { + if (!verified(candidate)) return false + if (state.direction === "pass") return true + if (state.target === undefined || candidate.result?.score === undefined) return false + if (state.direction === "maximize") return candidate.result.score >= state.target + return candidate.result.score <= state.target + } + + const expired = (state: State, now: number) => + state.budget.wallTimeMs !== undefined && now - state.startedAt >= state.budget.wallTimeMs + + const population = (budget: number): Population => { + const count = budget < 6 ? 1 : Math.min(4, Math.floor(Math.sqrt(budget / 2))) + return { + mode: "islands", + count, + initial: Math.min(2, count), + topology: "ring", + migrationInterval: Math.max(2, Math.ceil(budget / (count * 2))), + } + } + + const vacancy = (state: State) => { + const counts = Array.from({ length: state.population.count }, () => 0) + for (const candidate of candidates(state)) counts[candidate.island] = (counts[candidate.island] ?? 0) + 1 + const least = Math.min(...counts) + return counts.findIndex((count) => count === least) + } + + const events = (state: State, revision = state.revision): HarnessAdaptation.Event[] => + candidates(state).flatMap((candidate) => { + if ( + candidate.createdRevision === undefined || + candidate.createdRevision > revision || + candidate.result?.source !== "verified" || + candidate.result.recordedRevision === undefined || + candidate.result.recordedRevision > revision + ) { + return [] + } + return [ + HarnessAdaptation.Event.parse({ + candidateID: candidate.id, + island: candidate.island, + revision: candidate.result.recordedRevision, + status: candidate.result.status, + score: candidate.result.status === "passed" ? candidate.result.score : undefined, + }), + ] + }) + + function summary(state: State, revision = state.revision) { + if (!state.controller || state.direction === "pass") { + throw new Error(`Adaptive search requires a numeric controller contract`) + } + return HarnessAdaptation.derive({ + policy: state.controller, + direction: state.direction, + islands: state.population.count, + events: events(state, revision), + }) + } + + function controller(state: State, targetIsland: number, revision = state.revision) { + if (!state.controller || state.direction === "pass") { + throw new Error(`Adaptive search requires a numeric controller contract`) + } + return HarnessAdaptation.control({ + policy: state.controller, + direction: state.direction, + islands: state.population.count, + events: events(state, revision), + targetIsland, + key: `${state.runID}:${state.sessionID}:${revision}`, + }) + } + + const roots = (state: State) => + candidates(state).filter( + (candidate) => + !candidate.parentIDs.length && + (candidate.result === undefined || + candidate.result.source !== "verified" || + candidate.result.status === "passed"), + ) + + const rootLimit = (state: State) => { + if (state.proposalPolicy !== "adaptive-v4") { + return Math.min(4, Math.max(2, Math.ceil(Math.sqrt(state.budget.candidates)))) + } + const active = new Set(roots(state).map((candidate) => candidate.island)).size + const spawn = events(state).length && summary(state).globalStagnation ? 1 : 0 + return Math.min(state.population.count, Math.max(state.population.initial ?? 1, active) + spawn) + } + + const stop = (state: State, reason: Stop, now = Date.now()): State => ({ + ...state, + status: "completed", + stopReason: reason, + reservations: Object.fromEntries( + reservations(state).map((item) => [ + item.id, + item.status === "open" ? { ...item, status: "released" as const, updatedAt: now } : item, + ]), + ), + revision: state.revision + 1, + updatedAt: now, + }) + + function topology(state: State) { + if (state.proposalPolicy !== "advisory-v2" && state.population.mode !== "islands") { + throw new Error(`Leased search state must use the server-derived island policy`) + } + if ((state.proposalPolicy === "adaptive-v4") !== Boolean(state.controller)) { + throw new Error(`Adaptive proposal policy and controller must be declared together`) + } + if (state.population.mode === "legacy") { + if (candidates(state).some((candidate) => candidate.ordinal !== undefined || candidate.island !== 0)) { + throw new Error(`Legacy search state cannot contain island assignments`) + } + return + } + if (JSON.stringify(state.population) !== JSON.stringify(population(state.budget.candidates))) { + throw new Error(`Persisted island policy does not match the server-derived budget policy`) + } + const items = candidates(state).toSorted( + (a, b) => (a.ordinal ?? Number.MAX_SAFE_INTEGER) - (b.ordinal ?? Number.MAX_SAFE_INTEGER), + ) + if (items.some((candidate, index) => candidate.ordinal !== index)) { + throw new Error(`Persisted island candidate order is not contiguous`) + } + const seen = new Map() + const claimed = new Set() + const hashes = new Set() + const counts = Array.from({ length: state.population.count }, () => 0) + for (const candidate of items) { + if (identity(candidate) !== candidate.id) + throw new Error(`Persisted candidate identity does not match its content`) + if (hashes.has(candidate.artifact.sha256)) throw new Error(`Persisted search contains duplicate artifact content`) + hashes.add(candidate.artifact.sha256) + const parents = candidate.parentIDs.map((id) => seen.get(id)) + const inspirations = candidate.inspirationIDs.map((id) => seen.get(id)) + if (parents.some((parent) => !parent) || inspirations.some((item) => !item)) { + throw new Error(`Persisted candidate lineage must reference earlier candidates`) + } + if (parents.some((parent) => !verified(parent!)) || inspirations.some((item) => !verified(item!))) { + throw new Error(`Persisted candidate lineage must reference verified passing candidates`) + } + if (candidate.inspirationIDs.some((id) => candidate.parentIDs.includes(id))) { + throw new Error(`Candidate inspirations must be distinct from parents`) + } + if (candidate.reservationID) { + if (claimed.has(candidate.reservationID)) throw new Error(`A reservation may authorize only one candidate`) + claimed.add(candidate.reservationID) + } + if (state.proposalPolicy !== "advisory-v2") { + if (!candidate.lease) throw new Error(`Leased search candidate is missing recommendation provenance`) + if (candidate.lease.targetIsland !== candidate.island) { + throw new Error(`Persisted recommendation target does not match candidate island`) + } + const context = candidate.lease.contextIDs.map((id) => seen.get(id)) + if (context.some((item) => !item || !verified(item))) { + throw new Error(`Persisted recommendation context must reference earlier verified passing candidates`) + } + const expected = leaseID(state, { + revision: candidate.lease.revision, + strategy: candidate.lease.strategy, + mode: candidate.lease.mode, + parentIDs: candidate.parentIDs, + inspirationIDs: candidate.inspirationIDs, + targetIsland: candidate.lease.targetIsland, + contextIDs: candidate.lease.contextIDs, + control: candidate.lease.control, + }) + if (candidate.lease.id !== expected) { + throw new Error(`Persisted recommendation identity does not match its content`) + } + if (state.proposalPolicy === "adaptive-v4") { + if ( + !candidate.createdRevision || + (candidate.result?.source === "verified" && !candidate.result.recordedRevision) + ) { + throw new Error(`Adaptive candidates require revision-bound provenance`) + } + if (!candidate.lease.control) throw new Error(`Adaptive recommendation is missing controller provenance`) + if (candidate.createdRevision <= candidate.lease.revision || candidate.createdRevision > state.revision) { + throw new Error(`Adaptive candidate revision provenance is invalid`) + } + if ( + candidate.result?.recordedRevision !== undefined && + (candidate.result.recordedRevision <= candidate.createdRevision || + candidate.result.recordedRevision > state.revision) + ) { + throw new Error(`Adaptive evaluation revision provenance is invalid`) + } + const expectedControl = controller(state, candidate.lease.targetIsland, candidate.lease.revision) + if (JSON.stringify(candidate.lease.control) !== JSON.stringify(expectedControl)) { + throw new Error(`Adaptive recommendation controller does not match verified candidate history`) + } + } + } + const least = Math.min(...counts) + const expected = + state.proposalPolicy === "adaptive-v4" + ? candidate.lease?.targetIsland + : candidate.reservationID + ? candidate.lease?.targetIsland + : parents.length + ? parents.toSorted((a, b) => order(state, a!, b!))[0]!.island + : counts.findIndex((count) => count === least) + if (candidate.island !== expected || candidate.island >= state.population.count) { + throw new Error(`Persisted candidate island does not match server assignment`) + } + const generation = parents.length ? Math.max(...parents.map((parent) => parent!.generation)) + 1 : 0 + if (candidate.generation !== generation) { + throw new Error(`Persisted candidate generation does not match its lineage`) + } + counts[candidate.island] = counts[candidate.island]! + 1 + seen.set(candidate.id, candidate) + } + const tickets = reservations(state).toSorted((a, b) => a.ordinal - b.ordinal) + if (tickets.some((item, index) => item.ordinal !== index)) { + throw new Error(`Persisted reservation order is not contiguous`) + } + for (const item of tickets) { + if (reservationID(state, item) !== item.id) { + throw new Error(`Persisted reservation identity does not match its content`) + } + if ( + item.mandate && + mandateID({ + protocol: item.mandate.protocol, + operator: item.mandate.operator, + instruction: item.mandate.instruction, + }) !== item.mandate.id + ) { + throw new Error(`Persisted variation mandate identity does not match its content`) + } + if (item.updatedAt < item.createdAt) throw new Error(`Persisted reservation timestamps are invalid`) + if (item.inspirationIDs.some((id) => item.parentIDs.includes(id))) { + throw new Error(`Reservation inspirations must be distinct from parents`) + } + if (item.lease.targetIsland >= state.population.count) { + throw new Error(`Persisted reservation target island is outside the search population`) + } + const parents = item.parentIDs.map((id) => state.candidates[id]) + const inspirations = item.inspirationIDs.map((id) => state.candidates[id]) + const context = item.lease.contextIDs.map((id) => state.candidates[id]) + if ( + parents.some((candidate) => !candidate || !verified(candidate)) || + inspirations.some((candidate) => !candidate || !verified(candidate)) || + context.some((candidate) => !candidate || !verified(candidate)) + ) { + throw new Error(`Persisted reservation may reference only verified passing candidates`) + } + const expected = leaseID(state, { + revision: item.lease.revision, + strategy: item.lease.strategy, + mode: item.lease.mode, + parentIDs: item.parentIDs, + inspirationIDs: item.inspirationIDs, + targetIsland: item.lease.targetIsland, + contextIDs: item.lease.contextIDs, + control: item.lease.control, + }) + if (item.lease.id !== expected) throw new Error(`Persisted reservation lease does not match its content`) + if (state.proposalPolicy === "adaptive-v4") { + if (!item.lease.control) throw new Error(`Adaptive reservation is missing controller provenance`) + const expectedControl = controller(state, item.lease.targetIsland, item.lease.revision) + if (JSON.stringify(item.lease.control) !== JSON.stringify(expectedControl)) { + throw new Error(`Adaptive reservation controller does not match verified candidate history`) + } + } + if (item.status === "open" && item.candidateID) { + throw new Error(`An open reservation cannot name a candidate`) + } + if (item.status === "released" && item.candidateID && !state.candidates[item.candidateID]) { + throw new Error(`A duplicate-released reservation must name an existing candidate`) + } + if (item.status === "consumed") { + const candidate = item.candidateID ? state.candidates[item.candidateID] : undefined + if (!candidate || candidate.reservationID !== item.id) { + throw new Error(`A consumed reservation must name its authorized candidate`) + } + } + } + } + + export function adaptation(state: State) { + const parsed = State.parse(state) + topology(parsed) + return summary(parsed) + } + + function parse(data: Record) { + const migrated = + data.schemaVersion === 1 + ? { + ...data, + schemaVersion: 4, + proposalPolicy: "advisory-v2", + population: { mode: "legacy", count: 1, initial: 1, topology: "ring", migrationInterval: 1 }, + } + : data.schemaVersion === 2 + ? { + ...data, + schemaVersion: 4, + proposalPolicy: "advisory-v2", + population: { + ...(data.population as object), + initial: Math.min(2, Number((data.population as { count?: number } | undefined)?.count ?? 1)), + }, + } + : data.schemaVersion === 3 + ? { + ...data, + schemaVersion: 4, + population: { + ...(data.population as object), + initial: Math.min(2, Number((data.population as { count?: number } | undefined)?.count ?? 1)), + }, + } + : data + const parsed = State.parse(migrated) + const expected = archive(parsed).map((item) => item.id) + const state = "archiveIDs" in data ? parsed : State.parse({ ...parsed, archiveIDs: expected }) + if (JSON.stringify(state.archiveIDs) !== JSON.stringify(expected)) { + throw new Error(`Persisted Pareto archive does not match verified candidate results`) + } + topology(state) + return state + } + + export async function initialize(input: { + sessionID: string + candidates?: number + wallTimeMs?: number + stall?: number + target?: number + }) { + const contract = await HarnessContract.read(input.sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${input.sessionID}`) + if (contract.profile !== "optimize") throw new Error(`Harness search requires the optimize profile`) + const candidates = input.candidates ?? contract.budget.candidates + if (!candidates) throw new Error(`The optimize contract must declare a candidate budget`) + if (contract.budget.candidates !== undefined && candidates > contract.budget.candidates) { + throw new Error(`Harness search cannot exceed the contract candidate budget`) + } + if ( + contract.benchmark.target !== undefined && + input.target !== undefined && + contract.benchmark.target !== input.target + ) { + throw new Error(`Harness search target does not match the benchmark contract`) + } + const wallTimeMs = input.wallTimeMs ?? contract.budget.wallTimeMs + if ( + contract.budget.wallTimeMs !== undefined && + wallTimeMs !== undefined && + wallTimeMs > contract.budget.wallTimeMs + ) { + throw new Error(`Harness search cannot exceed the contract wall-time budget`) + } + if (contract.search && input.stall !== undefined && input.stall !== contract.search.stagnation.patience) { + throw new Error(`Adaptive search patience is fixed by the benchmark contract`) + } + const now = Date.now() + const initial: State = { + schemaVersion: 4, + proposalPolicy: contract.search ? "adaptive-v4" : "leased-v3", + runID: contract.runID, + sessionID: input.sessionID, + objective: contract.objective, + evaluator: contract.benchmark.evaluator, + metric: contract.benchmark.metric ?? "status", + direction: contract.benchmark.direction ?? "pass", + objectives: contract.benchmark.objectives ?? [], + controller: contract.search, + population: population(candidates), + ...(contract.benchmark.target === undefined && input.target === undefined + ? {} + : { target: contract.benchmark.target ?? input.target }), + budget: { + candidates, + ...(wallTimeMs === undefined ? {} : { wallTimeMs }), + stall: input.stall ?? contract.search?.stagnation.patience ?? 5, + }, + status: "active", + candidates: {}, + reservations: {}, + archiveIDs: [], + stalled: 0, + revision: 0, + startedAt: now, + updatedAt: now, + } + const expected = State.parse(initial) + await JsonStore.update(file(input.sessionID), (data) => { + if (!Object.keys(data).length) return expected + const state = parse(data) + const stable = [ + "runID", + "sessionID", + "objective", + "evaluator", + "metric", + "direction", + "target", + "objectives", + "controller", + "budget", + ] as const + if (stable.every((key) => JSON.stringify(state[key]) === JSON.stringify(expected[key]))) return state + throw new Error(`Harness search already exists with a different contract or budget`) + }) + return read(input.sessionID) + } + + export async function read(sessionID: string): Promise { + return parse(await JsonStore.read(file(sessionID))) + } + + export async function reserve(input: { sessionID: string; count: number }) { + const count = z.number().int().min(1).max(8).parse(input.count) + const issued: Reservation[] = [] + await JsonStore.update(file(input.sessionID), (data) => { + const state = parse(data) + const now = Date.now() + if (state.proposalPolicy === "advisory-v2") { + throw new Error(`Parallel reservations require a leased proposal policy`) + } + if (state.status !== "active") return state + if (expired(state, now)) return stop(state, "budget_exhausted", now) + const remaining = Math.max(0, state.budget.candidates - candidates(state).length - open(state).length) + const choice = recommend(state) + const current = roots(state) + const limit = rootLimit(state) + const openRoots = open(state).filter((item) => !item.parentIDs.length).length + const capacity = choice.parentIDs.length + ? remaining + : Math.max(0, Math.min(remaining, limit - current.length - openRoots)) + const size = Math.min(count, capacity) + if (!size) return state + const start = reservations(state).length + const plans = portfolio(state, size, start, openRoots) + const additions = plans.map((plan, index) => { + const lease = Lease.parse({ + id: plan.recommendation.id, + revision: plan.recommendation.revision, + strategy: plan.recommendation.strategy, + mode: plan.recommendation.mode, + targetIsland: plan.recommendation.targetIsland, + contextIDs: plan.recommendation.contextIDs, + control: plan.recommendation.control, + }) + const draft = { + ordinal: start + index, + parentIDs: plan.recommendation.parentIDs.toSorted(), + inspirationIDs: plan.recommendation.inspirationIDs.toSorted(), + lease, + mandate: plan.mandate, + status: "open" as const, + createdAt: now, + updatedAt: now, + } + return Reservation.parse({ id: reservationID(state, draft), ...draft }) + }) + issued.push(...additions) + return { + ...state, + reservations: { + ...state.reservations, + ...Object.fromEntries(additions.map((item) => [item.id, item])), + }, + revision: state.revision + 1, + updatedAt: now, + } + }) + return { reservations: issued, state: await read(input.sessionID) } + } + + export async function release(input: { sessionID: string; reservationID: string }) { + await JsonStore.update(file(input.sessionID), (data) => { + const state = parse(data) + const reservation = state.reservations[input.reservationID] + if (!reservation) throw new Error(`Unknown reservation ${input.reservationID}`) + if (reservation.status === "released") return state + if (reservation.status === "consumed") throw new Error(`A consumed reservation cannot be released`) + const now = Date.now() + return { + ...state, + reservations: { + ...state.reservations, + [reservation.id]: { ...reservation, status: "released", updatedAt: now }, + }, + revision: state.revision + 1, + updatedAt: now, + } + }) + return read(input.sessionID) + } + + export async function add(input: { + sessionID: string + recommendationID?: string + reservationID?: string + parentIDs: string[] + inspirationIDs?: string[] + branch: string + proposal: string + artifact: Artifact + }) { + if (input.recommendationID && input.reservationID) { + throw new Error(`A proposal must use either a recommendation or a reservation, not both`) + } + const artifact = Artifact.parse(input.artifact) + const parents = input.parentIDs.toSorted() + const inspirations = (input.inspirationIDs ?? []).toSorted() + const initial = identity({ + parentIDs: parents, + inspirationIDs: inspirations, + branch: input.branch, + proposal: input.proposal, + artifact, + }) + const out = { accepted: false, deduplicated: false, id: initial } + await JsonStore.update(file(input.sessionID), (data) => { + const state = parse(data) + const reservation = input.reservationID ? state.reservations[input.reservationID] : undefined + if (input.reservationID && !reservation) throw new Error(`Unknown reservation ${input.reservationID}`) + const existing = candidates(state).find((candidate) => candidate.artifact.sha256 === artifact.sha256) + if (existing) { + out.accepted = true + out.deduplicated = true + out.id = existing.id + if (!reservation) return state + if (reservation.candidateID === existing.id) return state + if (reservation.status !== "open") throw new Error(`Reservation is no longer open`) + const now = Date.now() + return { + ...state, + reservations: { + ...state.reservations, + [reservation.id]: { ...reservation, status: "released", candidateID: existing.id, updatedAt: now }, + }, + revision: state.revision + 1, + updatedAt: now, + } + } + const now = Date.now() + if (state.status !== "active") return state + if (expired(state, now) || candidates(state).length >= state.budget.candidates) { + return stop(state, "budget_exhausted", now) + } + if (!reservation && candidates(state).length + open(state).length >= state.budget.candidates) return state + if (reservation?.status !== undefined && reservation.status !== "open") { + throw new Error(`Reservation is no longer open`) + } + if (parents.length > 2) throw new Error(`A candidate may have at most two parents`) + if (new Set(parents).size !== parents.length) throw new Error(`Candidate parents must be unique`) + if (inspirations.length > 2) throw new Error(`A candidate may have at most two inspirations`) + if (new Set(inspirations).size !== inspirations.length) throw new Error(`Candidate inspirations must be unique`) + if (inspirations.some((id) => parents.includes(id))) { + throw new Error(`Candidate inspirations must be distinct from parents`) + } + const ancestors = parents.map((parent) => state.candidates[parent]) + const sources = inspirations.map((item) => state.candidates[item]) + if (ancestors.some((parent) => !parent)) throw new Error(`Every candidate parent must exist in the same search`) + if (sources.some((item) => !item)) throw new Error(`Every candidate inspiration must exist in the same search`) + if (ancestors.some((parent) => !verified(parent!))) { + throw new Error(`Candidates may only descend from externally verified passing parents`) + } + if (sources.some((item) => !verified(item!))) { + throw new Error(`Candidates may only use externally verified passing inspirations`) + } + const recommendation: Recommendation | undefined = reservation + ? { + ...reservation.lease, + parentIDs: reservation.parentIDs, + inspirationIDs: reservation.inspirationIDs, + reasons: ["budget-backed-parallel-reservation"], + } + : state.proposalPolicy !== "advisory-v2" + ? recommend(state) + : undefined + if (recommendation && !input.recommendationID && !reservation) { + throw new Error(`A current recommendation_id or reservation_id is required by the leased proposal policy`) + } + if (recommendation && !reservation && recommendation.id !== input.recommendationID) { + throw new Error(`Recommendation lease is stale or belongs to a different search state`) + } + if ( + recommendation && + (JSON.stringify(parents) !== JSON.stringify(recommendation.parentIDs.toSorted()) || + JSON.stringify(inspirations) !== JSON.stringify(recommendation.inspirationIDs.toSorted())) + ) { + throw new Error(`Proposal lineage does not match the leased recommendation`) + } + if (!parents.length) { + const current = roots(state) + const limit = rootLimit(state) + const held = reservation ? 0 : open(state).filter((item) => !item.parentIDs.length).length + if (current.length + held >= limit) throw new Error(`Independent candidate root budget is exhausted`) + if (current.some((candidate) => candidate.branch === input.branch)) { + throw new Error(`An active independent root already exists for branch ${input.branch}`) + } + } + const generation = ancestors.length ? Math.max(...ancestors.map((parent) => parent!.generation)) + 1 : 0 + const derived = ancestors.length ? ancestors.toSorted((a, b) => order(state, a!, b!))[0]!.island : vacancy(state) + const island = recommendation?.targetIsland ?? derived + if (recommendation && ancestors.length && derived !== recommendation.targetIsland) { + throw new Error(`Server island assignment does not match the leased recommendation`) + } + const lease = + reservation?.lease ?? + (recommendation + ? Lease.parse({ + id: recommendation.id, + revision: recommendation.revision, + strategy: recommendation.strategy, + mode: recommendation.mode, + targetIsland: recommendation.targetIsland, + contextIDs: recommendation.contextIDs, + control: recommendation.control, + }) + : undefined) + const id = identity({ + parentIDs: parents, + inspirationIDs: inspirations, + branch: input.branch, + proposal: input.proposal, + artifact, + lease, + reservationID: reservation?.id, + }) + const candidate: Candidate = Candidate.parse({ + id, + parentIDs: parents, + inspirationIDs: inspirations, + branch: input.branch, + generation, + island, + ...(state.population.mode === "legacy" ? {} : { ordinal: candidates(state).length }), + ...(state.proposalPolicy === "adaptive-v4" ? { createdRevision: state.revision + 1 } : {}), + proposal: input.proposal, + artifact, + lease, + reservationID: reservation?.id, + createdAt: now, + }) + out.accepted = true + out.id = id + return { + ...state, + candidates: { ...state.candidates, [id]: candidate }, + reservations: reservation + ? { + ...state.reservations, + [reservation.id]: { ...reservation, status: "consumed", candidateID: id, updatedAt: now }, + } + : state.reservations, + revision: state.revision + 1, + updatedAt: now, + } + }) + return { ...out, state: await read(input.sessionID) } + } + + export async function observe(input: { + sessionID: string + candidateID: string + status: HarnessEvaluation.Status + score?: number + metrics?: Record + evidence?: string[] + feedback?: string + }) { + await JsonStore.update(file(input.sessionID), (data) => { + const state = parse(data) + const candidate = state.candidates[input.candidateID] + if (!candidate) throw new Error(`Unknown candidate ${input.candidateID}`) + if (candidate.result?.source === "verified" || candidate.result?.source === "screened") { + throw new Error(`An external result cannot be replaced by an observation`) + } + const now = Date.now() + const result = Result.parse({ + source: "observed", + status: input.status, + ...(input.score === undefined ? {} : { score: input.score }), + metrics: input.metrics ?? {}, + evidence: input.evidence ?? [], + feedback: input.feedback, + evaluatedAt: now, + }) + return { + ...state, + candidates: { ...state.candidates, [candidate.id]: { ...candidate, result } }, + revision: state.revision + 1, + updatedAt: now, + } + }) + return read(input.sessionID) + } + + export async function verify(input: { sessionID: string; candidateID: string }) { + const evaluation = (await HarnessEvaluation.list(input.sessionID)).findLast( + (item) => + item.subject?.type === "candidate" && item.subject.id === input.candidateID && HarnessEvaluation.final(item), + ) + if (!evaluation) { + const latest = await HarnessEvaluation.read(input.sessionID) + if (latest) throw new Error(`The recorded evaluation is not bound to candidate ${input.candidateID}`) + throw new Error(`No recorded external evaluation exists for session ${input.sessionID}`) + } + if (evaluation.subject?.type !== "candidate" || evaluation.subject.id !== input.candidateID) { + throw new Error(`The recorded evaluation is not bound to candidate ${input.candidateID}`) + } + const evolution = evaluation.evolutionReceiptID + ? await HarnessEvolution.read(input.sessionID, evaluation.evolutionReceiptID) + : undefined + await JsonStore.update(file(input.sessionID), (data) => { + const state = parse(data) + const candidate = state.candidates[input.candidateID] + if (!candidate) throw new Error(`Unknown candidate ${input.candidateID}`) + if (state.runID !== evaluation.runID) throw new Error(`Evaluation belongs to a different harness run`) + if (state.evaluator !== evaluation.evaluator.name) throw new Error(`Evaluation belongs to a different evaluator`) + if (state.direction !== "pass" && evaluation.status === "passed" && evaluation.score === undefined) { + throw new Error(`A ${state.direction} search requires a numeric evaluator score`) + } + if (evaluation.status === "passed") { + const missing = state.objectives.find((item) => evaluation.metrics[item.metric] === undefined) + if (missing) throw new Error(`Passing evaluation is missing declared objective metric ${missing.metric}`) + } + const now = Date.now() + const before = state.bestID + const result = Result.parse({ + source: "verified", + status: evaluation.status, + score: evaluation.score, + metrics: evaluation.metrics, + checks: evaluation.checks, + evidence: evaluation.evidence, + usage: evaluation.usage, + fidelity: evaluation.fidelity, + evaluator: evaluation.evaluator.name, + evolution: evolution ? { receiptID: evolution.receiptID, ...evolution.diagnostics } : undefined, + feedback: evaluation.notes, + evaluatedAt: evaluation.evaluatedAt, + ...(state.proposalPolicy === "adaptive-v4" + ? { recordedRevision: candidate.result?.recordedRevision ?? state.revision + 1 } + : {}), + }) + if (candidate.result?.source === "verified") { + if (JSON.stringify(candidate.result) === JSON.stringify(result)) return state + throw new Error(`A verified candidate result is immutable`) + } + const updated = { ...candidate, result } + const pool = { ...state.candidates, [candidate.id]: updated } + const provisional = { ...state, candidates: pool } + const best = ranked(provisional)[0] + const archiveIDs = archive(provisional).map((item) => item.id) + const improved = best?.id === candidate.id && before !== candidate.id + const next: State = { + ...provisional, + bestID: best?.id, + archiveIDs, + stalled: improved ? 0 : state.stalled + 1, + revision: state.revision + 1, + updatedAt: now, + } + if (reached(next, updated)) return stop(next, "objective_met", now) + if ( + candidates(next).length >= next.budget.candidates && + candidates(next).every((item) => item.result?.source === "verified") + ) { + return stop(next, "budget_exhausted", now) + } + return next + }) + return read(input.sessionID) + } + + export async function screen(input: { sessionID: string; candidateID: string; evaluation: HarnessEvaluation.Info }) { + const evaluation = HarnessEvaluation.Info.parse(input.evaluation) + if (evaluation.fidelity?.final !== false) throw new Error(`Screening requires a non-final fidelity stage`) + if (evaluation.subject?.type !== "candidate" || evaluation.subject.id !== input.candidateID) { + throw new Error(`Screening evaluation is not bound to candidate ${input.candidateID}`) + } + const [contract, journal] = await Promise.all([ + HarnessContract.read(input.sessionID), + HarnessEvaluation.list(input.sessionID), + ]) + if (!contract?.benchmark.fidelities) throw new Error(`Screening requires a bound fidelity plan`) + const plan = contract.benchmark.fidelities + if (!journal.some((item) => HarnessEvaluation.fingerprint(item) === HarnessEvaluation.fingerprint(evaluation))) { + throw new Error(`Screening requires a recorded external evaluation`) + } + const index = plan.findIndex((item) => item.id === evaluation.fidelity?.stage) + if (index < 0) throw new Error(`Screening fidelity stage is not in the bound contract`) + const evolution = evaluation.evolutionReceiptID + ? await HarnessEvolution.read(input.sessionID, evaluation.evolutionReceiptID) + : undefined + await JsonStore.update(file(input.sessionID), (data) => { + const state = parse(data) + const candidate = state.candidates[input.candidateID] + if (!candidate) throw new Error(`Unknown candidate ${input.candidateID}`) + if (candidate.result?.source === "verified") throw new Error(`A final verified result is immutable`) + const result = Result.parse({ + source: "screened", + status: evaluation.status, + score: evaluation.score, + metrics: evaluation.metrics, + checks: evaluation.checks, + evidence: evaluation.evidence, + usage: evaluation.usage, + fidelity: evaluation.fidelity, + evaluator: evaluation.evaluator.name, + evolution: evolution ? { receiptID: evolution.receiptID, ...evolution.diagnostics } : undefined, + feedback: evaluation.notes, + evaluatedAt: evaluation.evaluatedAt, + }) + if (candidate.result?.source === "screened") { + if (JSON.stringify(candidate.result) === JSON.stringify(result)) return state + const current = plan.findIndex((item) => item.id === candidate.result?.fidelity?.stage) + if (current >= index) throw new Error(`A screening result cannot replace the same or a later fidelity stage`) + } + return { + ...state, + candidates: { ...state.candidates, [candidate.id]: { ...candidate, result } }, + revision: state.revision + 1, + updatedAt: Date.now(), + } + }) + return read(input.sessionID) + } + + export async function finish(sessionID: string, reason: Exclude) { + await JsonStore.update(file(sessionID), (data) => { + const state = parse(data) + if (state.status === "completed") return state + return stop(state, reason) + }) + return read(sessionID) + } + + function legacyRoute(state: State): Route { + const pool = ranked(state) + if (!pool.length) { + return { + strategy: "seed", + parentIDs: [], + inspirationIDs: [], + targetIsland: vacancy(state), + reasons: ["no-verified-candidate"], + } + } + const pareto = archive(state) + const prioritized = [...pareto, ...pool.filter((candidate) => !pareto.some((item) => item.id === candidate.id))] + const distinct = prioritized.filter( + (candidate, index) => prioritized.findIndex((item) => item.branch === candidate.branch) === index, + ) + if (state.stalled >= state.budget.stall * 2) { + return { + strategy: "diverge", + parentIDs: [pool[0]!.id], + inspirationIDs: [], + targetIsland: pool[0]!.island, + reasons: [`stalled:${state.stalled}`, "strategy-level-mutation", "preserve-best"], + } + } + if ( + state.population.mode === "islands" && + state.population.count > 1 && + candidates(state).length >= state.population.migrationInterval && + candidates(state).length % state.population.migrationInterval === 0 + ) { + const source = pool[0]! + const target = Array.from( + { length: state.population.count - 1 }, + (_, index) => (source.island + index + 1) % state.population.count, + ).find((island) => pool.some((candidate) => candidate.island === island)) + const anchor = target === undefined ? undefined : pool.find((candidate) => candidate.island === target) + if (anchor) { + return { + strategy: "migrate", + parentIDs: [anchor.id], + inspirationIDs: [source.id], + targetIsland: anchor.island, + reasons: [ + `candidates:${candidates(state).length}`, + `ring:${source.island}->${anchor.island}`, + "verified-inspiration", + "new-artifact-required", + ], + } + } + } + if (state.stalled >= state.budget.stall && distinct.length >= 2) { + const best = pool[0]! + const complement = prioritized.find((candidate) => candidate.branch !== best.branch)! + return { + strategy: "fuse", + parentIDs: [best.id, complement.id], + inspirationIDs: [], + targetIsland: best.island, + reasons: [ + `stalled:${state.stalled}`, + "cross-branch-fusion", + ...(state.objectives.length ? [`pareto-frontier:${pareto.length}`, "multi-metric-complementarity"] : []), + ], + } + } + const progress = candidates(state).length / state.budget.candidates + const roots = candidates(state).filter( + (candidate) => + !candidate.parentIDs.length && + (candidate.result === undefined || + (candidate.result.source === "verified" && candidate.result.status === "passed")), + ) + const rootTarget = Math.min(4, Math.max(2, Math.ceil(Math.sqrt(state.budget.candidates)))) + if (progress < 0.35 && roots.length < rootTarget) { + return { + strategy: "explore", + parentIDs: [], + inspirationIDs: [], + targetIsland: vacancy(state), + reasons: [`budget-progress:${progress.toFixed(2)}`, `independent-roots:${roots.length}/${rootTarget}`], + } + } + if (progress >= 0.5) { + return { + strategy: "exploit", + parentIDs: [pool[0]!.id], + inspirationIDs: [], + targetIsland: pool[0]!.island, + reasons: [`budget-progress:${progress.toFixed(2)}`, "verified-rank"], + } + } + const counts = new Map() + for (const candidate of candidates(state)) counts.set(candidate.branch, (counts.get(candidate.branch) ?? 0) + 1) + const visits = distinct.map((candidate) => counts.get(candidate.branch) ?? 1) + const least = Math.min(...visits) + const warmup = distinct.filter((candidate) => (counts.get(candidate.branch) ?? 1) === least) + const scale = Math.max(1, ...pool.map((candidate) => Math.abs(candidate.result?.score ?? 0))) + const total = Math.max(1, candidates(state).length) + const value = (candidate: Candidate) => { + const raw = candidate.result?.score ?? 0 + const quality = state.direction === "minimize" ? -raw / scale : raw / scale + const count = counts.get(candidate.branch) ?? 1 + return quality + Math.sqrt((2 * Math.log(total + 1)) / count) + } + const diverse = warmup.toSorted((a, b) => value(b) - value(a) || order(state, a, b)) + return { + strategy: "explore", + parentIDs: [diverse[0]!.id], + inspirationIDs: [], + targetIsland: diverse[0]!.island, + reasons: [ + `budget-progress:${progress.toFixed(2)}`, + `branch-visits:${least}`, + warmup.length < distinct.length ? "ucb-minimum-visits" : "ucb-quality-exploration", + ...(state.objectives.length ? [`pareto-frontier:${pareto.length}`] : []), + ], + } + } + + function adaptiveRoute(state: State): Route { + const pool = ranked(state) + const roots = candidates(state).filter( + (candidate) => + !candidate.parentIDs.length && + (candidate.result === undefined || + candidate.result.source !== "verified" || + candidate.result.status === "passed"), + ) + const occupied = new Set(roots.map((candidate) => candidate.island)) + const initial = state.population.initial ?? 1 + const missing = Array.from({ length: initial }, (_, island) => island).find((island) => !occupied.has(island)) + if (!pool.length) { + return { + strategy: "seed", + parentIDs: [], + inspirationIDs: [], + targetIsland: missing ?? 0, + reasons: ["no-verified-candidate", "adaptive-cold-start"], + } + } + if (missing !== undefined) { + return { + strategy: "explore", + parentIDs: [], + inspirationIDs: [], + targetIsland: missing, + reasons: [`adaptive-initial-island:${missing}`, `initial-islands:${occupied.size}/${initial}`], + } + } + const adaptive = summary(state) + const spawn = Array.from({ length: state.population.count }, (_, island) => island).find( + (island) => !occupied.has(island), + ) + if (adaptive.globalStagnation && spawn !== undefined) { + return { + strategy: "explore", + parentIDs: [], + inspirationIDs: [], + targetIsland: spawn, + reasons: [ + `adaptive-stalled:${adaptive.stalled}`, + `adaptive-max-signal:${Math.max(...adaptive.islands.map((item) => item.accumulatedImprovement)).toFixed(6)}`, + `adaptive-island-spawn:${spawn}`, + ], + } + } + if (adaptive.globalStagnation) { + const best = pool[0]! + return { + strategy: "diverge", + parentIDs: [best.id], + inspirationIDs: [], + targetIsland: best.island, + reasons: [ + `adaptive-stalled:${adaptive.stalled}`, + `adaptive-signal:${adaptive.islands[best.island]!.accumulatedImprovement.toFixed(6)}`, + "meta-guidance", + "strategy-level-mutation", + ], + } + } + if ( + state.population.count > 1 && + candidates(state).length >= state.population.migrationInterval && + candidates(state).length % state.population.migrationInterval === 0 + ) { + const source = pool[0]! + const target = Array.from( + { length: state.population.count - 1 }, + (_, index) => (source.island + index + 1) % state.population.count, + ).find((island) => pool.some((candidate) => candidate.island === island)) + const anchor = target === undefined ? undefined : pool.find((candidate) => candidate.island === target) + if (anchor) { + return { + strategy: "migrate", + parentIDs: [anchor.id], + inspirationIDs: [source.id], + targetIsland: anchor.island, + reasons: [ + `adaptive-events:${adaptive.events}`, + `ring:${source.island}->${anchor.island}`, + "verified-inspiration", + "migration-reward-not-precredited", + ], + } + } + } + const pareto = archive(state) + const prioritized = [...pareto, ...pool.filter((candidate) => !pareto.some((item) => item.id === candidate.id))] + const distinct = prioritized.filter( + (candidate, index) => prioritized.findIndex((item) => item.branch === candidate.branch) === index, + ) + if (adaptive.stalled >= state.controller!.stagnation.patience && distinct.length >= 2) { + const best = pool[0]! + const complement = prioritized.find((candidate) => candidate.branch !== best.branch)! + return { + strategy: "fuse", + parentIDs: [best.id, complement.id], + inspirationIDs: [], + targetIsland: best.island, + reasons: [`adaptive-stalled:${adaptive.stalled}`, "cross-branch-fusion", "global-signal-not-yet-meta-stagnant"], + } + } + const selected = adaptive.selectedIsland + const target = + selected !== undefined && pool.some((candidate) => candidate.island === selected) ? selected : pool[0]!.island + const control = controller(state, target) + const local = pool.filter((candidate) => candidate.island === target) + const children = new Map(local.map((candidate) => [candidate.id, 0])) + for (const candidate of candidates(state)) { + for (const parent of candidate.parentIDs) { + if (children.has(parent)) children.set(parent, children.get(parent)! + 1) + } + } + const least = Math.min(...local.map((candidate) => children.get(candidate.id) ?? 0)) + const diverse = local.filter((candidate) => (children.get(candidate.id) ?? 0) === least) + const parent = control.explore + ? diverse[Math.min(diverse.length - 1, Math.floor(control.draw * diverse.length))]! + : local[0]! + return { + strategy: control.explore ? "explore" : "exploit", + parentIDs: [parent.id], + inspirationIDs: [], + targetIsland: target, + reasons: [ + `adaptive-island:${target}`, + `adaptive-visits:${control.visits}`, + `adaptive-reward:${control.rewardMean.toFixed(6)}`, + `adaptive-signal:${control.accumulatedImprovement.toFixed(6)}`, + `adaptive-intensity:${control.intensity.toFixed(6)}`, + `adaptive-draw:${control.draw.toFixed(6)}`, + control.explore ? "adaptive-exploration" : "adaptive-exploitation", + ], + } + } + + function route(state: State): Route { + if (state.proposalPolicy === "adaptive-v4") return adaptiveRoute(state) + return legacyRoute(state) + } + + const trail = (state: State, id: string) => { + const found: string[] = [] + const visit = (key: string) => { + const candidate = state.candidates[key] + if (!candidate || !verified(candidate) || found.includes(key)) return + found.push(key) + for (const parent of candidate.parentIDs) visit(parent) + } + visit(id) + return found + } + + const instructions = { + "bug-fix": + "Diagnose a concrete correctness, execution, or validity failure and repair its root cause. Use tools and local checks freely, but return one new runnable artifact for external evaluation.", + "external-dependency": + "Investigate whether a justified external method, dataset, library, simulator, or reference implementation can improve this lineage. Integrate only what the task contract permits and return one reproducible artifact.", + "architectural-change": + "Pursue a materially different algorithm, representation, decomposition, or system architecture. Do not satisfy this mandate with parameter-only changes; plan, test, debug, and revise before returning one artifact.", + composition: + "Compose complementary mechanisms from the leased lineage and context into one coherent artifact. Resolve incompatibilities explicitly and test the integration before returning it.", + "local-refinement": + "Keep the core approach and make a focused evidence-driven improvement. Use the verified trajectory and feedback to choose the smallest high-leverage edit, then test and revise it.", + pruning: + "Remove, disable, or simplify a component that may be unnecessary, harmful, or overfit. Preserve the task contract and return a runnable artifact that makes the causal change inspectable.", + refactor: + "Reorganize the artifact to improve clarity, modularity, stability, or future evolvability without relying on a new scientific premise. Preserve intended behavior and validate the result.", + efficiency: + "Improve runtime, memory, sample efficiency, tool use, or cost while protecting the declared primary and secondary objectives. Measure locally when possible, then return one artifact for external evaluation.", + "hyperparameter-tuning": + "Keep the algorithmic structure fixed and tune constants, thresholds, schedules, or other parameters using a principled local search. Avoid presenting retuning as a new architecture.", + } satisfies Record + + const operators = { + seed: [ + "architectural-change", + "composition", + "efficiency", + "external-dependency", + "local-refinement", + "pruning", + "refactor", + "hyperparameter-tuning", + "bug-fix", + ], + explore: [ + "architectural-change", + "composition", + "external-dependency", + "efficiency", + "local-refinement", + "pruning", + "refactor", + "hyperparameter-tuning", + "bug-fix", + ], + exploit: [ + "local-refinement", + "efficiency", + "hyperparameter-tuning", + "pruning", + "refactor", + "composition", + "bug-fix", + "architectural-change", + "external-dependency", + ], + fuse: [ + "composition", + "architectural-change", + "local-refinement", + "efficiency", + "pruning", + "refactor", + "hyperparameter-tuning", + "bug-fix", + "external-dependency", + ], + migrate: [ + "composition", + "local-refinement", + "architectural-change", + "efficiency", + "pruning", + "refactor", + "hyperparameter-tuning", + "bug-fix", + "external-dependency", + ], + diverge: [ + "architectural-change", + "external-dependency", + "composition", + "pruning", + "efficiency", + "refactor", + "bug-fix", + "local-refinement", + "hyperparameter-tuning", + ], + } satisfies Record + + function mandate(strategy: Strategy, ordinal: number): Mandate { + const operator = operators[strategy][ordinal % operators[strategy].length]! + const body = { + protocol: "agentic-variation-v1" as const, + operator, + instruction: instructions[operator], + } + return Mandate.parse({ id: mandateID(body), ...body }) + } + + const routeID = (input: Route) => + JSON.stringify({ + strategy: input.strategy, + parentIDs: input.parentIDs.toSorted(), + inspirationIDs: input.inspirationIDs.toSorted(), + targetIsland: input.targetIsland, + }) + + function routes(state: State) { + const primary = route(state) + const pool = ranked(state) + if (!pool.length || !primary.parentIDs.length) return [primary] + const pareto = archive(state) + const ordered = [...pareto, ...pool.filter((candidate) => !pareto.some((item) => item.id === candidate.id))] + const distinct = ordered.filter( + (candidate, index) => ordered.findIndex((item) => item.branch === candidate.branch) === index, + ) + const options: Route[] = [primary] + const seen = new Set([routeID(primary)]) + const add = (choice: Route) => { + const id = routeID(choice) + if (seen.has(id)) return + seen.add(id) + options.push(choice) + } + if (primary.strategy === "fuse") { + const best = pool[0]! + for (const candidate of distinct) { + if (candidate.branch === best.branch) continue + add({ + strategy: "fuse", + parentIDs: [best.id, candidate.id], + inspirationIDs: [], + targetIsland: best.island, + reasons: ["portfolio-cross-branch-fusion", `source-branch:${candidate.branch}`], + }) + } + return options + } + if (primary.strategy === "migrate") { + const source = pool[0]! + const islands = pool.filter( + (candidate, index) => pool.findIndex((item) => item.island === candidate.island) === index, + ) + for (const anchor of islands) { + if (anchor.island === source.island) continue + add({ + strategy: "migrate", + parentIDs: [anchor.id], + inspirationIDs: [source.id], + targetIsland: anchor.island, + reasons: ["portfolio-ring-migration", `source:${source.island}`, `target:${anchor.island}`], + }) + } + return options + } + for (const candidate of distinct) { + add({ + strategy: primary.strategy, + parentIDs: [candidate.id], + inspirationIDs: [], + targetIsland: candidate.island, + reasons: ["portfolio-verified-lineage", `branch:${candidate.branch}`], + }) + } + return options + } + + function materialize(state: State, choice: Route): Recommendation { + const control = state.proposalPolicy === "adaptive-v4" ? controller(state, choice.targetIsland) : undefined + const mode: Mode = + choice.strategy === "seed" + ? "single-pass" + : choice.strategy === "exploit" || + (choice.strategy === "explore" && choice.parentIDs.length && !control?.explore) + ? "diff" + : "stepwise" + const roots = [...choice.parentIDs, ...choice.inspirationIDs] + const trails = roots.map((id) => trail(state, id)) + const depth = Math.max(0, ...trails.map((items) => items.length)) + const contextIDs = Array.from({ length: depth }, (_, index) => trails.map((items) => items[index])) + .flat() + .filter((id): id is string => !!id) + .filter((id, index, items) => items.indexOf(id) === index) + .slice(0, 6) + const body = { + revision: state.revision, + strategy: choice.strategy, + mode, + parentIDs: choice.parentIDs, + inspirationIDs: choice.inspirationIDs, + targetIsland: choice.targetIsland, + contextIDs, + control, + } + return { id: leaseID(state, body), ...body, reasons: choice.reasons } + } + + function portfolio(state: State, count: number, start: number, openRoots: number) { + const options = routes(state) + return Array.from({ length: count }, (_, index) => { + const selected = options[index % options.length]! + const choice = selected.parentIDs.length + ? selected + : { + ...selected, + targetIsland: (selected.targetIsland + openRoots + index) % state.population.count, + reasons: [...selected.reasons, `portfolio-root:${index}`], + } + return { + recommendation: materialize(state, choice), + mandate: mandate(choice.strategy, start + index), + } + }) + } + + export function recommend(state: State): Recommendation { + const parsed = State.parse(state) + topology(parsed) + return materialize(parsed, route(parsed)) + } +} diff --git a/backend/cli/src/session/harness/semantic.ts b/backend/cli/src/session/harness/semantic.ts new file mode 100644 index 00000000..8e7ee22f --- /dev/null +++ b/backend/cli/src/session/harness/semantic.ts @@ -0,0 +1,340 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessSemantic { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(240), + }) + .strict() + export type Subject = z.infer + + const Criterion = z + .object({ + id: z.string().min(1).max(100), + status: z.enum(["passed", "failed", "inconclusive"]), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + const Shortcut = z + .object({ + id: z.string().min(1).max(100), + observed: z.boolean(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + export const Review = z + .object({ + actor: z.string().min(1).max(200), + sessionID: z.string().min(1).max(240), + correctness: z.enum(["passed", "failed", "inconclusive"]), + alignment: z.enum(["intended", "reasonable_alternative", "misinterpreted", "ambiguous"]), + novelty: HarnessContract.Novelty, + vacuous: z.boolean(), + confidence: z.number().finite().min(0).max(1), + criteria: z + .array(Criterion) + .min(1) + .max(24) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Semantic review criterion IDs must be unique", + ), + shortcuts: z + .array(Shortcut) + .min(1) + .max(24) + .refine( + (items) => new Set(items.map((item) => item.id)).size === items.length, + "Semantic review shortcut IDs must be unique", + ), + literatureRefs: z.array(z.string().min(1).max(1_000)).max(32).default([]), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(64), + summary: z.string().min(1).max(4_000), + reviewedAt: z.number().int().positive(), + }) + .strict() + export type Review = z.infer + + export const Submit = z + .object({ + sessionID: z.string().min(1).max(240), + reviewerToken: Token, + subject: Subject, + reviews: z.array(Review).min(2).max(5), + }) + .strict() + export type Submit = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + reviewerToken: Token, + }) + .strict() + export type Access = z.infer + + const ReceiptBase = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("semantic-audit-receipt-v1"), + receiptID: Hash, + protocolSHA256: Hash, + sourceSessionID: z.string().min(1), + subject: Subject, + reviewer: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + source: z.enum(["gate", "human", "external"]), + }) + .strict(), + scope: HarnessContract.SemanticAudit.shape.scope, + reviews: z.array(Review).min(2).max(5), + status: z.enum(["meaningful", "technical_only", "ambiguous", "failed"]), + failures: z.array(z.string().min(1).max(500)).max(256), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(256), + reviewedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + + export const Receipt = ReceiptBase.superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Semantic audit receipt content hash is invalid" }) + }) + export type Receipt = z.infer + + const root = path.join(Global.Path.data, "harness", "semantics") + const file = (receiptID: string) => path.join(root, `${receiptID}.json`) + const rank: Record = { + not_required: -1, + known: 0, + rediscovery: 1, + minor: 2, + publication: 3, + major: 4, + } + + function derive(protocol: HarnessContract.SemanticAudit, reviews: Review[]) { + const incorrect = reviews.flatMap((review) => + review.correctness === "failed" ? [`${review.actor}:correctness_failed`] : [], + ) + const uncertain = reviews.flatMap((review) => [ + ...(review.correctness === "inconclusive" ? [`${review.actor}:correctness_inconclusive`] : []), + ...(review.alignment === "ambiguous" ? [`${review.actor}:alignment_ambiguous`] : []), + ...(review.confidence < protocol.minConfidence ? [`${review.actor}:low_confidence`] : []), + ...review.criteria.flatMap((item) => + item.status === "inconclusive" ? [`${review.actor}:criterion_${item.id}_inconclusive`] : [], + ), + ]) + const technical = reviews.flatMap((review) => [ + ...(review.alignment === "misinterpreted" ? [`${review.actor}:problem_misinterpreted`] : []), + ...(review.vacuous ? [`${review.actor}:vacuous_solution`] : []), + ...review.criteria.flatMap((item) => + item.status === "failed" ? [`${review.actor}:criterion_${item.id}_failed`] : [], + ), + ...review.shortcuts.flatMap((item) => (item.observed ? [`${review.actor}:shortcut_${item.id}_observed`] : [])), + ...(rank[review.novelty] < rank[protocol.scope.noveltyFloor] + ? [`${review.actor}:novelty_${review.novelty}_below_${protocol.scope.noveltyFloor}`] + : []), + ]) + const status = incorrect.length + ? ("failed" as const) + : uncertain.length + ? ("ambiguous" as const) + : technical.length + ? ("technical_only" as const) + : ("meaningful" as const) + return { status, failures: [...incorrect, ...uncertain, ...technical] } + } + + function inspect(protocol: HarnessContract.SemanticAudit, input: Review[]) { + if (input.length < protocol.minReviewers) { + throw new Error(`Semantic review requires at least ${protocol.minReviewers} independent reviewers`) + } + const reviews = input.toSorted( + (left, right) => left.actor.localeCompare(right.actor) || left.sessionID.localeCompare(right.sessionID), + ) + if (new Set(reviews.map((item) => `${item.actor}\0${item.sessionID}`)).size !== reviews.length) { + throw new Error(`Semantic reviewers must have unique actor-session identities`) + } + if (new Set(reviews.map((item) => item.actor)).size !== reviews.length) { + throw new Error(`Semantic reviewers must use distinct actors`) + } + if (new Set(reviews.map((item) => item.sessionID)).size !== reviews.length) { + throw new Error(`Semantic reviewers must use distinct sessions`) + } + const criteria = protocol.scope.criteria.map((item) => item.id).toSorted() + const shortcuts = protocol.scope.forbiddenShortcuts.map((item) => item.id).toSorted() + for (const review of reviews) { + if (JSON.stringify(review.criteria.map((item) => item.id).toSorted()) !== JSON.stringify(criteria)) { + throw new Error(`Semantic review criteria do not match the frozen problem scope`) + } + if (JSON.stringify(review.shortcuts.map((item) => item.id).toSorted()) !== JSON.stringify(shortcuts)) { + throw new Error(`Semantic review shortcuts do not match the frozen problem scope`) + } + if (protocol.scope.noveltyFloor !== "not_required" && !review.literatureRefs.length) { + throw new Error(`Novelty review requires literature evidence from the frozen corpus scope`) + } + } + return { reviews, ...derive(protocol, reviews) } + } + + function evidence(reviews: Review[]) { + return [ + ...new Set( + reviews.flatMap((review) => [ + ...review.evidence, + ...review.literatureRefs, + ...review.criteria.flatMap((item) => item.evidence), + ...review.shortcuts.flatMap((item) => item.evidence), + ]), + ), + ].toSorted() + } + + async function born(contract: HarnessContract.Info, subject: Subject) { + if (subject.type === "run") { + if (subject.id !== contract.runID) throw new Error(`Semantic review run subject does not match its contract`) + return contract.createdAt + } + const state = await import("./search") + .then((module) => module.HarnessSearch.read(contract.sessionID)) + .catch(() => null) + const candidate = state?.runID === contract.runID ? state.candidates[subject.id] : undefined + if (!candidate) throw new Error(`Semantic review candidate does not exist in the bound search`) + return candidate.createdAt + } + + export function prompt(contract: HarnessContract.Info) { + const protocol = contract.semanticAudit + if (!protocol) return "" + return [ + "", + "The evaluator froze this problem-meaning contract before execution. Treat it as evaluation policy, not as evidence that your answer passes it.", + `Objective SHA-256: ${protocol.scope.objectiveSHA256}`, + `Required meaning criteria (JSON): ${JSON.stringify(protocol.scope.criteria)}`, + `Forbidden shortcuts (JSON): ${JSON.stringify(protocol.scope.forbiddenShortcuts)}`, + `Literature scope: cutoff=${protocol.scope.literature.cutoff}; corpus_sha256=${protocol.scope.literature.corpusSHA256}`, + `Minimum novelty: ${protocol.scope.noveltyFloor}`, + `Independent review: at least ${protocol.minReviewers} reviewers at confidence >= ${protocol.minConfidence}`, + "A technically valid, vacuous, misinterpreted, shortcut-dependent, or insufficiently novel answer cannot pass the final evaluation.", + "Do not claim semantic compliance yourself; produce observable evidence for an independent panel.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + return contract ? prompt(contract) : "" + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const protocol = contract.semanticAudit + if (!protocol) throw new Error(`Harness contract does not require semantic review`) + if (value.sessionID !== contract.sessionID) throw new Error(`Semantic review session does not match its contract`) + const createdAt = await born(contract, value.subject) + const audit = inspect(protocol, value.reviews) + const now = Date.now() + for (const review of audit.reviews) { + if (review.reviewedAt < createdAt || review.reviewedAt > now) { + throw new Error(`Semantic review timestamp is outside the bound subject interval`) + } + } + const stable = { + schemaVersion: 1 as const, + protocolVersion: "semantic-audit-receipt-v1" as const, + protocolSHA256: digest(protocol), + sourceSessionID: contract.sessionID, + subject: value.subject, + reviewer: protocol.reviewer, + scope: protocol.scope, + reviews: audit.reviews, + status: audit.status, + failures: audit.failures, + evidence: evidence(audit.reviews), + reviewedAt: Math.max(...audit.reviews.map((item) => item.reviewedAt)), + } + const stored = { ...stable, recordedAt: now } + const receipt = Receipt.parse({ ...stored, receiptID: digest(stored) }) + await JsonStore.update(file(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Semantic audit receipt is immutable once recorded`) + }) + const saved = await read(receipt.receiptID) + if (!saved) throw new Error(`Semantic audit receipt was not durable after recording`) + return saved + } + + export async function read(receiptID: string) { + const id = Hash.parse(receiptID) + const data = await JsonStore.read(file(id)) + const parsed = Receipt.safeParse(data) + return parsed.success && parsed.data.receiptID === id ? parsed.data : null + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: Subject + evaluatedAt: number + recordedAt: number + requirePassed: boolean + }) { + const receipt = await read(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt semantic audit receipt ${input.receiptID}`) + const protocol = input.contract.semanticAudit + if (!protocol) throw new Error(`Evaluation cites a semantic receipt without a bound semantic audit protocol`) + if (receipt.protocolSHA256 !== digest(protocol)) { + throw new Error(`Semantic audit receipt does not match the bound review protocol`) + } + if (JSON.stringify(receipt.reviewer) !== JSON.stringify(protocol.reviewer)) { + throw new Error(`Semantic audit receipt uses a different review authority`) + } + if (JSON.stringify(receipt.scope) !== JSON.stringify(protocol.scope)) { + throw new Error(`Semantic audit receipt changed the frozen problem scope`) + } + if (receipt.sourceSessionID !== input.contract.sessionID) { + throw new Error(`Semantic audit receipt belongs to a different harness session`) + } + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Semantic audit receipt belongs to a different evaluation subject`) + } + const createdAt = await born(input.contract, input.subject) + const audit = inspect(protocol, receipt.reviews) + if ( + receipt.status !== audit.status || + JSON.stringify(receipt.failures) !== JSON.stringify(audit.failures) || + JSON.stringify(receipt.evidence) !== JSON.stringify(evidence(audit.reviews)) || + receipt.reviewedAt !== Math.max(...audit.reviews.map((item) => item.reviewedAt)) + ) { + throw new Error(`Semantic audit receipt does not match backend-derived review state`) + } + if (audit.reviews.some((review) => review.reviewedAt < createdAt || review.reviewedAt > receipt.recordedAt)) { + throw new Error(`Semantic audit receipt contains a review outside the bound subject interval`) + } + if (receipt.reviewedAt > input.evaluatedAt || receipt.recordedAt > input.recordedAt) { + throw new Error(`Evaluation predates its semantic audit receipt`) + } + if (input.requirePassed && receipt.status !== "meaningful") { + throw new Error(`A passing final evaluation requires a meaningful semantic audit receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/simulation.ts b/backend/cli/src/session/harness/simulation.ts new file mode 100644 index 00000000..216b0f24 --- /dev/null +++ b/backend/cli/src/session/harness/simulation.ts @@ -0,0 +1,330 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessSimulation { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + + export const Artifact = z + .object({ + uri: z.string().min(1).max(2_048), + sha256: Hash, + }) + .strict() + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(240), + artifact: Artifact, + }) + .strict() + + export const Level = z + .object({ + label: z.string().min(1).max(120), + h: z.number().finite().positive(), + error: z.number().finite().positive(), + residual: z.number().finite().nonnegative(), + invariants: z + .record(z.string().min(1).max(100), z.number().finite().nonnegative()) + .refine((value) => Object.keys(value).length <= 32, "A level may report at most 32 invariants"), + }) + .strict() + + export const Stress = z + .object({ + id: HarnessContract.SimulationStress, + status: z.enum(["passed", "failed", "inconclusive"]), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + export const Submit = z + .object({ + schemaVersion: z.literal(1), + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + subject: Subject, + engine: HarnessContract.SimulationEngine, + problemSHA256: Hash, + reference: HarnessContract.SimulationReference, + validationInputSHA256: Hash, + levels: z + .array(Level) + .min(3) + .max(24) + .refine( + (items) => new Set(items.map((item) => item.label)).size === items.length, + "Level labels must be unique", + ), + stressTests: z + .array(Stress) + .max(HarnessContract.SimulationStress.options.length) + .refine((items) => new Set(items.map((item) => item.id)).size === items.length, "Stress tests must be unique"), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + }) + .strict() + export type Submit = z.input + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + + export const Info = z + .object({ + schemaVersion: z.literal(1), + receiptID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + subject: Subject, + evaluator: z + .object({ + name: z.string().min(1).max(200), + version: z.string().min(1).max(200), + source: z.enum(["benchmark", "gate", "external"]), + }) + .strict(), + engine: HarnessContract.SimulationEngine, + problemSHA256: Hash, + reference: HarnessContract.SimulationReference, + validationInputSHA256: Hash, + levels: z.array(Level).min(3).max(24), + observedOrders: z.array(z.number().finite()).max(23), + medianObservedOrder: z.number().finite(), + stressTests: z.array(Stress).max(HarnessContract.SimulationStress.options.length), + checks: z + .object({ + enoughLevels: z.boolean(), + resolutionDecreases: z.boolean(), + errorDecreases: z.boolean(), + observedOrder: z.boolean(), + residualBound: z.boolean(), + invariants: z.record(z.string().min(1).max(100), z.boolean()), + stressTests: z.partialRecord(HarnessContract.SimulationStress, z.boolean()), + }) + .strict(), + status: z.enum(["passed", "failed"]), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(128), + evaluatedAt: z.number().int().positive(), + }) + .strict() + export type Info = z.infer + + const State = z + .object({ + schemaVersion: z.literal(1), + items: z.record(Hash, Info), + order: z.array(Hash), + }) + .strict() + .superRefine((value, ctx) => { + if (new Set(value.order).size !== value.order.length) { + ctx.addIssue({ code: "custom", path: ["order"], message: "Simulation receipt order must be unique" }) + } + for (const id of value.order) { + const receipt = value.items[id] + if (!receipt) { + ctx.addIssue({ code: "custom", path: ["order"], message: `Simulation receipt ${id} is missing` }) + continue + } + if (receipt.receiptID !== id) { + ctx.addIssue({ code: "custom", path: ["items", id], message: `Simulation receipt key does not match its ID` }) + } + const payload = structuredClone(receipt) as Record + delete payload.receiptID + if (digest(payload) !== id) { + ctx.addIssue({ code: "custom", path: ["items", id], message: `Simulation receipt content hash is invalid` }) + } + } + for (const id of Object.keys(value.items)) { + if (value.order.includes(id)) continue + ctx.addIssue({ + code: "custom", + path: ["items", id], + message: `Simulation receipt is absent from journal order`, + }) + } + }) + type State = z.infer + + const root = path.join(Global.Path.data, "harness", "simulations") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const empty = (): State => ({ schemaVersion: 1, items: {}, order: [] }) + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + const normalize = (value: Record) => + Object.fromEntries(Object.entries(value).toSorted(([left], [right]) => left.localeCompare(right))) + + function state(input: Record) { + return State.parse(Object.keys(input).length ? input : empty()) + } + + function median(values: number[]) { + const sorted = values.toSorted((left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + if (sorted.length % 2) return sorted[middle]! + return (sorted[middle - 1]! + sorted[middle]!) / 2 + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + const bound = HarnessContract.Info.parse(contract) + const protocol = bound.simulation + if (!protocol) throw new Error(`No simulator validation protocol is bound to session ${value.sessionID}`) + if (bound.sessionID !== value.sessionID || bound.runID !== value.runID) { + throw new Error(`Simulation receipt does not match the bound harness run`) + } + if (value.evaluatedAt < bound.createdAt) throw new Error(`Simulation receipt predates the harness contract`) + if (!same(value.engine, protocol.engine)) throw new Error(`Simulation engine does not match the bound protocol`) + if (value.problemSHA256 !== protocol.problemSHA256) { + throw new Error(`Simulation problem does not match the bound protocol`) + } + if (!same(value.reference, protocol.reference)) { + throw new Error(`Simulation reference does not match the bound protocol`) + } + if (value.levels.length > protocol.validation.maxLevels) { + throw new Error(`Simulation receipt exceeds the bound refinement-level limit`) + } + const required = protocol.validation.requiredStressTests.toSorted() + const submitted = value.stressTests.map((item) => item.id).toSorted() + if (!same(required, submitted)) throw new Error(`Simulation stress tests do not match the bound protocol`) + const invariantNames = Object.keys(protocol.validation.invariantTolerances).toSorted() + for (const level of value.levels) { + if (!same(Object.keys(level.invariants).toSorted(), invariantNames)) { + throw new Error(`Simulation level ${level.label} does not report every bound invariant`) + } + } + if (value.subject.type === "run" && value.subject.id !== bound.runID) { + throw new Error(`Run simulation receipt subject does not match the contract run`) + } + if (value.subject.type === "candidate") { + const search = await import("./search").then((module) => module.HarnessSearch.read(value.sessionID)) + const candidate = search.candidates[value.subject.id] + if (!candidate) throw new Error(`Simulation receipt candidate does not exist in the bound search`) + if (!same(candidate.artifact, value.subject.artifact)) { + throw new Error(`Simulation receipt artifact does not match the candidate artifact`) + } + } + + const levels = value.levels.map((level) => ({ ...level, invariants: normalize(level.invariants) })) + const pairs = levels.slice(0, -1).map((level, index) => [level, levels[index + 1]!] as const) + const resolution = pairs.every(([left, right]) => left.h > right.h) + const error = pairs.every(([left, right]) => left.error > right.error) + const orders = pairs + .map(([left, right]) => Math.log(left.error / right.error) / Math.log(left.h / right.h)) + .filter(Number.isFinite) + const order = orders.length === pairs.length ? median(orders) : 0 + const invariantChecks = Object.fromEntries( + invariantNames.map((name) => [ + name, + levels.every((level) => level.invariants[name]! <= protocol.validation.invariantTolerances[name]!), + ]), + ) + const stress = Object.fromEntries(value.stressTests.map((item) => [item.id, item.status === "passed"])) + const checks = { + enoughLevels: levels.length >= protocol.validation.minLevels, + resolutionDecreases: resolution, + errorDecreases: error, + observedOrder: + resolution && + error && + orders.length === pairs.length && + orders.every((value) => value >= protocol.validation.expectedOrder - protocol.validation.orderTolerance), + residualBound: levels.every((level) => level.residual <= protocol.validation.maxResidual), + invariants: invariantChecks, + stressTests: stress, + } + const passed = + checks.enoughLevels && + checks.resolutionDecreases && + checks.errorDecreases && + checks.observedOrder && + checks.residualBound && + Object.values(checks.invariants).every(Boolean) && + Object.values(checks.stressTests).every(Boolean) + const payload = { + schemaVersion: 1 as const, + runID: value.runID, + sessionID: value.sessionID, + contractFingerprint: HarnessContract.fingerprint(bound), + subject: value.subject, + evaluator: { + name: bound.benchmark.evaluator, + version: bound.benchmark.evaluatorVersion!, + source: bound.benchmark.evaluatorSource!, + }, + engine: value.engine, + problemSHA256: value.problemSHA256, + reference: value.reference, + validationInputSHA256: value.validationInputSHA256, + levels, + observedOrders: orders, + medianObservedOrder: order, + stressTests: value.stressTests.toSorted((left, right) => left.id.localeCompare(right.id)), + checks, + status: passed ? ("passed" as const) : ("failed" as const), + evidence: value.evidence.toSorted(), + evaluatedAt: value.evaluatedAt, + } + const receipt = Info.parse({ ...payload, receiptID: digest(payload) }) + await JsonStore.update(file(value.sessionID), (data) => { + const current = state(data) + const existing = current.items[receipt.receiptID] + if (existing) return current + return State.parse({ + ...current, + items: { ...current.items, [receipt.receiptID]: receipt }, + order: [...current.order, receipt.receiptID], + }) + }) + return receipt + } + + export async function read(sessionID: string, receiptID: string) { + const current = state(await JsonStore.read(file(sessionID))) + return current.items[Hash.parse(receiptID)] ?? null + } + + export async function list(sessionID: string) { + const current = state(await JsonStore.read(file(sessionID))) + return current.order.map((id) => current.items[id]!) + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + candidateID?: string + requirePassed: boolean + evaluatedAt: number + }) { + const receipt = await read(input.contract.sessionID, input.receiptID) + if (!receipt) throw new Error(`Simulation receipt ${input.receiptID} does not exist`) + if (receipt.runID !== input.contract.runID) throw new Error(`Simulation receipt does not match the harness run`) + if (receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract)) { + throw new Error(`Simulation receipt does not match the immutable harness contract`) + } + const type = input.candidateID ? "candidate" : "run" + const id = input.candidateID ?? input.contract.runID + if (receipt.subject.type !== type || receipt.subject.id !== id) { + throw new Error(`Simulation receipt does not match the evaluation subject`) + } + if (receipt.evaluatedAt > input.evaluatedAt) { + throw new Error(`Simulation evaluation predates its referenced validation receipt`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A passing evaluation requires a passing simulation receipt`) + } + return receipt + } +} diff --git a/backend/cli/src/session/harness/skill.ts b/backend/cli/src/session/harness/skill.ts new file mode 100644 index 00000000..16f63a25 --- /dev/null +++ b/backend/cli/src/session/harness/skill.ts @@ -0,0 +1,395 @@ +import fs from "fs/promises" +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { runtimeRegexPass, classifierInjectionRegexPass, suspiciousRegexPass } from "@/skill/install/review" +import { JsonStore } from "@/util/jsonstore" +import { HarnessAdapter } from "./adapter" +import { HarnessContract } from "./contract" +import { HarnessEvaluation } from "./evaluation" + +export namespace HarnessSkill { + const Name = z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/) + const SHA = z.string().regex(/^[a-f0-9]{64}$/) + + export const ProposalInput = z + .object({ + name: Name, + description: z.string().min(1).max(500), + content: z.string().min(1).max(64_000), + origin: z.enum(["conversation", "rsi"]), + sessionID: z.string().min(1).optional(), + runID: z.string().min(1).optional(), + createdAt: z.number().int().positive().optional(), + }) + .strict() + export type ProposalInput = z.input + type Proposal = z.output + + export const Trigger = z + .object({ + datasetSHA256: SHA, + split: z.literal("held_out"), + examples: z.number().int().min(20), + truePositive: z.number().int().nonnegative(), + falsePositive: z.number().int().nonnegative(), + trueNegative: z.number().int().nonnegative(), + falseNegative: z.number().int().nonnegative(), + }) + .strict() + .superRefine((value, ctx) => { + const total = value.truePositive + value.falsePositive + value.trueNegative + value.falseNegative + if (total !== value.examples) { + ctx.addIssue({ code: "custom", path: ["examples"], message: "Trigger confusion counts must sum to examples" }) + } + if (!value.truePositive && !value.falseNegative) { + ctx.addIssue({ code: "custom", path: ["truePositive"], message: "Trigger evaluation needs positive examples" }) + } + if (!value.trueNegative && !value.falsePositive) { + ctx.addIssue({ code: "custom", path: ["trueNegative"], message: "Trigger evaluation needs negative examples" }) + } + }) + export type Trigger = z.infer + + export const Evidence = z + .object({ + id: SHA, + proposalSHA256: SHA, + benchmark: z + .object({ + name: z.string().min(1), + version: z.string().min(1), + taskID: z.string().min(1), + split: z.enum(["held_out", "release"]), + metric: z.string().optional(), + direction: z.enum(["maximize", "minimize", "pass"]), + }) + .strict(), + candidate: z + .object({ + sessionID: z.string().min(1), + runID: z.string().min(1), + status: HarnessEvaluation.Status, + score: z.number().finite().optional(), + evaluationSHA256: SHA, + }) + .strict(), + control: z + .object({ + sessionID: z.string().min(1), + runID: z.string().min(1), + status: HarnessEvaluation.Status, + score: z.number().finite().optional(), + evaluationSHA256: SHA, + }) + .strict(), + nonregressing: z.boolean(), + improved: z.boolean(), + trigger: Trigger.safeExtend({ + precision: z.number().min(0).max(1), + recall: z.number().min(0).max(1), + }), + evaluator: z.object({ name: z.string().min(1), version: z.string().min(1) }).strict(), + recordedAt: z.number().int().positive(), + }) + .strict() + export type Evidence = z.infer + + export const Manifest = z + .object({ + schemaVersion: z.literal(1), + name: Name, + description: z.string().min(1).max(500), + contentSHA256: SHA, + origin: z.enum(["conversation", "rsi"]), + source: z.object({ sessionID: z.string().min(1).optional(), runID: z.string().min(1).optional() }).strict(), + status: z.enum(["pending", "qualified", "promoted", "rejected"]), + evidence: z.array(Evidence).max(100), + criteria: z + .object({ + tasks: z.literal(3), + improvements: z.literal(2), + triggerPrecision: z.literal(0.8), + triggerRecall: z.literal(0.8), + }) + .strict(), + createdAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + promotedAt: z.number().int().positive().optional(), + }) + .strict() + export type Manifest = z.infer + + export const Attestation = z + .object({ + name: Name, + candidate: z.object({ sessionID: z.string().min(1), evaluatorToken: z.string().min(32).max(1_024) }).strict(), + control: z.object({ sessionID: z.string().min(1), evaluatorToken: z.string().min(32).max(1_024) }).strict(), + trigger: Trigger, + recordedAt: z.number().int().positive().optional(), + }) + .strict() + export type Attestation = z.input + + const proposals = path.join(Global.Path.data, "learned-skill-proposals") + const active = path.join(Global.Path.data, "learned-skills") + const dir = (name: string) => path.join(proposals, name) + const manifest = (name: string) => path.join(dir(name), "manifest.json") + const skill = (name: string) => path.join(dir(name), "SKILL.md") + const digest = (input: string) => new Bun.CryptoHasher("sha256").update(input).digest("hex") + + function frontmatter(input: Proposal) { + const header = input.content.match(/^---\r?\n([\s\S]*?)\r?\n---/) + if (!header) throw new Error(`Learned skill proposals require YAML frontmatter`) + const name = header[1]! + .match(/^name:\s*(.+)$/m)?.[1] + ?.trim() + .replace(/^['"]|['"]$/g, "") + const description = header[1]! + .match(/^description:\s*(.+)$/m)?.[1] + ?.trim() + .replace(/^['"]|['"]$/g, "") + if (name !== input.name) throw new Error(`Skill frontmatter name must match ${input.name}`) + if (description !== input.description) throw new Error(`Skill frontmatter description must match the proposal`) + } + + function review(input: Proposal) { + const entry = { + namespace: "learned", + name: input.name, + description: input.description, + content: input.content, + scripts: [], + references: [], + } + const rejected = [...runtimeRegexPass([entry]).rejected, ...classifierInjectionRegexPass([entry]).rejected] + if (rejected.length) throw new Error(`Learned skill rejected: ${rejected.map((item) => item.reason).join(", ")}`) + const warnings = suspiciousRegexPass([entry]).warnings + if (warnings.length) throw new Error(`Learned skill requires manual review: ${warnings[0]!.pattern}`) + } + + export async function propose(input: ProposalInput) { + const value = ProposalInput.parse(input) + frontmatter(value) + review(value) + const sha = digest(value.content) + const now = Date.now() + const proposal = Manifest.parse({ + schemaVersion: 1, + name: value.name, + description: value.description, + contentSHA256: sha, + origin: value.origin, + source: { sessionID: value.sessionID, runID: value.runID }, + status: "pending", + evidence: [], + criteria: { tasks: 3, improvements: 2, triggerPrecision: 0.8, triggerRecall: 0.8 }, + createdAt: value.createdAt ?? now, + updatedAt: now, + }) + await fs.mkdir(dir(value.name), { recursive: true }) + await JsonStore.update(manifest(value.name), (data) => { + if (!Object.keys(data).length) return proposal + const current = Manifest.parse(data) + if (current.contentSHA256 === sha) return current + throw new Error(`Skill proposal ${value.name} is immutable; use a new versioned name`) + }) + const exists = await Bun.file(skill(value.name)).exists() + if (exists && digest(await Bun.file(skill(value.name)).text()) !== sha) { + throw new Error(`Skill proposal content does not match its immutable manifest`) + } + if (!exists) await Bun.write(skill(value.name), value.content, { mode: 0o600 }) + return read(value.name) + } + + export async function read(name: string) { + const data = await JsonStore.read(manifest(Name.parse(name))) + const parsed = Manifest.safeParse(data) + return parsed.success ? parsed.data : null + } + + export async function list() { + const names = await fs.readdir(proposals).catch(() => []) + const items = await Promise.all(names.map((name) => read(name).catch(() => null))) + return items.filter((item): item is Manifest => item !== null).toSorted((a, b) => b.updatedAt - a.updatedAt) + } + + export function assess(input: Evidence[]) { + const evidence = input.map((item) => Evidence.parse(item)) + const task = (item: Evidence) => `${item.benchmark.name}\0${item.benchmark.version}\0${item.benchmark.taskID}` + const tasks = new Set(evidence.map(task)).size + const improvements = new Set(evidence.filter((item) => item.improved).map(task)).size + const regressions = evidence.filter((item) => !item.nonregressing).length + const failures = evidence.filter((item) => item.candidate.status !== "passed").length + const triggerFailures = evidence.filter((item) => item.trigger.precision < 0.8 || item.trigger.recall < 0.8).length + const triggers = new Set( + evidence + .filter((item) => item.trigger.precision >= 0.8 && item.trigger.recall >= 0.8) + .map((item) => item.trigger.datasetSHA256), + ).size + return { + qualified: tasks >= 3 && improvements >= 2 && !regressions && !failures && !triggerFailures && triggers >= 1, + tasks, + improvements, + regressions, + failures, + triggerFailures, + triggerDatasets: triggers, + } + } + + function comparable(input: { proposal: Manifest; candidate: HarnessContract.Info; control: HarnessContract.Info }) { + const match = input.candidate.skills.filter( + (item) => item.name === input.proposal.name && item.sha256 === input.proposal.contentSHA256, + ) + if (match.length !== 1) throw new Error(`Candidate contract must pin the exact proposed skill SHA`) + if (input.control.skills.some((item) => item.name === input.proposal.name)) { + throw new Error(`Control contract must not contain the proposed skill`) + } + const strip = (contract: HarnessContract.Info) => ({ + objective: contract.objective, + benchmark: contract.benchmark, + profile: contract.profile, + orchestration: contract.orchestration, + search: contract.search, + audit: contract.audit, + integrity: contract.integrity, + evolution: contract.evolution, + metaHarness: contract.metaHarness, + interventions: contract.interventions, + simulation: contract.simulation, + evaluatorAudit: contract.evaluatorAudit, + semanticAudit: contract.semanticAudit, + synthesis: contract.synthesis, + autonomy: contract.autonomy, + formalProof: contract.formalProof, + replication: contract.replication, + confirmation: contract.confirmation, + packs: contract.packs ?? [], + model: contract.model, + tools: contract.tools.toSorted(), + skills: contract.skills + .filter((item) => item.name !== input.proposal.name) + .toSorted((a, b) => a.name.localeCompare(b.name)), + budget: contract.budget, + seed: contract.seed, + intervention: contract.intervention, + contamination: contract.contamination, + }) + if (JSON.stringify(strip(input.candidate)) !== JSON.stringify(strip(input.control))) { + throw new Error(`Skill candidate and control contracts differ outside the proposed skill`) + } + if (input.candidate.sessionID === input.control.sessionID || input.candidate.runID === input.control.runID) { + throw new Error(`Skill candidate and control must be separate runs`) + } + } + + export async function attest(input: Attestation) { + const value = Attestation.parse(input) + const proposal = await read(value.name) + if (!proposal) throw new Error(`Unknown learned skill proposal ${value.name}`) + if (proposal.status === "rejected") throw new Error(`Rejected skill proposals cannot receive evidence`) + if (proposal.status === "promoted") throw new Error(`Promoted skills require a new versioned proposal`) + const [candidate, control] = await Promise.all([ + HarnessAdapter.authorize(value.candidate.sessionID, value.candidate.evaluatorToken), + HarnessAdapter.authorize(value.control.sessionID, value.control.evaluatorToken), + ]) + comparable({ proposal, candidate, control }) + if (!(["held_out", "release"] as string[]).includes(candidate.benchmark.split)) { + throw new Error(`Skill qualification requires a held-out or release benchmark split`) + } + const [candidateEvaluation, controlEvaluation] = await Promise.all( + [candidate.sessionID, control.sessionID].map(async (sessionID) => + (await HarnessEvaluation.list(sessionID)).findLast(HarnessEvaluation.final), + ), + ) + if (!candidateEvaluation || !controlEvaluation) throw new Error(`Both skill runs require external evaluations`) + const direction = candidate.benchmark.direction ?? "pass" + const delta = (() => { + if (candidateEvaluation.score === undefined || controlEvaluation.score === undefined) return undefined + if (direction === "maximize") return candidateEvaluation.score - controlEvaluation.score + if (direction === "minimize") return controlEvaluation.score - candidateEvaluation.score + return 0 + })() + const nonregressing = + candidateEvaluation.status === "passed" && + controlEvaluation.status === "passed" && + (direction === "pass" || (delta !== undefined && delta >= 0)) + const improved = nonregressing && direction !== "pass" && delta !== undefined && delta > 0 + const trigger = Trigger.parse(value.trigger) + const precision = trigger.truePositive / (trigger.truePositive + trigger.falsePositive) + const recall = trigger.truePositive / (trigger.truePositive + trigger.falseNegative) + const evidence = Evidence.parse({ + id: digest(`${proposal.contentSHA256}\0${candidate.runID}\0${control.runID}`), + proposalSHA256: proposal.contentSHA256, + benchmark: { + name: candidate.benchmark.name, + version: candidate.benchmark.version, + taskID: candidate.benchmark.taskID, + split: candidate.benchmark.split, + metric: candidate.benchmark.metric, + direction, + }, + candidate: { + sessionID: candidate.sessionID, + runID: candidate.runID, + status: candidateEvaluation.status, + score: candidateEvaluation.score, + evaluationSHA256: HarnessEvaluation.fingerprint(candidateEvaluation), + }, + control: { + sessionID: control.sessionID, + runID: control.runID, + status: controlEvaluation.status, + score: controlEvaluation.score, + evaluationSHA256: HarnessEvaluation.fingerprint(controlEvaluation), + }, + nonregressing, + improved, + trigger: { ...trigger, precision, recall }, + evaluator: { name: candidateEvaluation.evaluator.name, version: candidateEvaluation.evaluator.version }, + recordedAt: value.recordedAt ?? Date.now(), + }) + await JsonStore.update(manifest(value.name), (data) => { + const current = Manifest.parse(data) + if (current.contentSHA256 !== evidence.proposalSHA256) throw new Error(`Skill proposal SHA changed`) + const existing = current.evidence.find((item) => item.id === evidence.id) + if ( + existing && + JSON.stringify({ ...existing, recordedAt: 0 }) !== JSON.stringify({ ...evidence, recordedAt: 0 }) + ) { + throw new Error(`Skill evidence for this candidate/control pair is immutable`) + } + const items = existing ? current.evidence : [...current.evidence, evidence] + const status = assess(items).qualified ? "qualified" : "pending" + return Manifest.parse({ ...current, evidence: items, status, updatedAt: Date.now() }) + }) + const updated = await read(value.name) + if (!updated) throw new Error(`Skill proposal ${value.name} disappeared after attestation`) + return { manifest: updated, assessment: assess(updated.evidence) } + } + + export async function promote(name: string) { + const value = Name.parse(name) + const proposal = await read(value) + if (!proposal) throw new Error(`Unknown learned skill proposal ${value}`) + if (proposal.status !== "qualified" && proposal.status !== "promoted") { + throw new Error(`Skill proposal ${value} has not met held-out qualification criteria`) + } + const content = await Bun.file(skill(value)).text() + if (digest(content) !== proposal.contentSHA256) throw new Error(`Skill proposal content hash changed`) + const destination = path.join(active, value, "SKILL.md") + const existing = await Bun.file(destination) + .text() + .catch(() => null) + if (existing !== null && digest(existing) !== proposal.contentSHA256) { + throw new Error(`An active skill named ${value} already exists with different content`) + } + await fs.mkdir(path.dirname(destination), { recursive: true }) + if (existing === null) await Bun.write(destination, content, { mode: 0o600 }) + await JsonStore.update(manifest(value), (data) => + Manifest.parse({ ...Manifest.parse(data), status: "promoted", promotedAt: Date.now(), updatedAt: Date.now() }), + ) + return { manifest: await read(value), path: destination } + } +} diff --git a/backend/cli/src/session/harness/synthesis.ts b/backend/cli/src/session/harness/synthesis.ts new file mode 100644 index 00000000..31ff12c7 --- /dev/null +++ b/backend/cli/src/session/harness/synthesis.ts @@ -0,0 +1,532 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" +import { HarnessJudge } from "./judge" + +export namespace HarnessSynthesis { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Token = z.string().min(32).max(1_024) + const ID = z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$/) + const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + const same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right) + + export const Subject = z + .object({ + type: z.enum(["run", "candidate"]), + id: z.string().min(1).max(240), + }) + .strict() + export type Subject = z.infer + + export const Access = z + .object({ + sessionID: z.string().min(1).max(240), + evaluatorToken: Token, + }) + .strict() + export type Access = z.infer + + const Fact = z + .object({ + id: ID, + commitment: Hash, + }) + .strict() + + export const GeneratedFact = Fact.extend({ + verdict: z.enum(["supported", "contradicted", "unsupported", "judge_error"]), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }).strict() + export type GeneratedFact = z.infer + + export const ReferenceFact = Fact.extend({ + coverage: z.enum(["covered", "missed", "judge_error"]), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }).strict() + export type ReferenceFact = z.infer + + export const Violation = z.enum([ + "forbidden_domain", + "reference_title", + "post_cutoff", + "unknown_date", + "duplicate_output", + ]) + export type Violation = z.infer + + const ToolInput = z + .object({ + sequence: z.number().int().positive(), + tool: HarnessContract.SynthesisTool, + requestSHA256: Hash, + responseSHA256: Hash, + sourceSHA256: Hash, + publishedAt: z.iso.date().optional(), + matches: z + .object({ + forbiddenDomain: z.boolean(), + referenceTitle: z.boolean(), + }) + .strict(), + decision: z.enum(["allowed", "blocked"]), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + + export const ToolEvent = ToolInput.extend({ + eventID: Hash, + violations: z.array(Violation).max(Violation.options.length), + }).strict() + export type ToolEvent = z.infer + + const Decomposition = z + .object({ + status: z.enum(["passed", "failed"]), + outputSHA256: Hash.optional(), + evidence: z.array(z.string().min(1).max(1_000)).min(1).max(32), + }) + .strict() + .superRefine((value, ctx) => { + if (value.status === "passed" && !value.outputSHA256) { + ctx.addIssue({ code: "custom", path: ["outputSHA256"], message: "A passing decomposition needs an output" }) + } + if (value.status === "failed" && value.outputSHA256) { + ctx.addIssue({ code: "custom", path: ["outputSHA256"], message: "A failed decomposition has no output" }) + } + }) + + export const Submit = Access.extend({ + subject: Subject, + conclusionSHA256: Hash, + evaluatorAuditReceiptID: Hash, + trace: z + .object({ + owner: z.literal("evaluator_runtime"), + complete: z.literal(true), + schemaSHA256: Hash, + filterPolicySHA256: Hash, + events: z.array(ToolInput).max(10_000), + }) + .strict(), + decomposition: Decomposition, + generatedFacts: z.array(GeneratedFact).max(512), + referenceFacts: z.array(ReferenceFact).min(1).max(2_048), + evaluatedAt: z.number().int().positive(), + }).strict() + export type Submit = z.infer + + const Violations = z.record(Violation, z.number().int().nonnegative()) + + export const Metrics = z + .object({ + toolEvents: z.number().int().nonnegative(), + allowedSources: z.number().int().nonnegative(), + blockedSources: z.number().int().nonnegative(), + violations: Violations, + generatedFacts: z.number().int().nonnegative(), + supported: z.number().int().nonnegative(), + contradicted: z.number().int().nonnegative(), + unsupported: z.number().int().nonnegative(), + precisionJudgeErrors: z.number().int().nonnegative(), + referenceFacts: z.number().int().positive(), + covered: z.number().int().nonnegative(), + missed: z.number().int().nonnegative(), + recallJudgeErrors: z.number().int().nonnegative(), + precision: z.number().min(0).max(1).optional(), + recall: z.number().min(0).max(1).optional(), + f1: z.number().min(0).max(1).optional(), + }) + .strict() + export type Metrics = z.infer + + export const Receipt = z + .object({ + schemaVersion: z.literal(1), + protocolVersion: z.literal("scientific-synthesis-receipt-v1"), + receiptID: Hash, + runID: z.string().min(1).max(240), + sessionID: z.string().min(1).max(240), + contractFingerprint: Hash, + protocolSHA256: Hash, + subject: Subject, + conclusionSHA256: Hash, + evaluatorAuditReceiptID: Hash, + traceSHA256: Hash, + events: z.array(ToolEvent).max(10_000), + decomposition: Decomposition, + generatedFacts: z.array(GeneratedFact).max(512), + referenceFacts: z.array(ReferenceFact).min(1).max(2_048), + metrics: Metrics, + status: z.enum(["passed", "failed", "inconclusive"]), + failures: z.array(z.string().min(1).max(500)).max(64), + evaluatedAt: z.number().int().positive(), + recordedAt: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + const stable = structuredClone(value) as Record + delete stable.receiptID + delete stable.recordedAt + if (digest(stable) === value.receiptID) return + ctx.addIssue({ code: "custom", path: ["receiptID"], message: "Scientific synthesis receipt hash is invalid" }) + }) + export type Receipt = z.infer + + export function prompt(contract: HarnessContract.Info) { + const protocol = contract.synthesis + if (!protocol) return "" + return [ + "", + "Synthesize the conclusion from admissible evidence; the hidden reference and atomic answer facts remain evaluator-private.", + `Use only the declared retrieval tools (${protocol.tools.join(", ")}) and at most ${protocol.maxToolEvents} tool events.`, + `Treat ${protocol.cutoff} as the inclusive publication cutoff. Do not use undated or post-cutoff sources, forbidden reference domains, direct reference-title matches, or repeated cached outputs.`, + `Produce at least ${protocol.minGeneratedFacts} atomic facts. The external evaluator will derive factual precision, reference-fact recall, contradiction-adjusted F1, and require precision>=${protocol.minPrecision}, recall>=${protocol.minRecall}, F1>=${protocol.minF1}.`, + "Do not retrieve, infer, or ask for the hidden reference, fact manifest, salt, judge prompts, blocked outputs, or evaluator capabilities. Judge failures are inconclusive, not evidence against the answer.", + "", + ].join("\n") + } + + export async function context(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + return contract ? prompt(contract) : "" + } + + const root = path.join(Global.Path.data, "harness", "syntheses") + const receiptFile = (receiptID: string) => path.join(root, "receipts", `${receiptID}.json`) + const subjectFile = (sessionID: string, subject: Subject) => + path.join( + root, + "subjects", + encodeURIComponent(sessionID), + `${encodeURIComponent(`${subject.type}:${subject.id}`)}.json`, + ) + + export function referenceManifest(input: Array>) { + return digest(input.map((item) => ({ id: item.id, commitment: item.commitment }))) + } + + const ordered = (items: Array<{ id: string }>) => + items.every((item, index) => !index || items[index - 1]!.id < item.id) + + const unique = (items: Array<{ id: string; commitment: string }>) => + new Set(items.map((item) => item.id)).size === items.length && + new Set(items.map((item) => item.commitment)).size === items.length + + async function born(contract: HarnessContract.Info, subject: Subject) { + if (subject.type === "run") { + if (subject.id !== contract.runID) { + throw new Error(`Scientific synthesis run subject does not match its bound contract`) + } + return contract.createdAt + } + const state = await import("./search") + .then((module) => module.HarnessSearch.read(contract.sessionID)) + .catch(() => null) + const candidate = state?.runID === contract.runID ? state.candidates[subject.id] : undefined + if (!candidate) throw new Error(`Scientific synthesis candidate does not exist in the bound search`) + return candidate.createdAt + } + + function events(input: z.infer[], protocol: HarnessContract.ScientificSynthesis) { + const prior = new Set() + return input.map((item, index) => { + if (item.sequence !== index + 1) throw new Error(`Scientific synthesis tool trace must be contiguous from one`) + if (!protocol.tools.includes(item.tool)) { + throw new Error(`Scientific synthesis used undeclared tool ${item.tool}`) + } + const violations = Violation.options.filter((violation) => { + if (violation === "forbidden_domain") return item.matches.forbiddenDomain + if (violation === "reference_title") return item.matches.referenceTitle + if (violation === "post_cutoff") return Boolean(item.publishedAt && item.publishedAt > protocol.cutoff) + if (violation === "unknown_date") return !item.publishedAt + return prior.has(item.responseSHA256) + }) + const decision = violations.length ? "blocked" : "allowed" + if (item.decision !== decision) { + throw new Error( + `Scientific synthesis tool event ${item.sequence} changed its backend-derived clean-room decision`, + ) + } + prior.add(item.responseSHA256) + const stable = { ...item, violations } + return ToolEvent.parse({ ...stable, eventID: digest(stable) }) + }) + } + + function assess(input: { + protocol: HarnessContract.ScientificSynthesis + trace: ToolEvent[] + decomposition: z.infer + generated: GeneratedFact[] + reference: ReferenceFact[] + }) { + const supported = input.generated.filter((item) => item.verdict === "supported").length + const contradicted = input.generated.filter((item) => item.verdict === "contradicted").length + const unsupported = input.generated.filter((item) => item.verdict === "unsupported").length + const precisionJudgeErrors = input.generated.filter((item) => item.verdict === "judge_error").length + const covered = input.reference.filter((item) => item.coverage === "covered").length + const missed = input.reference.filter((item) => item.coverage === "missed").length + const recallJudgeErrors = input.reference.filter((item) => item.coverage === "judge_error").length + const validPrecision = + input.decomposition.status === "passed" && !precisionJudgeErrors && input.generated.length > 0 + const validRecall = !recallJudgeErrors + const precision = validPrecision + ? (supported / input.generated.length) * (1 - contradicted / input.generated.length) + : undefined + const recall = validRecall ? covered / input.reference.length : undefined + const f1 = + precision === undefined || recall === undefined + ? undefined + : precision + recall + ? (2 * precision * recall) / (precision + recall) + : 0 + const violations = Object.fromEntries( + Violation.options.map((violation) => [ + violation, + input.trace.filter((item) => item.violations.includes(violation)).length, + ]), + ) as Record + const metrics = Metrics.parse({ + toolEvents: input.trace.length, + allowedSources: input.trace.filter((item) => item.decision === "allowed").length, + blockedSources: input.trace.filter((item) => item.decision === "blocked").length, + violations, + generatedFacts: input.generated.length, + supported, + contradicted, + unsupported, + precisionJudgeErrors, + referenceFacts: input.reference.length, + covered, + missed, + recallJudgeErrors, + precision, + recall, + f1, + }) + const inconclusive = [ + ...(input.decomposition.status === "failed" ? ["atomic fact decomposition failed"] : []), + ...(precisionJudgeErrors ? [`${precisionJudgeErrors} precision judgments failed`] : []), + ...(recallJudgeErrors ? [`${recallJudgeErrors} recall judgments failed`] : []), + ] + if (inconclusive.length) return { metrics, status: "inconclusive" as const, failures: inconclusive } + const failures = [ + ...(precision! < input.protocol.minPrecision + ? [`factual precision ${precision} is below ${input.protocol.minPrecision}`] + : []), + ...(recall! < input.protocol.minRecall ? [`factual recall ${recall} is below ${input.protocol.minRecall}`] : []), + ...(f1! < input.protocol.minF1 ? [`factual F1 ${f1} is below ${input.protocol.minF1}`] : []), + ] + return { metrics, status: failures.length ? ("failed" as const) : ("passed" as const), failures } + } + + function verify(receipt: Receipt, protocol: HarnessContract.ScientificSynthesis) { + if ( + !ordered(receipt.referenceFacts) || + !unique(receipt.referenceFacts) || + receipt.referenceFacts.length !== protocol.referenceFactCount || + referenceManifest(receipt.referenceFacts) !== protocol.referenceFactsSHA256 + ) { + throw new Error(`Scientific synthesis receipt changed its frozen reference facts`) + } + if (!ordered(receipt.generatedFacts) || !unique(receipt.generatedFacts)) { + throw new Error(`Scientific synthesis receipt changed its generated fact manifest`) + } + if ( + (receipt.decomposition.status === "passed" && receipt.generatedFacts.length < protocol.minGeneratedFacts) || + (receipt.decomposition.status === "failed" && receipt.generatedFacts.length > 0) + ) { + throw new Error(`Scientific synthesis receipt changed its decomposition outcome`) + } + const trace = events( + receipt.events.map((item) => + ToolInput.parse( + Object.fromEntries(Object.entries(item).filter(([key]) => key !== "eventID" && key !== "violations")), + ), + ), + protocol, + ) + if (!same(trace, receipt.events) || digest(trace) !== receipt.traceSHA256) { + throw new Error(`Scientific synthesis receipt does not match its backend-replayed retrieval trace`) + } + const result = assess({ + protocol, + trace, + decomposition: receipt.decomposition, + generated: receipt.generatedFacts, + reference: receipt.referenceFacts, + }) + if ( + !same(result.metrics, receipt.metrics) || + result.status !== receipt.status || + !same(result.failures, receipt.failures) + ) { + throw new Error(`Scientific synthesis receipt does not match backend-derived factuality metrics`) + } + } + + export async function record(input: Submit, contract: HarnessContract.Info) { + const value = Submit.parse(input) + if (value.sessionID !== contract.sessionID) { + throw new Error(`Scientific synthesis session does not match its bound harness contract`) + } + const protocol = contract.synthesis + if (!protocol) throw new Error(`Harness contract does not require scientific synthesis validation`) + if (value.trace.schemaSHA256 !== protocol.traceSchemaSHA256) { + throw new Error(`Scientific synthesis trace schema does not match the bound protocol`) + } + if (value.trace.filterPolicySHA256 !== protocol.filterPolicySHA256) { + throw new Error(`Scientific synthesis filter policy does not match the bound protocol`) + } + if (value.trace.events.length > protocol.maxToolEvents) { + throw new Error(`Scientific synthesis tool trace exceeds its frozen event budget`) + } + if (!ordered(value.referenceFacts) || !unique(value.referenceFacts)) { + throw new Error(`Scientific synthesis reference facts must be unique and sorted by ID`) + } + if (value.referenceFacts.length !== protocol.referenceFactCount) { + throw new Error(`Scientific synthesis reference fact count does not match the bound protocol`) + } + if (referenceManifest(value.referenceFacts) !== protocol.referenceFactsSHA256) { + throw new Error(`Scientific synthesis reference facts do not match the hidden manifest commitment`) + } + if (!ordered(value.generatedFacts) || !unique(value.generatedFacts)) { + throw new Error(`Scientific synthesis generated facts must be unique and sorted by ID`) + } + if (value.decomposition.status === "passed" && value.generatedFacts.length < protocol.minGeneratedFacts) { + throw new Error(`Scientific synthesis decomposition produced too few atomic facts`) + } + if (value.decomposition.status === "failed" && value.generatedFacts.length) { + throw new Error(`A failed scientific synthesis decomposition cannot contain generated facts`) + } + const recordedAt = Date.now() + const createdAt = await born(contract, value.subject) + if (value.evaluatedAt < createdAt || value.evaluatedAt > recordedAt) { + throw new Error(`Scientific synthesis evaluation falls outside its bound subject interval`) + } + await HarnessJudge.assert({ + contract, + receiptID: value.evaluatorAuditReceiptID, + recordedAt, + requirePassed: true, + }) + const trace = events(value.trace.events, protocol) + const result = assess({ + protocol, + trace, + decomposition: value.decomposition, + generated: value.generatedFacts, + reference: value.referenceFacts, + }) + const stable = { + schemaVersion: 1 as const, + protocolVersion: "scientific-synthesis-receipt-v1" as const, + runID: contract.runID, + sessionID: contract.sessionID, + contractFingerprint: HarnessContract.fingerprint(contract), + protocolSHA256: digest(protocol), + subject: value.subject, + conclusionSHA256: value.conclusionSHA256, + evaluatorAuditReceiptID: value.evaluatorAuditReceiptID, + traceSHA256: digest(trace), + events: trace, + decomposition: value.decomposition, + generatedFacts: value.generatedFacts, + referenceFacts: value.referenceFacts, + metrics: result.metrics, + status: result.status, + failures: result.failures, + evaluatedAt: value.evaluatedAt, + } + const receipt = Receipt.parse({ ...stable, receiptID: digest(stable), recordedAt }) + const claimed = await JsonStore.read(subjectFile(receipt.sessionID, receipt.subject)) + if (Object.keys(claimed).length) { + const current = Receipt.parse(claimed) + if (current.receiptID !== receipt.receiptID) { + throw new Error(`Scientific synthesis subject already has a canonical receipt`) + } + } + await JsonStore.update(receiptFile(receipt.receiptID), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Scientific synthesis receipt is immutable once recorded`) + }) + await JsonStore.update(subjectFile(receipt.sessionID, receipt.subject), (data) => { + if (!Object.keys(data).length) return receipt + const current = Receipt.parse(data) + if (current.receiptID === receipt.receiptID) return current + throw new Error(`Scientific synthesis subject already has a canonical receipt`) + }) + const saved = await readReceipt(receipt.receiptID) + if (!saved) throw new Error(`Scientific synthesis receipt was not durable after recording`) + return saved + } + + export async function readReceipt(receiptID: string) { + const id = Hash.parse(receiptID) + const parsed = Receipt.safeParse(await JsonStore.read(receiptFile(id))) + if (!parsed.success || parsed.data.receiptID !== id) return null + const canonical = Receipt.safeParse(await JsonStore.read(subjectFile(parsed.data.sessionID, parsed.data.subject))) + if (!canonical.success || canonical.data.receiptID !== id || !same(canonical.data, parsed.data)) return null + return parsed.data + } + + export async function read(receiptID: string, contract: HarnessContract.Info) { + const receipt = await readReceipt(receiptID) + if (!receipt || receipt.sessionID !== contract.sessionID) { + throw new Error(`Unknown scientific synthesis receipt ${receiptID}`) + } + const protocol = contract.synthesis + if (!protocol || receipt.contractFingerprint !== HarnessContract.fingerprint(contract)) { + throw new Error(`Scientific synthesis receipt belongs to a different harness run`) + } + verify(receipt, protocol) + return receipt + } + + export async function assert(input: { + contract: HarnessContract.Info + receiptID: string + subject: Subject + score?: number + evaluatedAt: number + recordedAt: number + requirePassed: boolean + }) { + const receipt = await readReceipt(input.receiptID) + if (!receipt) throw new Error(`Unknown or corrupt scientific synthesis receipt ${input.receiptID}`) + const protocol = input.contract.synthesis + if (!protocol) throw new Error(`Evaluation cites a synthesis receipt without a bound protocol`) + if ( + receipt.contractFingerprint !== HarnessContract.fingerprint(input.contract) || + receipt.protocolSHA256 !== digest(protocol) || + receipt.sessionID !== input.contract.sessionID || + receipt.runID !== input.contract.runID + ) { + throw new Error(`Scientific synthesis receipt belongs to a different harness run`) + } + verify(receipt, protocol) + if (receipt.subject.type !== input.subject.type || receipt.subject.id !== input.subject.id) { + throw new Error(`Scientific synthesis receipt belongs to a different evaluation subject`) + } + if (receipt.evaluatedAt > input.evaluatedAt || receipt.recordedAt > input.recordedAt) { + throw new Error(`Evaluation predates its scientific synthesis receipt`) + } + if (input.requirePassed && receipt.status !== "passed") { + throw new Error(`A passing final evaluation requires a passing scientific synthesis receipt`) + } + if (input.requirePassed && (input.score === undefined || input.score !== receipt.metrics.f1)) { + throw new Error(`A passing scientific synthesis score must equal the backend-derived factual F1`) + } + await HarnessJudge.assert({ + contract: input.contract, + receiptID: receipt.evaluatorAuditReceiptID, + recordedAt: input.recordedAt, + requirePassed: input.requirePassed, + }) + return receipt + } +} diff --git a/backend/cli/src/session/harness/world.ts b/backend/cli/src/session/harness/world.ts new file mode 100644 index 00000000..772add0c --- /dev/null +++ b/backend/cli/src/session/harness/world.ts @@ -0,0 +1,521 @@ +import path from "path" +import z from "zod" +import { Global } from "@/global" +import { JsonStore } from "@/util/jsonstore" +import { HarnessContract } from "./contract" + +export namespace HarnessWorld { + const Hash = z.string().regex(/^[a-f0-9]{64}$/) + const Key = z.string().regex(/^[a-z0-9][a-z0-9._:-]{0,79}$/) + + export const Kind = z.enum(["hypothesis", "observation", "strategy", "memory", "skill", "subagent"]) + export type Kind = z.infer + + export const Authority = z.enum(["self", "tool", "evaluator", "human"]) + export type Authority = z.infer + + export const Evidence = z + .object({ + ref: z.string().min(1).max(1_000), + authority: Authority, + }) + .strict() + export type Evidence = z.infer + + export const Entry = z + .object({ + id: Hash, + key: Key, + kind: Kind, + content: z.string().min(1).max(4_000), + confidence: z.number().int().min(1).max(5), + evidence: z + .array(Evidence) + .max(16) + .refine( + (items) => new Set(items.map((item) => `${item.authority}:\0${item.ref}`)).size === items.length, + "World-model evidence references must be unique", + ), + updatedAt: z.number().int().positive(), + revision: z.number().int().positive(), + }) + .strict() + .superRefine((value, ctx) => { + const independent = value.evidence.filter((item) => item.authority !== "self") + if (value.confidence >= 4 && !independent.length) { + ctx.addIssue({ + code: "custom", + path: ["confidence"], + message: "Confidence 4 or 5 requires evidence beyond self-report", + }) + } + if ( + value.confidence === 5 && + (independent.length < 2 || !independent.some((item) => ["evaluator", "human"].includes(item.authority))) + ) { + ctx.addIssue({ + code: "custom", + path: ["confidence"], + message: "Confidence 5 requires two non-self references including evaluator or human evidence", + }) + } + }) + export type Entry = z.infer + + export const EventType = z.enum(["analysis", "tool", "evaluation", "failure", "milestone", "stagnation", "manual"]) + export type EventType = z.infer + + export const Event = z + .object({ + id: Hash, + type: EventType, + summary: z.string().min(1).max(1_000), + evidenceRefs: z.array(z.string().min(1).max(1_000)).max(16), + changed: z.boolean(), + createdAt: z.number().int().positive(), + }) + .strict() + export type Event = z.infer + + export const Reason = z.enum(["manual", "failure", "stagnation", "milestone", "periodic"]) + export type Reason = z.infer + + export const Patch = z.discriminatedUnion("op", [ + z + .object({ + op: z.literal("upsert"), + key: Key, + kind: Kind, + content: z.string().min(1).max(4_000), + confidence: z.number().int().min(1).max(5), + evidence: z.array(Evidence).max(16).default([]), + }) + .strict(), + z + .object({ + op: z.literal("remove"), + key: Key, + }) + .strict(), + ]) + export type Patch = z.infer + + export const AgentPatch = z.discriminatedUnion("op", [ + z + .object({ + op: z.literal("upsert"), + key: Key, + kind: Kind, + content: z.string().min(1).max(4_000), + confidence: z.number().int().min(1).max(3), + evidenceRefs: z.array(z.string().min(1).max(1_000)).max(16).default([]), + }) + .strict(), + z + .object({ + op: z.literal("remove"), + key: Key, + }) + .strict(), + ]) + export type AgentPatch = z.infer + + export const EvaluatorPatch = z.discriminatedUnion("op", [ + z + .object({ + op: z.literal("upsert"), + key: Key, + kind: Kind, + content: z.string().min(1).max(4_000), + confidence: z.number().int().min(1).max(5), + evidenceRefs: z.array(z.string().min(1).max(1_000)).max(16).default([]), + }) + .strict(), + z + .object({ + op: z.literal("remove"), + key: Key, + }) + .strict(), + ]) + + export const EvaluatorRefine = z + .object({ + evaluatorToken: z.string().min(32).max(1_024), + expectedRevision: z.number().int().nonnegative(), + reason: Reason, + patches: z.array(EvaluatorPatch).min(1).max(6), + }) + .strict() + export type EvaluatorRefine = z.infer + + const Snapshot = z + .object({ + revision: z.number().int().nonnegative(), + entries: z.record(z.string(), Entry), + sha256: Hash, + createdAt: z.number().int().positive(), + }) + .strict() + + export const State = z + .object({ + schemaVersion: z.literal(1), + sessionID: z.string().min(1), + runID: z.string().min(1), + basePromptSHA256: Hash, + entries: z.record(z.string(), Entry), + events: z.array(Event).max(128), + snapshots: z.array(Snapshot).max(10), + revision: z.number().int().nonnegative(), + contextEpoch: z.number().int().nonnegative(), + eventsSinceRefine: z.number().int().nonnegative(), + refinement: z + .object({ + recommended: z.boolean(), + trigger: Reason.optional(), + }) + .strict(), + createdAt: z.number().int().positive(), + updatedAt: z.number().int().positive(), + }) + .strict() + export type State = z.infer + + const root = path.join(Global.Path.data, "harness", "worlds") + const file = (sessionID: string) => path.join(root, `${encodeURIComponent(sessionID)}.json`) + const digest = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + const canonical = (value: unknown) => JSON.stringify(value) + const clip = (value: string, max = 1_000) => + value + .replace(/[\u0000-\u001f\u007f\u200b-\u200f\u202a-\u202e\u2060-\u2064\ufeff]/g, " ") + .trim() + .slice(0, max) + const escape = (value: string) => + value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """) + const promptHash = (contract: HarnessContract.Info) => + digest( + canonical({ + contract: HarnessContract.fingerprint(contract), + objective: contract.objective, + task: contract.benchmark.task, + }), + ) + + function empty(contract: HarnessContract.Info): State { + const now = Date.now() + return { + schemaVersion: 1, + sessionID: contract.sessionID, + runID: contract.runID, + basePromptSHA256: promptHash(contract), + entries: {}, + events: [], + snapshots: [], + revision: 0, + contextEpoch: 0, + eventsSinceRefine: 0, + refinement: { recommended: false }, + createdAt: now, + updatedAt: now, + } + } + + function verify(state: State, contract: HarnessContract.Info) { + if (state.runID !== contract.runID || state.basePromptSHA256 !== promptHash(contract)) { + throw new Error("Continual world model does not match the immutable harness contract") + } + return state + } + + export async function read(sessionID: string) { + const contract = await HarnessContract.read(sessionID) + if (!contract) throw new Error(`No harness contract is bound to session ${sessionID}`) + const data = await JsonStore.read(file(sessionID)) + const parsed = State.safeParse(data) + if (parsed.success) return verify(parsed.data, contract) + const state = empty(contract) + await JsonStore.update(file(sessionID), (current) => (Object.keys(current).length ? State.parse(current) : state)) + return state + } + + export function summary(state: State) { + return { + runID: state.runID, + basePromptSHA256: state.basePromptSHA256, + revision: state.revision, + contextEpoch: state.contextEpoch, + eventsSinceRefine: state.eventsSinceRefine, + refinement: state.refinement, + entries: Object.values(state.entries) + .toSorted((a, b) => b.confidence - a.confidence || b.updatedAt - a.updatedAt || a.key.localeCompare(b.key)) + .map((entry) => ({ + key: entry.key, + kind: entry.kind, + content: entry.content, + confidence: entry.confidence, + evidence: entry.evidence, + revision: entry.revision, + })), + recentEvents: state.events.slice(-12), + rollbackRevisions: state.snapshots.map((item) => item.revision).toReversed(), + } + } + + export async function event(input: { + sessionID: string + type: EventType + summary: string + evidenceRefs?: string[] + changed?: boolean + }) { + const current = await read(input.sessionID) + const changed = input.changed ?? input.type !== "analysis" + if (input.type === "analysis" && changed) throw new Error("Analysis events cannot advance the context epoch") + const createdAt = Date.now() + const item = Event.parse({ + id: digest( + canonical({ + sessionID: input.sessionID, + type: input.type, + summary: input.summary, + evidenceRefs: input.evidenceRefs ?? [], + createdAt, + revision: current.revision, + }), + ), + type: input.type, + summary: clip(input.summary), + evidenceRefs: input.evidenceRefs ?? [], + changed, + createdAt, + }) + const urgent = ["failure", "stagnation", "milestone", "manual"].includes(item.type) + const count = current.eventsSinceRefine + 1 + const periodic = count >= 6 + const trigger = + current.refinement.trigger ?? (urgent ? Reason.parse(item.type) : periodic ? ("periodic" as const) : undefined) + const next = State.parse({ + ...current, + events: [...current.events, item].slice(-128), + revision: current.revision + 1, + contextEpoch: current.contextEpoch + (changed ? 1 : 0), + eventsSinceRefine: count, + refinement: { recommended: Boolean(trigger), trigger }, + updatedAt: createdAt, + }) + await JsonStore.update(file(input.sessionID), (data) => { + const state = State.parse(data) + if (state.revision !== current.revision) { + throw new Error("Continual world model changed while recording an event") + } + return next + }) + return next + } + + export async function refine(input: { + sessionID: string + expectedRevision: number + reason: Reason + patches: Patch[] + actor: "agent" | "evaluator" + }) { + const patches = z.array(Patch).min(1).max(6).parse(input.patches) + const content = patches.reduce((total, patch) => total + (patch.op === "upsert" ? patch.content.length : 0), 0) + if (content > 12_000) throw new Error("A refinement may add at most 12,000 characters") + if ( + input.actor === "agent" && + patches.some((patch) => patch.op === "upsert" && patch.evidence.some((item) => item.authority !== "self")) + ) { + throw new Error("Agent refinements may only record self-attributed evidence") + } + const current = await read(input.sessionID) + if (current.revision !== input.expectedRevision) { + throw new Error(`Expected world-model revision ${input.expectedRevision}, found ${current.revision}`) + } + const revision = current.revision + 1 + const updatedAt = Date.now() + const entries = patches.reduce>((items, patch) => { + if (patch.op === "remove") { + if (!items[patch.key]) throw new Error(`Cannot remove unknown world-model entry ${patch.key}`) + return Object.fromEntries(Object.entries(items).filter(([key]) => key !== patch.key)) + } + const entry = Entry.parse({ + id: digest(`${input.sessionID}\0${patch.key}`), + key: patch.key, + kind: patch.kind, + content: clip(patch.content, 4_000), + confidence: patch.confidence, + evidence: patch.evidence, + updatedAt, + revision, + }) + return { ...items, [patch.key]: entry } + }, current.entries) + if (Object.keys(entries).length > 48) throw new Error("A continual world model may contain at most 48 entries") + const snapshot = Snapshot.parse({ + revision: current.revision, + entries: current.entries, + sha256: digest(canonical(current.entries)), + createdAt: updatedAt, + }) + const event = Event.parse({ + id: digest( + canonical({ + sessionID: input.sessionID, + reason: input.reason, + patches, + updatedAt, + revision, + }), + ), + type: input.reason === "periodic" ? "manual" : input.reason, + summary: `Applied ${input.reason} world-model refinement with ${patches.length} patch(es)`, + evidenceRefs: [ + ...new Set(patches.flatMap((patch) => (patch.op === "upsert" ? patch.evidence.map((item) => item.ref) : []))), + ].slice(0, 16), + changed: true, + createdAt: updatedAt, + }) + const next = State.parse({ + ...current, + entries, + events: [...current.events, event].slice(-128), + snapshots: [...current.snapshots, snapshot].slice(-10), + revision, + contextEpoch: current.contextEpoch + 1, + eventsSinceRefine: 0, + refinement: { recommended: false }, + updatedAt, + }) + await JsonStore.update(file(input.sessionID), (data) => { + const state = State.parse(data) + if (state.revision !== current.revision) { + throw new Error("Continual world model changed while applying a refinement") + } + return next + }) + return next + } + + export async function agentRefine(input: { + sessionID: string + expectedRevision: number + reason: Reason + patches: AgentPatch[] + }) { + const patches = z + .array(AgentPatch) + .min(1) + .max(6) + .parse(input.patches) + .map( + (patch): Patch => + patch.op === "remove" + ? patch + : { + op: "upsert", + key: patch.key, + kind: patch.kind, + content: patch.content, + confidence: patch.confidence, + evidence: patch.evidenceRefs.map((ref) => ({ ref, authority: "self" as const })), + }, + ) + return refine({ ...input, patches, actor: "agent" }) + } + + export async function evaluatorRefine(input: { + sessionID: string + expectedRevision: number + reason: Reason + patches: z.infer[] + }) { + const patches = z + .array(EvaluatorPatch) + .min(1) + .max(6) + .parse(input.patches) + .map( + (patch): Patch => + patch.op === "remove" + ? patch + : { + op: "upsert", + key: patch.key, + kind: patch.kind, + content: patch.content, + confidence: patch.confidence, + evidence: patch.evidenceRefs.map((ref) => ({ ref, authority: "evaluator" as const })), + }, + ) + return refine({ ...input, patches, actor: "evaluator" }) + } + + export async function rollback(input: { sessionID: string; expectedRevision: number; targetRevision?: number }) { + const current = await read(input.sessionID) + if (current.revision !== input.expectedRevision) { + throw new Error(`Expected world-model revision ${input.expectedRevision}, found ${current.revision}`) + } + const target = + input.targetRevision === undefined + ? current.snapshots.at(-1) + : current.snapshots.findLast((item) => item.revision === input.targetRevision) + if (!target) throw new Error("No matching world-model snapshot is available for rollback") + if (digest(canonical(target.entries)) !== target.sha256) throw new Error("World-model rollback snapshot is corrupt") + const updatedAt = Date.now() + const snapshot = Snapshot.parse({ + revision: current.revision, + entries: current.entries, + sha256: digest(canonical(current.entries)), + createdAt: updatedAt, + }) + const next = State.parse({ + ...current, + entries: target.entries, + snapshots: [...current.snapshots, snapshot].slice(-10), + revision: current.revision + 1, + contextEpoch: current.contextEpoch + 1, + eventsSinceRefine: 0, + refinement: { recommended: false }, + updatedAt, + }) + await JsonStore.update(file(input.sessionID), (data) => { + const state = State.parse(data) + if (state.revision !== current.revision) { + throw new Error("Continual world model changed while applying a rollback") + } + return next + }) + return next + } + + export async function prompt(sessionID: string) { + const state = await read(sessionID) + const entries = Object.values(state.entries).toSorted( + (a, b) => b.confidence - a.confidence || b.updatedAt - a.updatedAt || a.key.localeCompare(b.key), + ) + if (!entries.length) return "" + const lines = [ + ``, + "Session-local mutable working state. Confidence is provenance-gated; self-authored entries are tentative and must not override external evidence.", + ] + for (const entry of entries) { + const evidence = entry.evidence + .slice(0, 4) + .map((item) => `${item.authority}:${escape(item.ref).slice(0, 180)}`) + .join(", ") + const block = [ + ``, + escape(entry.content).slice(0, 1_000), + `Evidence: ${evidence || "self-report only"}`, + "", + ] + if ([...lines, ...block, ""].join("\n").length > 4_500) break + lines.push(...block) + } + lines.push("") + return lines.join("\n") + } +} diff --git a/backend/cli/src/session/llm.ts b/backend/cli/src/session/llm.ts index 62bdeac0..3e22753b 100644 --- a/backend/cli/src/session/llm.ts +++ b/backend/cli/src/session/llm.ts @@ -106,7 +106,7 @@ export namespace LLM { const variant = !input.small && input.model.variants && input.user.variant ? input.model.variants[input.user.variant] : {} - const base = input.small + const base: Record = input.small ? ProviderTransform.smallOptions(input.model) : ProviderTransform.options({ model: input.model, diff --git a/backend/cli/src/session/prompt.ts b/backend/cli/src/session/prompt.ts index 642352b7..91975efe 100644 --- a/backend/cli/src/session/prompt.ts +++ b/backend/cli/src/session/prompt.ts @@ -62,6 +62,17 @@ import { PlanMode } from "@/tool/plan-mode" import { Inference } from "@/provider/inference" import { OpenScience } from "@/openscience" import { assertExternalDirectory } from "@/tool/external-directory" +import { HarnessProfile } from "./harness/profile" +import { HarnessMemory } from "./harness/memory" +import { HarnessClaims } from "./harness/claims" +import { HarnessDomain } from "./harness/domain" +import { HarnessBlueprint } from "./harness/blueprint" +import { HarnessFormal } from "./harness/formal" +import { HarnessSemantic } from "./harness/semantic" +import { HarnessReplication } from "./harness/replication" +import { HarnessSynthesis } from "./harness/synthesis" +import { HarnessWorld } from "./harness/world" +import { SessionTraceStore } from "./trace-store" // @ts-ignore globalThis.AI_SDK_LOG_WARNINGS = false @@ -820,6 +831,35 @@ export namespace SessionPrompt { const sessionMessages = clone(msgs) + const request = + lastUserMsg?.parts + .flatMap((part) => (part.type === "text" && !part.synthetic && !part.ignored ? [part.text] : [])) + .join("\n") ?? "" + const profile = await HarnessProfile.resolve({ sessionID, agent: agent.name, text: request }) + const hindsight = + profile.id === "optimize" + ? await HarnessMemory.prompt({ sessionID, query: request, stage: "planning" }).catch(() => "") + : "" + const claims = profile.id === "react" ? "" : await HarnessClaims.prompt(sessionID).catch(() => "") + const domain = await HarnessDomain.resolve({ sessionID, agent: agent.name, profile: profile.id, text: request }) + const methodology = HarnessDomain.prompt(domain) + const [world, semantics, synthesis, replication, formal, blueprint] = await Promise.all([ + HarnessWorld.prompt(sessionID).catch(() => ""), + HarnessSemantic.context(sessionID).catch(() => ""), + HarnessSynthesis.context(sessionID).catch(() => ""), + HarnessReplication.context(sessionID).catch(() => ""), + HarnessFormal.context(sessionID).catch(() => ""), + HarnessBlueprint.context(sessionID).catch(() => ""), + ]) + await SessionTraceStore.recordProfile({ + sessionID, + messageID: lastUser.id, + id: profile.id, + source: profile.source, + confidence: profile.confidence, + reasons: profile.reasons, + }) + // Ephemerally wrap queued user messages with a reminder to stay on track if (step > 1 && lastFinished) { for (const msg of sessionMessages) { @@ -863,6 +903,16 @@ export namespace SessionPrompt { ...(await SystemPrompt.compute()), ...(await InstructionPrompt.system()), ...(SKILL_ROUTING_AGENTS.has(agent.name) ? [await SystemPrompt.availableSkills(agent.permission)] : []), + profile.prompt, + ...(world ? [world] : []), + ...(hindsight ? [hindsight] : []), + ...(claims ? [claims] : []), + ...(methodology ? [methodology] : []), + ...(semantics ? [semantics] : []), + ...(synthesis ? [synthesis] : []), + ...(replication ? [replication] : []), + ...(formal ? [formal] : []), + ...(blueprint ? [blueprint] : []), ...artifactContext, ] diff --git a/backend/cli/src/session/rsi/critic.ts b/backend/cli/src/session/rsi/critic.ts index 3ed7bc6b..57ee909b 100644 --- a/backend/cli/src/session/rsi/critic.ts +++ b/backend/cli/src/session/rsi/critic.ts @@ -31,7 +31,8 @@ export namespace RSICritic { ## Trajectory - Agent: ${trajectory.agent} - Hypothesis: ${trajectory.hypothesis} -- Outcome: ${trajectory.outcome} +- Agent-reported outcome: ${trajectory.reportedOutcome} +- Externally verified outcome: ${trajectory.outcome} - Steps: ${trajectory.steps.length} - Token cost: ~${trajectory.tokenCost} @@ -100,43 +101,27 @@ Respond with ONLY a JSON object: } } - /** Heuristic evaluate — deterministic scorer, no LLM call. - * Base: success=70, partial=45, failure=20. - * Modifiers: step efficiency (±10), tool diversity (±10), reproducibility (±10). */ + /** Deterministic process score. Correctness is zero until an external + * evaluator is attached; tool activity can never stand in for truth. */ export function evaluate(trajectory: RSITrajectory.Trajectory): CriticScore { - // Base score from outcome - const base = trajectory.outcome === "success" ? 70 : trajectory.outcome === "partial" ? 45 : 20 - - // Step efficiency: penalize >20 steps, reward <10 const stepCount = trajectory.steps.length - const efficiencyMod = stepCount <= 5 ? 10 : stepCount <= 10 ? 5 : stepCount <= 20 ? 0 : stepCount <= 40 ? -5 : -10 - - // Tool diversity: reward using multiple distinct tools const uniqueTools = new Set(trajectory.steps.map((s) => s.tool)).size - const diversityMod = uniqueTools >= 5 ? 10 : uniqueTools >= 3 ? 5 : uniqueTools >= 2 ? 0 : -5 - - // Reproducibility: reward having a hypothesis and moderate step count const hasHypothesis = trajectory.hypothesis.length > 20 const hasReasonableSteps = stepCount >= 3 && stepCount <= 30 - const reproducibilityMod = (hasHypothesis ? 5 : -5) + (hasReasonableSteps ? 5 : -5) - - const total = clamp(base + efficiencyMod + diversityMod + reproducibilityMod, 0, 100) - - // Distribute across dimensions (proportional to total) + const correctness = trajectory.outcome === "success" ? 25 : trajectory.outcome === "partial" ? 10 : 0 + const efficiency = stepCount <= 5 ? 25 : stepCount <= 10 ? 22 : stepCount <= 20 ? 18 : stepCount <= 40 ? 12 : 5 + const coverage = uniqueTools >= 5 ? 20 : uniqueTools >= 3 ? 16 : uniqueTools >= 2 ? 12 : uniqueTools === 1 ? 8 : 0 + const reproducibility = (hasHypothesis ? 12 : 4) + (hasReasonableSteps ? 13 : 4) const score: CriticScore = { - correctness: clamp( - Math.round(25 * (trajectory.outcome === "success" ? 1 : trajectory.outcome === "partial" ? 0.6 : 0.2)), - 0, - 25, - ), - efficiency: clamp(Math.round(25 * ((efficiencyMod + 10) / 20)), 0, 25), - coverage: clamp(Math.round(25 * ((diversityMod + 10) / 20)), 0, 25), - reproducibility: clamp(Math.round(25 * ((reproducibilityMod + 10) / 20)), 0, 25), - total, - notes: `Heuristic: outcome=${trajectory.outcome}, steps=${stepCount}, tools=${uniqueTools}`, + correctness, + efficiency, + coverage, + reproducibility, + total: correctness + efficiency + coverage + reproducibility, + notes: `Process heuristic: verification=${trajectory.outcome}, reported=${trajectory.reportedOutcome}, steps=${stepCount}, tools=${uniqueTools}`, } - log.info("heuristic evaluation", { sessionId: trajectory.sessionId, total, outcome: trajectory.outcome }) + log.info("process evaluation", { sessionId: trajectory.sessionId, total: score.total, outcome: trajectory.outcome }) return score } diff --git a/backend/cli/src/session/rsi/distill.ts b/backend/cli/src/session/rsi/distill.ts index b771a115..c43e6145 100644 --- a/backend/cli/src/session/rsi/distill.ts +++ b/backend/cli/src/session/rsi/distill.ts @@ -1,44 +1,51 @@ /** - * RSI Skill Distillation — Extracts learned skills from high-scoring trajectories. + * RSI Skill Proposals — Drafts inert skills from externally verified trajectories. * * When a trajectory scores >= 75/100 from the critic, this module: * 1. Extracts the decomposition pattern, tool sequence, and failure recovery * 2. Generates a SKILL.md in the standard format - * 3. Writes to ~/.openscience/learned-skills/{name}/SKILL.md + * 3. Writes to ~/.openscience/learned-skill-proposals/{name}/SKILL.md + * + * Proposals are deliberately outside learned-skills and are not uploaded or + * discoverable. Promotion requires independent held-out evidence in the + * lifecycle layer. */ -import path from "path" -import fs from "fs/promises" -import { Global } from "@/global" import { Log } from "@/util/log" import { RSITrajectory } from "./trajectory" +import { HarnessSkill } from "../harness/skill" export namespace RSIDistill { const log = Log.create({ service: "rsi-distill" }) - const LEARNED_SKILLS_DIR = path.join(Global.Path.data, "learned-skills") const SCORE_THRESHOLD = 75 - /** Distill a learned skill from a scored trajectory. - * Only generates a skill if score >= threshold. Returns the skill name or null. */ - export async function distill(trajectory: RSITrajectory.Trajectory): Promise { - if (!trajectory.score || trajectory.score < SCORE_THRESHOLD) { - log.info("trajectory below threshold, skipping distill", { + /** Draft a proposal only when an external evaluator passed the trajectory. */ + export async function propose(trajectory: RSITrajectory.Trajectory): Promise { + if (!trajectory.score || trajectory.score < SCORE_THRESHOLD || trajectory.outcome !== "success") { + log.info("trajectory ineligible for skill proposal", { sessionId: trajectory.sessionId, score: trajectory.score, + outcome: trajectory.outcome, }) return null } + if (!trajectory.verification || trajectory.verification.status !== "passed") return null const hash = trajectory.sessionId.slice(-8) const name = `learned-${trajectory.agent}-${hash}` const description = generateDescription(trajectory) const content = generateSkillContent(name, description, trajectory) - // Write to local disk - const dir = path.join(LEARNED_SKILLS_DIR, name) - await fs.mkdir(dir, { recursive: true }) - await Bun.write(path.join(dir, "SKILL.md"), content) - log.info("learned skill distilled", { name, score: trajectory.score }) + await HarnessSkill.propose({ + name, + description, + content, + origin: "rsi", + sessionID: trajectory.sessionId, + runID: trajectory.verification.runID, + createdAt: trajectory.timestamp, + }) + log.info("learned skill proposal drafted", { name, score: trajectory.score }) return name } @@ -57,7 +64,8 @@ export namespace RSIDistill { return `--- name: ${name} description: ${description} -source: rsi +source: rsi-proposal +status: pending trajectory_id: ${trajectory.sessionId} score: ${trajectory.score} metadata: @@ -68,23 +76,24 @@ metadata: ## Overview -This skill was automatically distilled from a high-scoring research trajectory -(score: ${trajectory.score}/100) by the RSI (Recursive Self-Improvement) system. -It captures a validated research workflow pattern. +This is an inert skill proposal drafted from an externally evaluated research +trajectory. It is not active until held-out evaluation and review promote it. ## Origin - **Agent**: ${trajectory.agent} - **Hypothesis**: ${trajectory.hypothesis} - **Outcome**: ${trajectory.outcome} +- **Evaluator**: ${trajectory.verification?.evaluator} +- **Evaluation status**: ${trajectory.verification?.status} - **Score**: ${trajectory.score}/100 - **Steps**: ${trajectory.steps.length} - **Distilled**: ${new Date(trajectory.timestamp).toISOString()} ## Workflow Pattern -This research pattern was validated through execution and critic evaluation. -Follow these steps when encountering similar research questions: +This pattern passed one external evaluation. Treat it as a candidate procedure +to test on independent tasks, not as established scientific guidance. ${toolSequence} diff --git a/backend/cli/src/session/rsi/lifecycle.ts b/backend/cli/src/session/rsi/lifecycle.ts index b629d308..8feb9b4b 100644 --- a/backend/cli/src/session/rsi/lifecycle.ts +++ b/backend/cli/src/session/rsi/lifecycle.ts @@ -3,20 +3,22 @@ * * - Tracks usage count per learned skill * - Archives skills with 0 uses after 30 days - * - Flags high performers (>10 uses) for potential promotion + * - Reports frequently used skills without treating usage as performance */ import path from "path" import fs from "fs/promises" import { Global } from "@/global" import { Log } from "@/util/log" +import { HarnessEvaluation } from "../harness/evaluation" export namespace RSILifecycle { const log = Log.create({ service: "rsi-lifecycle" }) const LEARNED_SKILLS_DIR = path.join(Global.Path.data, "learned-skills") const STATS_PATH = path.join(LEARNED_SKILLS_DIR, ".stats.json") + const PROPOSAL_STATS_PATH = path.join(Global.Path.data, "learned-skill-proposals", ".stats.json") const ARCHIVE_AFTER_DAYS = 30 - const HIGH_PERFORMER_THRESHOLD = 10 + const FREQUENTLY_USED_THRESHOLD = 10 interface Stats { skills: Record @@ -29,6 +31,20 @@ export namespace RSILifecycle { created: number } + interface ProposalStats { + proposals: Record< + string, + { + status: "pending" | "promoted" | "rejected" + sessionID: string + runID: string + evaluator: string + score?: number + created: number + } + > + } + async function readStats(): Promise { try { return await Bun.file(STATS_PATH).json() @@ -74,6 +90,23 @@ export namespace RSILifecycle { } } + /** Register inert evaluator-backed output for later held-out review. */ + export async function registerProposal(name: string, evaluation: HarnessEvaluation.Info): Promise { + const stats = (await Bun.file(PROPOSAL_STATS_PATH) + .json() + .catch(() => ({ proposals: {} }))) as ProposalStats + stats.proposals[name] = { + status: "pending", + sessionID: evaluation.sessionID, + runID: evaluation.runID, + evaluator: evaluation.evaluator.name, + score: evaluation.score, + created: Date.now(), + } + await fs.mkdir(path.dirname(PROPOSAL_STATS_PATH), { recursive: true }) + await Bun.write(PROPOSAL_STATS_PATH, JSON.stringify(stats, null, 2) + "\n") + } + /** Get stats for a skill. */ export async function getStats(skillName: string): Promise { const stats = await readStats() @@ -106,15 +139,15 @@ export namespace RSILifecycle { return archived } - /** Find high-performing skills (>HIGH_PERFORMER_THRESHOLD uses). */ - export async function highPerformers(): Promise { + /** Usage is an adoption signal only; it is not evidence of correctness. */ + export async function frequentlyUsed(): Promise { const stats = await readStats() return Object.entries(stats.skills) - .filter(([, s]) => s.usageCount > HIGH_PERFORMER_THRESHOLD) + .filter(([, s]) => s.usageCount > FREQUENTLY_USED_THRESHOLD) .map(([name]) => name) } - /** Startup lifecycle check — archive unused, log high performers. */ + /** Startup lifecycle check — archive unused, report adoption. */ export async function startupCheck(): Promise { try { const archived = await archiveUnused() @@ -122,9 +155,9 @@ export namespace RSILifecycle { log.info("startup: archived unused learned skills", { count: archived }) } - const performers = await highPerformers() - if (performers.length > 0) { - log.info("startup: high-performing learned skills", { skills: performers }) + const frequent = await frequentlyUsed() + if (frequent.length > 0) { + log.info("startup: frequently used learned skills", { skills: frequent }) } } catch (e) { log.warn("lifecycle startup check failed", { error: e instanceof Error ? e.message : String(e) }) diff --git a/backend/cli/src/session/rsi/trajectory.ts b/backend/cli/src/session/rsi/trajectory.ts index 8d8d517d..a15ac189 100644 --- a/backend/cli/src/session/rsi/trajectory.ts +++ b/backend/cli/src/session/rsi/trajectory.ts @@ -12,12 +12,13 @@ import { RLMState } from "../rlm/state" import { RSICritic } from "./critic" import { RSIDistill } from "./distill" import { RSILifecycle } from "./lifecycle" +import { HarnessEvaluation } from "../harness/evaluation" export namespace RSITrajectory { const log = Log.create({ service: "rsi-trajectory" }) const TRAJECTORIES_DIR = path.join(Global.Path.data, "trajectories") - export const ARTIFACT_AGENTS = ["research", "biology", "ml"] as const + export const ARTIFACT_AGENTS = ["research", "biology", "physics", "ml"] as const export interface TrajectoryStep { tool: string @@ -32,9 +33,17 @@ export namespace RSITrajectory { agent: string hypothesis: string steps: TrajectoryStep[] - outcome: "success" | "partial" | "failure" + reportedOutcome: "success" | "partial" | "failure" + outcome: "unverified" | "success" | "partial" | "failure" tokenCost: number score?: number + verification?: { + runID: string + evaluator: string + status: HarnessEvaluation.Status + score?: number + evaluatedAt: number + } } /** Capture a trajectory from a completed ultra agent session. @@ -96,27 +105,23 @@ export namespace RSITrajectory { } } - // Determine outcome from last RLM state or heuristic - let outcome: Trajectory["outcome"] = "success" - for (let i = messages.length - 1; i >= 0; i--) { - const msg = messages[i] - if (msg.info.role !== "assistant") continue - for (const part of msg.parts) { - if (part.type === "text") { - const state = RLMState.parseResearchState(part.text) - if (state) { - const hasFailures = state.plan.some((o) => o.status === "failed") - const allDone = state.plan.every((o) => o.status === "done" || o.status === "failed") - const allFailed = state.plan.every((o) => o.status === "failed") - if (allFailed) outcome = "failure" - else if (hasFailures) outcome = "partial" - else if (state.status === "complete" || allDone) outcome = "success" - break - } - } - } - break - } + // This is the agent's own process report, not scientific verification. + // Keep it for diagnosis, but never use it to activate learned behavior. + const reportedOutcome = (() => { + const assistant = messages.findLast((message) => message.info.role === "assistant") + if (!assistant) return "partial" as const + const states = assistant.parts + .filter((part) => part.type === "text") + .map((part) => RLMState.parseResearchState(part.text)) + .filter((state) => state !== null) + const state = states.at(-1) + if (!state) return "partial" as const + const allFailed = state.plan.length > 0 && state.plan.every((item) => item.status === "failed") + if (allFailed) return "failure" as const + if (state.plan.some((item) => item.status === "failed")) return "partial" as const + const done = state.plan.length > 0 && state.plan.every((item) => item.status === "done") + return state.status === "complete" || done ? ("success" as const) : ("partial" as const) + })() // Estimate token cost from message count (rough heuristic) const tokenCost = messages.reduce((acc, m) => { @@ -129,7 +134,8 @@ export namespace RSITrajectory { agent, hypothesis, steps, - outcome, + reportedOutcome, + outcome: "unverified", tokenCost: Math.round(tokenCost), } @@ -153,7 +159,16 @@ export namespace RSITrajectory { export async function read(sessionId: string): Promise { try { const filePath = path.join(TRAJECTORIES_DIR, `${sessionId}.json`) - return await Bun.file(filePath).json() + const value = (await Bun.file(filePath).json()) as Trajectory & { + reportedOutcome?: Trajectory["reportedOutcome"] + } + if (value.reportedOutcome) return value + return { + ...value, + reportedOutcome: value.outcome === "failure" ? "failure" : value.outcome === "partial" ? "partial" : "success", + outcome: "unverified", + verification: undefined, + } } catch { return null } @@ -178,8 +193,8 @@ export namespace RSITrajectory { await Bun.write(filePath, JSON.stringify(trajectory, null, 2)) } - /** Full RSI pipeline: capture → evaluate → score → distill → register. - * All errors caught internally — safe to fire-and-forget. */ + /** Capture and process-score a trajectory. Scientific correctness remains + * unverified until recordEvaluation receives an external evaluator result. */ export async function pipeline(sessionID: string): Promise { try { const trajectory = await capture(sessionID) @@ -187,21 +202,43 @@ export namespace RSITrajectory { const score = RSICritic.evaluate(trajectory) await setScore(sessionID, score.total) - - if (score.total >= 75) { - const name = await RSIDistill.distill({ ...trajectory, score: score.total }) - if (name) { - await RSILifecycle.registerSkill(name) - log.info("pipeline: skill distilled and registered", { sessionId: sessionID, name, score: score.total }) - } - } else { - log.info("pipeline: score below threshold, skipping distill", { sessionId: sessionID, score: score.total }) - } + log.info("pipeline: trajectory awaits external evaluation", { sessionId: sessionID, processScore: score.total }) } catch (e) { log.error("pipeline failed", { sessionId: sessionID, error: e instanceof Error ? e.message : String(e) }) } } + /** Persist an external result and, only for a verified pass, draft an inert + * skill proposal. Proposals are not discoverable skills until promoted. */ + export async function recordEvaluation(input: HarnessEvaluation.Info) { + const evaluation = await HarnessEvaluation.record(input) + const trajectory = await read(evaluation.sessionID) + if (!trajectory) throw new Error(`No RSI trajectory exists for session ${evaluation.sessionID}`) + + trajectory.verification = { + runID: evaluation.runID, + evaluator: evaluation.evaluator.name, + status: evaluation.status, + score: evaluation.score, + evaluatedAt: evaluation.evaluatedAt, + } + trajectory.outcome = + evaluation.status === "passed" ? "success" : evaluation.status === "failed" ? "failure" : "partial" + const score = RSICritic.evaluate(trajectory) + trajectory.score = score.total + await persist(trajectory) + + if (!HarnessEvaluation.verified(evaluation)) return { trajectory, proposal: null } + const proposal = await RSIDistill.propose(trajectory) + if (proposal) await RSILifecycle.registerProposal(proposal, evaluation) + return { trajectory, proposal } + } + + async function persist(trajectory: Trajectory) { + await fs.mkdir(TRAJECTORIES_DIR, { recursive: true }) + await Bun.write(path.join(TRAJECTORIES_DIR, `${trajectory.sessionId}.json`), JSON.stringify(trajectory, null, 2)) + } + function summarize(text: string, maxLen: number): string { if (text.length <= maxLen) return text return text.slice(0, maxLen - 3) + "..." diff --git a/backend/cli/src/session/trace-store.ts b/backend/cli/src/session/trace-store.ts index 211d9c7d..2e1cc3e3 100644 --- a/backend/cli/src/session/trace-store.ts +++ b/backend/cli/src/session/trace-store.ts @@ -4,6 +4,7 @@ import { Global } from "@/global" import { JsonStore } from "@/util/jsonstore" import { Log } from "@/util/log" import z from "zod" +import { HarnessContract } from "./harness/contract" export namespace SessionTraceStore { const log = Log.create({ service: "session.trace.store" }) @@ -34,13 +35,24 @@ export namespace SessionTraceStore { }) export type Retry = z.infer + export const Profile = z.object({ + messageID: z.string(), + id: HarnessContract.Profile, + source: z.enum(["contract", "heuristic", "control"]), + confidence: z.number().min(0).max(1), + reasons: z.array(z.string()), + selectedAt: z.number(), + }) + export type Profile = z.infer + const State = z.object({ approvals: z.record(z.string(), Approval).default({}), retries: z.array(Retry).default([]), + profiles: z.record(z.string(), Profile).default({}), }) export type State = z.infer - const empty = (): State => ({ approvals: {}, retries: [] }) + const empty = (): State => ({ approvals: {}, retries: [], profiles: {} }) const file = (sessionID: string) => path.join(Global.Path.data, "trace", `${encodeURIComponent(sessionID)}.json`) async function update(sessionID: string, fn: (state: State) => State) { @@ -111,6 +123,26 @@ export namespace SessionTraceStore { return update(input.sessionID, (state) => ({ ...state, retries: [...state.retries, item] })) } + export function recordProfile(input: Omit & { sessionID: string }) { + return update(input.sessionID, (state) => { + if (state.profiles[input.messageID]) return state + return { + ...state, + profiles: { + ...state.profiles, + [input.messageID]: { + messageID: input.messageID, + id: input.id, + source: input.source, + confidence: input.confidence, + reasons: input.reasons, + selectedAt: Date.now(), + }, + }, + } + }) + } + export async function remove(sessionID: string) { await fs.unlink(file(sessionID)).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return diff --git a/backend/cli/src/session/trace.ts b/backend/cli/src/session/trace.ts index ed0d2b9e..e1922cd2 100644 --- a/backend/cli/src/session/trace.ts +++ b/backend/cli/src/session/trace.ts @@ -217,6 +217,7 @@ export namespace SessionTrace { reviewerFindings: z.array(Finding), failures: z.array(Failure), retries: z.array(SessionTraceStore.Retry), + profiles: z.array(SessionTraceStore.Profile), privacy: z.object({ local: z.literal(true), atlasRequired: z.literal(false), @@ -643,6 +644,7 @@ export namespace SessionTrace { reviewerFindings, failures, retries: stored.retries, + profiles: Object.values(stored.profiles).toSorted((a, b) => a.selectedAt - b.selectedAt), privacy: { local: true, atlasRequired: false, diff --git a/backend/cli/src/tool/claim.ts b/backend/cli/src/tool/claim.ts new file mode 100644 index 00000000..f6cf19ff --- /dev/null +++ b/backend/cli/src/tool/claim.ts @@ -0,0 +1,144 @@ +import z from "zod" +import { HarnessClaims } from "@/session/harness/claims" +import { Tool } from "./tool" +import DESCRIPTION from "./claim.txt" + +const Parameters = z.object({ + action: z.enum(["declare", "observe", "status"]), + claim_id: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For observe/status: claim identity"), + text: z.string().min(1).max(4_000).optional().describe("For declare: exact scientific claim"), + kind: HarnessClaims.Kind.optional().describe("For declare: epistemic claim kind"), + importance: z.enum(["supporting", "headline"]).optional().describe("For declare: role in the final conclusion"), + subject_uri: z + .string() + .min(1) + .max(2_048) + .optional() + .describe("For declare: exact artifact, output, or report subject"), + subject_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For declare: immutable subject digest"), + provenance_id: z.string().min(1).max(200).optional().describe("For declare: local provenance node"), + independent_sources: z.number().int().min(1).max(5).optional().describe("For declare: strengthen source count"), + required_checks: z + .array(z.string().min(1).max(100)) + .max(24) + .optional() + .describe("For declare: extra required checks"), + evidence_kind: z + .enum(["observation", "measurement", "statistical_test", "citation", "artifact", "review"]) + .optional() + .describe("For observe: provisional evidence kind"), + stance: z.enum(["supports", "refutes", "inconclusive"]).optional().describe("For observe: provisional relation"), + summary: z.string().min(1).max(2_000).optional().describe("For observe: what was observed"), + source_uri: z.string().min(1).max(2_048).optional().describe("For observe: exact source reference"), + source_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For observe: source digest"), + evidence: z.array(z.string().min(1).max(1_000)).max(32).optional().describe("For observe: evidence references"), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "At most 128 measured values") + .optional() + .describe("For observe: measured values"), +}) + +const result = (title: string, output: unknown, metadata: Record = {}) => ({ + title, + output: typeof output === "string" ? output : JSON.stringify(output, null, 2), + metadata, +}) + +const summary = (claim: HarnessClaims.View) => ({ + id: claim.id, + text: claim.text, + kind: claim.kind, + importance: claim.importance, + status: claim.status, + requirements: claim.requirements, + independentSources: claim.independentSources, + passedChecks: claim.passedChecks, + missingChecks: claim.missingChecks, + evidence: claim.evidence.map((item) => ({ + id: item.id, + origin: item.origin, + stance: item.stance, + kind: item.kind, + summary: item.summary, + actor: item.source.actor, + })), +}) + +export const ClaimTool = Tool.define("claim", { + description: DESCRIPTION, + parameters: Parameters, + async execute(params, ctx) { + if (params.action === "declare") { + if (!params.text || !params.kind || !params.importance || !params.subject_uri) { + return result("Invalid claim", "declare requires text, kind, importance, and subject_uri") + } + const claim = await HarnessClaims.declare({ + sessionID: ctx.sessionID, + actor: ctx.agent, + messageID: ctx.messageID, + text: params.text, + kind: params.kind, + importance: params.importance, + subject: { + uri: params.subject_uri, + sha256: params.subject_sha256, + provenanceID: params.provenance_id, + }, + requirements: { + independentSources: params.independent_sources, + checks: params.required_checks, + }, + }) + const view = await HarnessClaims.get(ctx.sessionID, claim.id) + return result("Scientific claim declared", summary(view!), { claimID: claim.id, status: view!.status }) + } + + if (params.action === "observe") { + if (!params.claim_id || !params.evidence_kind || !params.stance || !params.summary || !params.source_uri) { + return result( + "Invalid claim evidence", + "observe requires claim_id, evidence_kind, stance, summary, and source_uri", + ) + } + const evidence = await HarnessClaims.observe({ + sessionID: ctx.sessionID, + claimID: params.claim_id, + actor: ctx.agent, + kind: params.evidence_kind, + stance: params.stance, + summary: params.summary, + source: { uri: params.source_uri, sha256: params.source_sha256 }, + evidence: params.evidence, + metrics: params.metrics, + }) + const view = await HarnessClaims.get(ctx.sessionID, params.claim_id) + return result("Provisional claim evidence recorded", summary(view!), { + claimID: params.claim_id, + evidenceID: evidence.id, + origin: "observed", + status: view!.status, + }) + } + + if (params.claim_id) { + const claim = await HarnessClaims.get(ctx.sessionID, params.claim_id) + if (!claim) return result("Claim not found", `No claim ${params.claim_id}.`, { found: false }) + return result("Scientific claim status", summary(claim), { claimID: claim.id, status: claim.status }) + } + const claims = await HarnessClaims.list(ctx.sessionID) + return result("Scientific claim ledger", claims.map(summary), { count: claims.length }) + }, +}) diff --git a/backend/cli/src/tool/claim.txt b/backend/cli/src/tool/claim.txt new file mode 100644 index 00000000..2968a129 --- /dev/null +++ b/backend/cli/src/tool/claim.txt @@ -0,0 +1,3 @@ +Maintain the typed scientific claim ledger for this session. Use `declare` before presenting a material scientific conclusion or benchmark-performance claim. Claim requirements are selected by kind and can only be strengthened. Use `observe` to attach measurements, citations, artifacts, reviews, or other provisional evidence; agent observations never count as verified support. Use `status` to inspect the derived result, missing checks, independent-source count, and evidence trail. + +There is intentionally no action for verification or changing status. Only backend-recorded held-out evaluation, clean replay, independent implementation or derivation, or adversarial review can add verified evidence. The ledger derives `supported`, `refuted`, or `inconclusive`; do not rewrite that status in prose. diff --git a/backend/cli/src/tool/harness.ts b/backend/cli/src/tool/harness.ts new file mode 100644 index 00000000..6110e9d0 --- /dev/null +++ b/backend/cli/src/tool/harness.ts @@ -0,0 +1,539 @@ +import z from "zod" +import { HarnessMemory } from "@/session/harness/memory" +import { HarnessOrchestrator } from "@/session/harness/orchestrator" +import { HarnessSearch } from "@/session/harness/search" +import { HarnessWorld } from "@/session/harness/world" +import { Tool } from "./tool" +import DESCRIPTION from "./harness.txt" + +const Parameters = z.object({ + action: z.enum([ + "start", + "status", + "dispatch", + "release", + "propose", + "observe", + "hindsight", + "coalition_start", + "coalition_status", + "coalition_complete", + "coalition_fail", + "world_status", + "world_event", + "world_refine", + "world_rollback", + ]), + stall: z + .number() + .int() + .min(1) + .max(50) + .optional() + .describe("For start: evaluations without improvement before fusion"), + parent_ids: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(2) + .optional() + .describe("For propose: verified parents"), + inspiration_ids: z + .array(z.string().regex(/^[a-f0-9]{64}$/)) + .max(2) + .optional() + .describe("For propose: verified inspirations returned by a migration recommendation"), + recommendation_id: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For propose: current content-addressed recommendation lease returned by start or status"), + reservation_id: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For propose/release: budget-backed parallel variation reservation"), + count: z.number().int().min(1).max(8).optional().describe("For dispatch: maximum parallel reservations to issue"), + branch: z.string().min(1).max(120).optional().describe("For propose: stable diversity branch label"), + proposal: z.string().min(1).max(4_000).optional().describe("For propose: concise description of the change"), + artifact_uri: z.string().min(1).max(2_048).optional().describe("For propose: immutable candidate artifact reference"), + artifact_sha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For propose: SHA-256 of the exact candidate artifact"), + candidate_id: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For observe: candidate identity"), + status: z.enum(["passed", "failed", "inconclusive"]).optional().describe("For observe: provisional status"), + score: z.number().finite().optional().describe("For observe: provisional primary score"), + metrics: z + .record(z.string().max(200), z.number().finite()) + .refine((value) => Object.keys(value).length <= 128, "At most 128 provisional metrics") + .optional() + .describe("For observe: provisional metric values"), + evidence: z.array(z.string().min(1).max(500)).max(12).optional().describe("For observe: provisional references"), + feedback: z.string().max(4_000).optional().describe("For observe: provisional evaluator or process feedback"), + query: z.string().min(1).max(2_000).optional().describe("For hindsight: current problem or failure query"), + stage: HarnessMemory.Stage.optional().describe("For hindsight: current search stage"), + limit: z.number().int().min(1).max(6).optional().describe("For hindsight: maximum precedents"), + event_type: HarnessWorld.EventType.optional().describe("For world_event: analysis or external-state event type"), + event_summary: z.string().min(1).max(1_000).optional().describe("For world_event: concise observed event"), + state_changed: z.boolean().optional().describe("For world_event: whether external state changed"), + expected_revision: z + .number() + .int() + .nonnegative() + .optional() + .describe("For world_refine/world_rollback: exact current world-model revision"), + world_reason: HarnessWorld.Reason.optional().describe("For world_refine: refinement trigger"), + world_patches: z + .array(HarnessWorld.AgentPatch) + .min(1) + .max(6) + .optional() + .describe("For world_refine: small self-attributed upserts or removals"), + target_revision: z + .number() + .int() + .nonnegative() + .optional() + .describe("For world_rollback: prior snapshot revision, defaulting to the latest"), + work_id: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional() + .describe("For coalition_complete/coalition_fail: orchestration work identity"), + worker_session_id: z + .string() + .min(1) + .optional() + .describe( + "For coalition_complete/coalition_fail: exact ready.resumeSessionID for a resumed producer lane, otherwise a fresh Task child session identity", + ), + result_summary: z.string().min(1).max(8_000).optional().describe("For coalition_complete: concise result"), + artifact_refs: z + .array(z.string().min(1).max(2_048)) + .max(32) + .optional() + .describe("For coalition_complete: immutable artifact references"), + evidence_refs: z + .array(z.string().min(1).max(2_048)) + .max(32) + .optional() + .describe("For coalition_complete: observable evidence references"), + usage: z + .object({ + steps: z.number().int().nonnegative().optional(), + tokens: z.number().int().nonnegative().optional(), + costUSD: z.number().nonnegative().optional(), + wallTimeMs: z.number().int().nonnegative().optional(), + }) + .strict() + .optional() + .describe("For coalition_complete: actual resource use"), + failure: z.string().min(1).max(4_000).optional().describe("For coalition_fail: failure reason"), + verdict: z + .enum(["support", "reject", "abstain"]) + .optional() + .describe("For verification coalition_complete: blinded structured verdict"), + verdict_severity: z + .enum(["none", "minor", "critical", "unknown"]) + .optional() + .describe("For verifier_loop coalition_complete: none, minor, critical, or unknown diagnosis"), + verdict_confidence: z + .number() + .min(0) + .max(1) + .optional() + .describe("For verification coalition_complete: calibrated verdict confidence"), + verdict_checks: z + .array( + z + .object({ + id: z.string().min(1).max(200), + status: z.enum(["passed", "failed", "inconclusive"]), + evidence_refs: z.array(z.string().min(1).max(2_048)).min(1).max(16), + }) + .strict(), + ) + .min(1) + .max(64) + .optional() + .describe("For verification coalition_complete: evidence-backed checks"), +}) + +const result = (title: string, output: unknown, metadata: Record = {}) => ({ + title, + output: typeof output === "string" ? output : JSON.stringify(output, null, 2), + metadata, +}) + +const view = (candidate: HarnessSearch.Candidate, mandate?: HarnessSearch.Mandate) => ({ + id: candidate.id, + parentIDs: candidate.parentIDs, + inspirationIDs: candidate.inspirationIDs, + branch: candidate.branch, + generation: candidate.generation, + island: candidate.island, + ordinal: candidate.ordinal, + proposal: candidate.proposal.slice(0, 1_000), + artifact: candidate.artifact, + lease: candidate.lease, + reservationID: candidate.reservationID, + mandate, + source: candidate.result?.source, + status: candidate.result?.status, + score: candidate.result?.score, + metrics: candidate.result?.metrics, + feedback: candidate.result?.feedback?.slice(0, 2_000), +}) + +const summary = (state: HarnessSearch.State) => { + const recommendation = state.status === "active" ? HarnessSearch.recommend(state) : undefined + return { + runID: state.runID, + status: state.status, + stopReason: state.stopReason, + metric: state.metric, + direction: state.direction, + target: state.target, + population: state.population, + proposalPolicy: state.proposalPolicy, + budget: state.budget, + used: Object.keys(state.candidates).length, + bestID: state.bestID, + archiveIDs: state.archiveIDs, + stalled: state.stalled, + revision: state.revision, + recommendation, + recommendationContext: recommendation?.contextIDs.map((id) => { + const candidate = state.candidates[id]! + const mandate = candidate.reservationID ? state.reservations[candidate.reservationID]?.mandate : undefined + return view(candidate, mandate) + }), + reservations: { + open: Object.values(state.reservations).filter((item) => item.status === "open").length, + consumed: Object.values(state.reservations).filter((item) => item.status === "consumed").length, + released: Object.values(state.reservations).filter((item) => item.status === "released").length, + ready: Object.values(state.reservations) + .filter((item) => item.status === "open") + .map((item) => ({ + id: item.id, + parentIDs: item.parentIDs, + inspirationIDs: item.inspirationIDs, + lease: item.lease, + mandate: item.mandate, + })), + }, + candidates: Object.values(state.candidates) + .toSorted((a, b) => b.createdAt - a.createdAt) + .slice(0, 20) + .map((candidate) => { + const mandate = candidate.reservationID ? state.reservations[candidate.reservationID]?.mandate : undefined + return view(candidate, mandate) + }), + } +} + +const coalition = (state: HarnessOrchestrator.State) => ({ + runID: state.runID, + status: state.status, + protocolVersion: state.protocolVersion, + sessionPolicy: state.sessionPolicy, + workerPolicy: state.workerPolicy, + topology: state.selection.topology, + selectionSource: state.selection.source, + selectionReasons: state.selection.reasons, + traits: state.selection.traits, + maxWorkers: state.maxWorkers, + maxRounds: state.maxRounds, + minIndependentVerifiers: state.minIndependentVerifiers, + adaptive: state.adaptive, + repair: state.repair, + consensus: state.consensus, + revision: state.revision, + progress: Object.fromEntries( + ["pending", "executed", "completed", "failed", "cancelled"].map((status) => [ + status, + Object.values(state.work).filter((item) => item.status === status).length, + ]), + ), + executed: Object.values(state.work) + .filter((work) => work.status === "executed") + .map((work) => ({ + id: work.id, + role: work.role, + label: work.label, + workerSessionID: work.workerSessionID, + receiptID: work.workerReceipt?.id, + outcome: work.workerReceipt?.outcome, + usage: work.workerReceipt?.usage, + })), + ready: HarnessOrchestrator.ready(state) + .slice(0, state.maxWorkers) + .map((work) => ({ + id: work.id, + role: work.role, + label: work.label, + round: work.round, + agent: work.agent, + lane: work.lane, + resumeSessionID: work.resumeSessionID, + prompt: work.prompt, + allocation: work.allocation, + context: work.context, + })), +}) + +export const HarnessTool = Tool.define("harness", { + description: DESCRIPTION, + parameters: Parameters, + async execute(params, ctx) { + if (params.action === "world_status") { + const state = await HarnessWorld.read(ctx.sessionID) + return result("Continual world model", HarnessWorld.summary(state), { + revision: state.revision, + contextEpoch: state.contextEpoch, + refinementRecommended: state.refinement.recommended, + }) + } + + if (params.action === "world_event") { + if (!params.event_type || !params.event_summary) { + return result("Invalid world event", "world_event requires event_type and event_summary") + } + const state = await HarnessWorld.event({ + sessionID: ctx.sessionID, + type: params.event_type, + summary: params.event_summary, + changed: params.state_changed, + evidenceRefs: params.evidence, + }) + return result("World-model event recorded", HarnessWorld.summary(state), { + revision: state.revision, + contextEpoch: state.contextEpoch, + refinementRecommended: state.refinement.recommended, + }) + } + + if (params.action === "world_refine") { + if (params.expected_revision === undefined || !params.world_reason || !params.world_patches) { + return result( + "Invalid world-model refinement", + "world_refine requires expected_revision, world_reason, and world_patches", + ) + } + const state = await HarnessWorld.agentRefine({ + sessionID: ctx.sessionID, + expectedRevision: params.expected_revision, + reason: params.world_reason, + patches: params.world_patches, + }) + return result("World model refined", HarnessWorld.summary(state), { + revision: state.revision, + contextEpoch: state.contextEpoch, + }) + } + + if (params.action === "world_rollback") { + if (params.expected_revision === undefined) { + return result("Invalid world-model rollback", "world_rollback requires expected_revision") + } + const state = await HarnessWorld.rollback({ + sessionID: ctx.sessionID, + expectedRevision: params.expected_revision, + targetRevision: params.target_revision, + }) + return result("World model rolled back", HarnessWorld.summary(state), { + revision: state.revision, + contextEpoch: state.contextEpoch, + }) + } + + if (params.action === "coalition_start") { + const state = await HarnessOrchestrator.initialize(ctx.sessionID) + return result("Scientific coalition initialized", coalition(state), { + runID: state.runID, + topology: state.selection.topology, + revision: state.revision, + }) + } + + if (params.action === "coalition_status") { + const state = await HarnessOrchestrator.read(ctx.sessionID) + return result("Scientific coalition checkpoint", coalition(state), { + runID: state.runID, + topology: state.selection.topology, + revision: state.revision, + }) + } + + if (params.action === "coalition_complete") { + if (!params.work_id || !params.worker_session_id || !params.result_summary) { + return result( + "Invalid coalition completion", + "coalition_complete requires work_id, worker_session_id, and result_summary", + ) + } + const verdict = [params.verdict, params.verdict_confidence, params.verdict_checks] + if (verdict.some((value) => value !== undefined) && verdict.some((value) => value === undefined)) { + return result( + "Invalid coalition completion", + "verification completion requires verdict, verdict_confidence, and verdict_checks together", + ) + } + if (params.verdict_severity && !params.verdict) { + return result("Invalid coalition completion", "verdict_severity requires a structured verdict") + } + const state = await HarnessOrchestrator.complete({ + sessionID: ctx.sessionID, + workID: params.work_id, + workerSessionID: params.worker_session_id, + result: { + summary: params.result_summary, + artifactRefs: params.artifact_refs ?? [], + evidenceRefs: params.evidence_refs ?? [], + usage: params.usage, + verdict: + params.verdict && params.verdict_confidence !== undefined && params.verdict_checks + ? { + decision: params.verdict, + severity: params.verdict_severity, + confidence: params.verdict_confidence, + checks: params.verdict_checks.map((check) => ({ + id: check.id, + status: check.status, + evidenceRefs: check.evidence_refs, + })), + } + : undefined, + }, + }) + return result("Coalition work completed", coalition(state), { + workID: params.work_id, + provisional: true, + revision: state.revision, + }) + } + + if (params.action === "coalition_fail") { + if (!params.work_id || !params.worker_session_id || !params.failure) { + return result("Invalid coalition failure", "coalition_fail requires work_id, worker_session_id, and failure") + } + const state = await HarnessOrchestrator.fail({ + sessionID: ctx.sessionID, + workID: params.work_id, + workerSessionID: params.worker_session_id, + failure: params.failure, + }) + return result("Coalition work failed", coalition(state), { + workID: params.work_id, + provisional: true, + revision: state.revision, + }) + } + + if (params.action === "start") { + const state = await HarnessSearch.initialize({ sessionID: ctx.sessionID, stall: params.stall }) + return result("Optimization search started", summary(state), { runID: state.runID, revision: state.revision }) + } + + if (params.action === "status") { + const state = await HarnessSearch.read(ctx.sessionID) + return result("Optimization search checkpoint", summary(state), { + runID: state.runID, + revision: state.revision, + bestID: state.bestID, + }) + } + + if (params.action === "dispatch") { + const reserved = await HarnessSearch.reserve({ sessionID: ctx.sessionID, count: params.count ?? 1 }) + return result("Parallel variations reserved", summary(reserved.state), { + reservationIDs: reserved.reservations.map((item) => item.id), + issued: reserved.reservations.length, + revision: reserved.state.revision, + }) + } + + if (params.action === "release") { + if (!params.reservation_id) return result("Invalid release", "release requires reservation_id") + const state = await HarnessSearch.release({ + sessionID: ctx.sessionID, + reservationID: params.reservation_id, + }) + return result("Parallel variation released", summary(state), { + reservationID: params.reservation_id, + revision: state.revision, + }) + } + + if (params.action === "propose") { + const admission = [params.recommendation_id, params.reservation_id].filter((item) => item !== undefined) + if ( + admission.length !== 1 || + !params.branch || + !params.proposal || + !params.artifact_uri || + !params.artifact_sha256 + ) { + return result( + "Invalid proposal", + "propose requires exactly one of recommendation_id or reservation_id, plus branch, proposal, artifact_uri, and artifact_sha256", + ) + } + const added = await HarnessSearch.add({ + sessionID: ctx.sessionID, + recommendationID: params.recommendation_id, + reservationID: params.reservation_id, + parentIDs: params.parent_ids ?? [], + inspirationIDs: params.inspiration_ids ?? [], + branch: params.branch, + proposal: params.proposal, + artifact: { uri: params.artifact_uri, sha256: params.artifact_sha256 }, + }) + return result(added.accepted ? "Candidate registered" : "Candidate rejected", summary(added.state), { + candidateID: added.id, + accepted: added.accepted, + deduplicated: added.deduplicated, + revision: added.state.revision, + }) + } + + if (params.action === "observe") { + if (!params.candidate_id || !params.status) { + return result("Invalid observation", "observe requires candidate_id and status") + } + const state = await HarnessSearch.observe({ + sessionID: ctx.sessionID, + candidateID: params.candidate_id, + status: params.status, + score: params.score, + metrics: params.metrics, + evidence: params.evidence, + feedback: params.feedback, + }) + return result("Unverified observation recorded", summary(state), { + candidateID: params.candidate_id, + verified: false, + revision: state.revision, + }) + } + + if (!params.query) return result("Invalid hindsight query", "hindsight requires query") + const prompt = await HarnessMemory.prompt({ + sessionID: ctx.sessionID, + query: params.query, + stage: params.stage, + limit: params.limit, + }) + return result( + prompt ? "Verified hindsight" : "No verified hindsight", + prompt || "No relevant verified precedents.", + { + found: !!prompt, + }, + ) + }, +}) diff --git a/backend/cli/src/tool/harness.txt b/backend/cli/src/tool/harness.txt new file mode 100644 index 00000000..1a37756c --- /dev/null +++ b/backend/cli/src/tool/harness.txt @@ -0,0 +1,15 @@ +Operate an evaluation-bound scientific search. This tool is available only as a control surface over an immutable harness contract created by the generic adapter; it cannot create or change that contract. + +Use `world_status` to inspect the session-local continual world model. Record analysis boundaries with `world_event` and `state_changed=false`; record tool results, failures, milestones, stagnation, and other external changes with the correct event type. External changes advance the context epoch, while analysis alone preserves the current reasoning chain. When the backend recommends refinement, use `world_refine` with the exact revision and at most six small upserts or removals. Agent-authored beliefs are capped at confidence 3 and remain self-attributed; only evaluator-owned product APIs may record stronger evidence. The base prompt is immutable. Use `world_rollback` with an exact revision when a refinement regresses the working state. + +For hard scientific tasks, use `coalition_start` once to select and persist a contract-bound execution topology. The policy only adds coordination when the inferred decomposability, uncertainty, novelty, tool intensity, and verification risk justify its overhead. It can select a direct solution, centralized review, bounded fork/join, pairwise tournament, or an evolutionary generate-cluster-reflect-rank loop. After every restart use `coalition_status`. For each returned `ready` unit, call the Task tool with `harness_work_id` set to its exact `id`, exactly its `agent`, and a prompt containing its `prompt` plus the bounded upstream `context`. When `resumeSessionID` is present, continue that exact Task session through `session_id`; otherwise create a fresh child session. Only server-assigned generation/evolution units in the same producer `lane` can resume. Critique, ranking, investigation, and verification always remain fresh, and one session can never cross lanes. Run no more than `maxWorkers` concurrently. The Task tool writes a content-addressed receipt for the exact work, child session, Task turn, agent, canonical prompt, full prompt, measured usage, tool-call counts, timestamps, and outcome. A caller cannot complete or fail pending work without that receipt, reuse one Task turn for multiple units, substitute another session, or overwrite measured usage. After Task returns, report the child session and observable artifacts with `coalition_complete`; report actual execution failure with `coalition_fail`. If a Task call errors before returning its session, use `coalition_status` and settle the matching `executed` entry by its recorded `workerSessionID`. Verification units must submit `verdict`, `verdict_confidence`, and evidence-backed `verdict_checks` together. Do not show a verifier another verifier's result or reuse a producer session. Consensus appears only after the entire blinded panel settles: one verdict is insufficient and a split or abstaining panel is disputed. Adaptive evolution can return `awaiting_checkpoint` with no ready work; the hidden external evaluator must then authenticate the next utility checkpoint through the harness API. Workers cannot submit utility or unlock their own next round. Task receipts, producer memory, coalition outputs, and consensus are provisional search context and never count as benchmark evidence. + +Use `start` once to open the persisted candidate graph with the contract's budget. Use `status` after restarts and immediately before proposing work. The backend owns a deterministic ring of quality-diversity islands and issues a content-addressed recommendation lease bound to the exact search revision, strategy, lineage, target island, generation mode, and verified trajectory context. `recommendationContext` supplies the exact selected candidates with their artifacts, evaluator metrics, and feedback even when they fall outside the recent-candidate window. For serial work, pass the recommendation `id` as `recommendation_id` and preserve all of its `parentIDs` and `inspirationIDs`; a stale lease or changed lineage is rejected transactionally. When candidate generation is genuinely parallelizable, call `dispatch` once with the desired bounded `count`. Each returned ready reservation owns one candidate-budget slot, a server-selected lease, and a distinct content-addressed `mandate`. Give one reservation to one producer, include its mandate as the producer's primary variation mechanism, and let that producer plan, inspect, test, debug, and revise autonomously inside the mandate before calling `propose` with `reservation_id` and the reservation's exact lineage. The backend distributes verified branch or Pareto lineages across siblings before route reuse when alternatives exist. A mandate is guidance for search diversity, never proof of compliance or fitness. Reservations may finish out of order, but each can admit only one new artifact. Call `release` when a producer fails or is cancelled so its capacity returns; rediscovered bytes release their reservation automatically. Never use both admission IDs on one proposal. `single-pass` requests a direct seed, `stepwise` requests plan/review/implementation for independent or structural moves, and `diff` requests a focused edit against the supplied verified context. An empty `parentIDs` list opens an independent root in `targetIsland`; `exploit` refines the verified best; `fuse` combines two verified lineages; `migrate` mutates a verified target-island parent using a distinct verified source-island inspiration; and `diverge` changes the search strategy or paradigm, not just parameters. Migration requires a newly hashed artifact and a fresh external evaluation. Use `propose` to register the exact content-addressed artifact, lineage, and recommendation inspirations. The same artifact SHA is globally idempotent within a run even when proposal text, URI, or lineage changes, so retrying known bytes does not consume another candidate slot. Use `observe` for provisional feedback only: observations are explicitly unverified and can never become the best candidate, authorize descendants or inspirations, or enter recommendation context or retrospective memory. Reservations, mandates, and producer completion are also provisional; only the bound external evaluator can promote a candidate. If a verified result includes evaluator-owned evolution diagnostics, use its reintroduction and cycle fields to choose a genuinely different next move, but never treat low novelty as failure or novelty as fitness. If the contract requires controlled interventions, the evaluator must freeze replay, retuning/ablation/repair, and transfer pairs after tracing and before final evaluation; those effects qualify stability or mechanism claims and never influence recommendations or fitness. Official verification, tracing, and intervention execution are intentionally not actions in this tool. Use `hindsight` to retrieve bounded evaluator-linked successes and failures for this exact benchmark task. + +Keep the direct ReAct path for ordinary tasks. Do not start a candidate search without an optimize contract, a metric, and a finite contract budget. + +When the contract includes replicated evaluation, treat every candidate result as provisional until the evaluator executes the complete frozen stratum-by-independent-cluster grid under the committed environment and records a backend-derived receipt. Final promotion requires the receipt's robust aggregate and conservative confidence bound; a best replicate, cherry-picked subset, point estimate, or agent-authored replication claim cannot authorize success. The evaluator capability and official receipt operation are intentionally unavailable through this tool. + +When the contract includes human-AI autonomy tracing, assume every benchmark, human, and agent interaction and artifact edit is recorded by the evaluator runtime from the frozen run boundary. Do not request unrecorded side-channel help or describe the coarse `intervention` label as proof of autonomy. The backend derives essentially-autonomous, collaborative, or primarily-human provenance from the complete hash-chained trace, treats unclear classifications as inconclusive, and requires the last artifact transition to match the evaluated candidate. The evaluator capability, raw prompts, and receipt operation are intentionally unavailable through this tool. + +When the contract includes formal proof validation, preserve the exact trusted Lean challenge, declaration, module, proof relation, toolchain, dependency closure, and verifier tier. Compiler success is not sufficient: the evaluator separately audits transitive axioms including axiom types, performs any frozen fresh replay or sandboxed external cross-check, and rejects a repair or refutation presented as an exact proof. Formal verification establishes only the frozen statement; it does not by itself establish that the statement faithfully expresses the intended informal mathematics. The checker artifacts, transcripts, sandbox, evaluator capability, and receipt operation are intentionally unavailable through this tool. diff --git a/backend/cli/src/tool/learn.ts b/backend/cli/src/tool/learn.ts index 29c9c582..b4153f01 100644 --- a/backend/cli/src/tool/learn.ts +++ b/backend/cli/src/tool/learn.ts @@ -1,42 +1,39 @@ -import path from "path" -import fs from "fs/promises" import z from "zod" import { Tool } from "./tool" -import { Global } from "@/global" -import { RSILifecycle } from "@/session/rsi/lifecycle" -import { Log } from "@/util/log" - -const log = Log.create({ service: "tool.learn" }) +import { HarnessContract } from "@/session/harness/contract" +import { HarnessSkill } from "@/session/harness/skill" export const LearnTool = Tool.define("learn", { description: - "Save a private local skill distilled from conversation analysis and register it for lifecycle tracking. Called as the final step of /learn analysis.", + "Quarantine a learned skill proposal distilled from conversation analysis. The proposal stays inactive until independent held-out and trigger evaluations qualify it.", parameters: z.object({ name: z.string().describe("Skill identifier (kebab-case, e.g. 'debug-oom-pytorch')"), description: z.string().describe("One-line description of what this skill teaches"), content: z.string().describe("Full SKILL.md content including frontmatter"), }), - async execute(params) { - const dir = path.join(Global.Path.data, "learned-skills", params.name) - const filepath = path.join(dir, "SKILL.md") - - await fs.mkdir(dir, { recursive: true }) - await Bun.write(filepath, params.content) - log.info("learned skill written", { name: params.name, path: filepath }) - - await RSILifecycle.registerSkill(params.name).catch(() => {}) + async execute(params, ctx) { + const contract = await HarnessContract.read(ctx.sessionID) + const proposal = await HarnessSkill.propose({ + name: params.name, + description: params.description, + content: params.content, + origin: "conversation", + sessionID: ctx.sessionID, + runID: contract?.runID, + createdAt: Date.now(), + }) return { - title: `Learned skill: ${params.name}`, + title: `Skill proposal: ${params.name}`, output: [ - `Learned skill "${params.name}" saved successfully.`, - ` Path: ${filepath}`, - " Storage: private to this OpenScience installation", - ` Description: ${params.description}`, + `Learned skill proposal "${params.name}" is quarantined and inactive.`, + ` SHA-256: ${proposal?.contentSHA256}`, + ` Required: 3 distinct held-out tasks, 2 strict paired improvements, no regressions,`, + ` and held-out trigger precision/recall of at least 0.8.`, "", - "The skill will be available in future sessions via the skill tool.", + "Only the evaluator-authenticated harness can qualify it for explicit promotion.", ].join("\n"), - metadata: { name: params.name, local: true }, + metadata: { name: params.name, status: proposal?.status, sha256: proposal?.contentSHA256 }, } }, }) diff --git a/backend/cli/src/tool/registry.ts b/backend/cli/src/tool/registry.ts index 4f71c103..32f150d2 100644 --- a/backend/cli/src/tool/registry.ts +++ b/backend/cli/src/tool/registry.ts @@ -40,6 +40,9 @@ import { AtlasRecordTool } from "./atlas-record" import { ArtifactSnapshotTool } from "./artifact-snapshot" import { ModalTool } from "./modal" import { ComputeJobTool } from "./compute-job" +import { HarnessTool } from "./harness" +import { ClaimTool } from "./claim" +import { MemoryTool } from "./memory" export namespace ToolRegistry { const log = Log.create({ service: "tool.registry" }) @@ -143,14 +146,24 @@ export namespace ToolRegistry { LearnTool, ModalTool, ComputeJobTool, + MemoryTool, + HarnessTool, + ClaimTool, ...custom, ] } const ARTIFACT_TOOL_ID = "artifact" - const ARTIFACT_AGENTS = ["research", "biology", "ml"] + const ARTIFACT_AGENTS = ["research", "biology", "physics", "ml"] const MODAL_AGENTS = ["research", "biology", "physics", "ml"] + // Memory tool: only user-facing primary agents may read/write persistent + // memory; subagents (title, compaction, explore, ...) cannot. Plan mode is + // excluded because PlanMode.enforce blocks all mutating tools there anyway. + const MEMORY_TOOL_ID = "memory" + const MEMORY_AGENTS = ["research", "biology", "physics", "ml"] + const HARNESS_TOOL_ID = "harness" + const CLAIM_TOOL_ID = "claim" export async function ids() { return all().then((x) => x.map((t) => t.id)) @@ -181,6 +194,14 @@ export namespace ToolRegistry { return !!agent?.name && MODAL_AGENTS.includes(agent.name) } + if (t.id === HARNESS_TOOL_ID) { + return !!agent?.name && MEMORY_AGENTS.includes(agent.name) + } + + if (t.id === CLAIM_TOOL_ID) { + return !!agent?.name && MEMORY_AGENTS.includes(agent.name) + } + // Enable websearch/codesearch for zen users OR via enable flag if (t.id === "codesearch" || t.id === "websearch") { return model.providerID === "synsci" || Flag.OPENSCIENCE_ENABLE_EXA diff --git a/backend/cli/src/tool/task.ts b/backend/cli/src/tool/task.ts index 1de91f7c..57b78700 100644 --- a/backend/cli/src/tool/task.ts +++ b/backend/cli/src/tool/task.ts @@ -13,8 +13,9 @@ import { Config } from "../config/config" import { PermissionNext } from "@/permission/next" import { RLMState } from "../session/rlm/state" import { HierarchicalSemaphore } from "../util/semaphore" +import { HarnessOrchestrator } from "../session/harness/orchestrator" -const ARTIFACT_AGENTS = ["research", "biology", "ml"] +const ARTIFACT_AGENTS = ["research", "biology", "physics", "ml"] const COMPUTE_SUBAGENTS = new Set(["biology", "ml", "physics"]) export const MAX_CHILD_AGENTS = 2 const childSlots = new HierarchicalSemaphore(MAX_CHILD_AGENTS) @@ -23,11 +24,16 @@ const MAX_COMPUTE_SUBAGENTS = Number.isFinite(configuredComputeCap) && configuredComputeCap >= 1 ? Math.floor(configuredComputeCap) : 2 const computeSlots = new HierarchicalSemaphore(MAX_COMPUTE_SUBAGENTS) -const parameters = z.object({ +export const TaskParameters = z.object({ description: z.string().describe("A short (3-5 words) description of the task"), prompt: z.string().describe("The task for the agent to perform"), subagent_type: z.string().describe("The type of specialized agent to use for this task"), session_id: z.string().describe("Existing Task session to continue").optional(), + harness_work_id: z + .string() + .regex(/^[a-f0-9]{64}$/) + .describe("Ready coalition work identity this exact Task turn executes") + .optional(), command: z.string().describe("The command that triggered this task").optional(), }) @@ -48,8 +54,8 @@ export const TaskTool = Tool.define("task", async (ctx) => { ) return { description, - parameters, - async execute(params: z.infer, ctx) { + parameters: TaskParameters, + async execute(params: z.infer, ctx) { const config = await Config.get() const started = Date.now() @@ -178,8 +184,9 @@ export const TaskTool = Tool.define("task", async (ctx) => { ctx.abort.addEventListener("abort", cancel) using _ = defer(() => ctx.abort.removeEventListener("abort", cancel)) const promptParts = await SessionPrompt.resolvePromptParts(params.prompt) + const history = new Set((await Session.messages({ sessionID: session.id })).map((item) => item.info.id)) - const result = await SessionPrompt.prompt({ + const attempt = await SessionPrompt.prompt({ messageID, sessionID: session.id, model: { @@ -194,12 +201,23 @@ export const TaskTool = Tool.define("task", async (ctx) => { ...Object.fromEntries((config.experimental?.primary_tools ?? []).map((t) => [t, false])), }, parts: promptParts, - }).finally(() => { - unsub() }) + .then( + (result) => ({ result }) as const, + (error: unknown) => ({ error }) as const, + ) + .finally(() => { + unsub() + }) const messages = await Session.messages({ sessionID: session.id }) - const summary = messages + const turnID = + messages.find((item) => item.info.role === "user" && !history.has(item.info.id))?.info.id ?? + ("result" in attempt && attempt.result.info.role === "assistant" + ? attempt.result.info.parentID + : `task-${ctx.sessionID}-${ctx.messageID}-${ctx.callID}`) + const turn = messages.filter((item) => item.info.role === "assistant" && !history.has(item.info.id)) + const summary = turn .filter((x) => x.info.role === "assistant") .flatMap((msg) => msg.parts.filter((part): part is MessageV2.ToolPart => part.type === "tool")) .map((part) => ({ @@ -210,12 +228,13 @@ export const TaskTool = Tool.define("task", async (ctx) => { title: part.state.status === "completed" ? part.state.title : undefined, }, })) - const usage = messages.reduce( + const usage = turn.reduce( (total, message) => { if (message.info.role !== "assistant") return total total.cost += message.info.cost total.tokens.input += message.info.tokens.input total.tokens.output += message.info.tokens.output + total.tokens.reasoning += message.info.tokens.reasoning total.tokens.cache.read += message.info.tokens.cache.read total.tokens.cache.write += message.info.tokens.cache.write return total @@ -225,10 +244,42 @@ export const TaskTool = Tool.define("task", async (ctx) => { tokens: { input: 0, output: 0, + reasoning: 0, cache: { read: 0, write: 0 }, }, }, ) + const completed = Date.now() + const failed = + "error" in attempt || (attempt.result.info.role === "assistant" && attempt.result.info.error !== undefined) + const receipt = params.harness_work_id + ? await HarnessOrchestrator.attest({ + sessionID: ctx.sessionID, + workID: params.harness_work_id, + workerSessionID: session.id, + turnID, + agent: HarnessOrchestrator.WorkerAgent.parse(agent.name), + prompt: params.prompt, + outcome: failed ? "failed" : "completed", + usage: { + steps: turn.length, + tokens: + usage.tokens.input + + usage.tokens.output + + usage.tokens.reasoning + + usage.tokens.cache.read + + usage.tokens.cache.write, + costUSD: usage.cost, + wallTimeMs: completed - started, + }, + toolCalls: summary.length, + failedToolCalls: summary.filter((part) => part.state.status === "error").length, + startedAt: started, + completedAt: completed, + }) + : undefined + if ("error" in attempt) throw attempt.error + const result = attempt.result const text = result.parts.findLast((x) => x.type === "text")?.text ?? "" const callingAgent = msg.info.agent @@ -266,6 +317,9 @@ export const TaskTool = Tool.define("task", async (ctx) => { toolCalls: summary.length, failedToolCalls: summary.filter((part) => part.state.status === "error").length, usage, + workerReceiptID: params.harness_work_id + ? receipt?.work[params.harness_work_id]?.workerReceipt?.id + : undefined, maxConcurrentChildren: MAX_CHILD_AGENTS, }, output, diff --git a/backend/cli/src/tool/task.txt b/backend/cli/src/tool/task.txt index 0e80664f..99aec550 100644 --- a/backend/cli/src/tool/task.txt +++ b/backend/cli/src/tool/task.txt @@ -24,3 +24,4 @@ Rules: 4. Each invocation is stateless unless you provide a session_id. Give it a bounded objective, relevant context, allowed actions, and an exact return contract. 5. Treat child output as evidence to inspect, not authority to trust blindly. 6. The child result is not directly visible to the user. Report only the useful merged outcome and material limitations. +7. For harness coalition work, pass the exact ready unit as `harness_work_id`, use its required agent and canonical prompt, and resume only its declared `session_id`. This binds the actual Task turn and measured usage before coalition settlement. diff --git a/backend/cli/test/server/harness.test.ts b/backend/cli/test/server/harness.test.ts new file mode 100644 index 00000000..55a3c3f3 --- /dev/null +++ b/backend/cli/test/server/harness.test.ts @@ -0,0 +1,368 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessOrchestrator } from "../../src/session/harness/orchestrator" +import { HarnessRoutes } from "../../src/server/routes/harness" + +const sessionID = "route-harness-adapter" +const token = "route-evaluator-capability-token-000000000000000000" +const skill = "route-harness-skill" +const receipts = new Set() +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + ["bindings", "contracts", "evaluations", "orchestration", "worlds"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ) + await fs.rm(path.join(Global.Path.data, "harness", "audits", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "audit-receipts", `${receiptID}.json`), { force: true }), + ), + ) + receipts.clear() + await Promise.all( + ["learned-skill-proposals", "learned-skills"].map((name) => + fs.rm(path.join(Global.Path.data, name, skill), { recursive: true, force: true }), + ), + ) +}) + +describe("/harness routes", () => { + test("binds a generic run and ingests an authenticated result", async () => { + const app = HarnessRoutes() + const bound = await app.request("/runs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + schemaVersion: 1, + runID: "route-run", + sessionID, + benchmark: "local-statistics-suite", + title: "Local statistics evaluation", + family: "data", + task: "Run and verify a chi-square analysis", + version: "1", + taskID: "chi-square-1", + split: "validation", + evaluator: { name: "route-evaluator", version: "1", source: "external", token }, + objective: "Run and verify a chi-square analysis", + audit: { mode: "hybrid", budget: 2, minSamples: 2 }, + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + packs: ["statistics"], + budget: { steps: 10 }, + seed: 4, + intervention: "autonomous", + contamination: { policy: "hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }), + }) + expect(bound.status).toBe(200) + const contract = (await bound.json()) as { + runID: string + packs: Array<"statistics"> + benchmark: { name: string } + } + expect(contract).toMatchObject({ + packs: ["statistics"], + benchmark: { + name: "local-statistics-suite", + title: "Local statistics evaluation", + family: "data", + }, + }) + expect(JSON.stringify(contract)).not.toContain(token) + + const started = await app.request(`/runs/${sessionID}/orchestration`, { method: "POST" }) + expect(started.status).toBe(200) + expect(await started.json()).toMatchObject({ protocolVersion: "coalition-v1", revision: 0, status: "active" }) + const orchestration = await app.request(`/runs/${sessionID}/orchestration`) + expect(orchestration.status).toBe(200) + expect(await orchestration.json()).toMatchObject({ protocolVersion: "coalition-v1", revision: 0 }) + + const audit = await app.request("/audits", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID, + evaluatorToken: token, + subject: { type: "run", id: contract.runID, artifactSHA256: hash("route-artifact") }, + probes: [ + { + id: "probe-a", + commitment: hash("hidden-a"), + features: [0], + stratum: "a", + weight: 1, + priorLoss: 0.5, + }, + { + id: "probe-b", + commitment: hash("hidden-b"), + features: [1], + stratum: "b", + weight: 1, + priorLoss: 0.5, + }, + ], + }), + }) + expect(audit.status).toBe(200) + const auditState = (await audit.json()) as { auditID: string } + const selected = await app.request(`/audits/${auditState.auditID}/selection`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID, evaluatorToken: token }), + }) + expect(selected.status).toBe(200) + const probe = (await selected.json()) as { probeID: string } + const observed = await app.request(`/audits/${auditState.auditID}/observations`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID, + evaluatorToken: token, + probeID: probe.probeID, + loss: 0.2, + failure: false, + evidence: ["route:probe-receipt"], + }), + }) + expect(observed.status).toBe(200) + expect(await observed.json()).toMatchObject({ estimate: { observed: 1 }, revision: 2 }) + const selectedAgain = await app.request(`/audits/${auditState.auditID}/selection`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID, evaluatorToken: token }), + }) + const second = (await selectedAgain.json()) as { probeID: string } + const completed = await app.request(`/audits/${auditState.auditID}/observations`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID, + evaluatorToken: token, + probeID: second.probeID, + loss: 0.3, + failure: false, + evidence: ["route:second-probe-receipt"], + }), + }) + expect(await completed.json()).toMatchObject({ status: "completed", estimate: { observed: 2 } }) + const sealed = await app.request(`/audits/${auditState.auditID}/receipt`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID, evaluatorToken: token }), + }) + expect(sealed.status).toBe(200) + const receipt = (await sealed.json()) as { receiptID: string; qualified: boolean } + receipts.add(receipt.receiptID) + expect(receipt.qualified).toBe(false) + + const checks = HarnessDomain.compose(contract.packs).map((check) => ({ + id: check.id, + status: "passed", + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + const evaluated = await app.request("/evaluations", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + schemaVersion: 1, + runID: "route-run", + sessionID, + evaluatorToken: token, + status: "passed", + score: 1, + metrics: { score: 1 }, + checks, + evidence: ["local:evaluation-receipt"], + evaluatedAt: Date.now(), + }), + }) + expect(evaluated.status).toBe(200) + expect(await evaluated.json()).toMatchObject({ evaluation: { status: "passed", score: 1 } }) + + const world = await app.request(`/runs/${sessionID}/world`) + expect(world.status).toBe(200) + expect(await world.json()).toMatchObject({ + revision: 1, + contextEpoch: 1, + refinement: { recommended: true, trigger: "milestone" }, + }) + const rejected = await app.request(`/runs/${sessionID}/world/refinements`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + evaluatorToken: "wrong-evaluator-capability-token-000000000000000", + expectedRevision: 1, + reason: "milestone", + patches: [{ op: "remove", key: "missing" }], + }), + }) + expect(rejected.status).not.toBe(200) + const refined = await app.request(`/runs/${sessionID}/world/refinements`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + evaluatorToken: token, + expectedRevision: 1, + reason: "milestone", + patches: [ + { + op: "upsert", + key: "verified-result", + kind: "observation", + content: "The external evaluation passed every declared check", + confidence: 5, + evidenceRefs: ["local:evaluation-receipt", "local:check-journal"], + }, + ], + }), + }) + expect(refined.status).toBe(200) + expect(await refined.json()).toMatchObject({ + revision: 2, + contextEpoch: 2, + entries: { "verified-result": { confidence: 5 } }, + }) + + const stored = await app.request(`/runs/${sessionID}/evaluations`) + expect(stored.status).toBe(200) + expect((await stored.json()) as unknown[]).toHaveLength(1) + const read = await app.request(`/runs/${sessionID}/contract`) + expect(await read.json()).toMatchObject({ runID: "route-run" }) + }) + + test("creates and lists an inactive learned skill proposal", async () => { + const app = HarnessRoutes() + const description = "Use when a held-out route workflow has been qualified." + const proposed = await app.request("/skills", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: skill, + description, + content: `---\nname: ${skill}\ndescription: ${description}\n---\n\n# Route workflow\n`, + origin: "conversation", + }), + }) + expect(proposed.status).toBe(200) + expect(await proposed.json()).toMatchObject({ name: skill, status: "pending" }) + + const listed = await app.request("/skills") + expect(listed.status).toBe(200) + expect(await listed.json()).toContainEqual(expect.objectContaining({ name: skill, status: "pending" })) + }) + + test("authenticates external marginal-utility checkpoints before unlocking evolution", async () => { + const app = HarnessRoutes() + const bound = await app.request("/runs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + schemaVersion: 1, + runID: "route-adaptive-run", + sessionID, + benchmark: "local-statistics-suite", + title: "Local adaptive evaluation", + family: "data", + task: "Evolve a robust statistical method", + version: "1", + taskID: "adaptive-1", + split: "validation", + evaluator: { name: "route-evaluator", version: "1", source: "external", token }, + objective: "Evolve a robust statistical method", + orchestration: { + topology: "evolution", + maxWorkers: 2, + maxRounds: 2, + minIndependentVerifiers: 2, + adaptive: { + protocolVersion: "marginal-utility-v1", + minRounds: 1, + patience: 1, + minUtilityGain: 0.05, + maxUncertainty: 0.05, + }, + }, + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + packs: ["statistics"], + budget: { steps: 100 }, + seed: 4, + intervention: "autonomous", + contamination: { policy: "hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }), + }) + expect(bound.status).toBe(200) + const initial = await HarnessOrchestrator.initialize(sessionID) + const advance = async (state: HarnessOrchestrator.State): Promise => { + if (state.status === "awaiting_checkpoint") return state + const work = HarnessOrchestrator.ready(state)[0]! + const worker = work.resumeSessionID ?? `route-worker-${state.revision}` + const completedAt = Date.now() + await HarnessOrchestrator.attest({ + sessionID, + workID: work.id, + workerSessionID: worker, + turnID: `route-task-turn-${state.revision}`, + agent: work.agent, + prompt: `Execute:\n${work.prompt}`, + outcome: "completed", + usage: { steps: 1 }, + toolCalls: 1, + failedToolCalls: 0, + startedAt: Math.max(state.createdAt, completedAt - 1), + completedAt, + }) + const next = await HarnessOrchestrator.complete({ + sessionID, + workID: work.id, + workerSessionID: worker, + result: { + summary: work.label, + artifactRefs: [`artifact://${work.label}`], + evidenceRefs: [`evidence://${work.label}`], + usage: { steps: 1 }, + }, + }) + return advance(next) + } + const waiting = await advance(initial) + expect(waiting.status).toBe("awaiting_checkpoint") + + const request = (evaluatorToken: string) => + app.request(`/runs/${sessionID}/orchestration/checkpoints`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + evaluatorToken, + round: 1, + utility: 0.5, + uncertainty: 0.01, + evidenceRefs: ["evidence://local-round-1"], + evaluatedAt: Date.now(), + }), + }) + expect((await request("wrong-evaluator-capability-token-0000000000000000000")).status).not.toBe(200) + const checkpoint = await request(token) + expect(checkpoint.status).toBe(200) + const state = await checkpoint.json() + expect(state).toMatchObject({ + status: "active", + adaptive: { phase: "searching", checkpoints: [{ round: 1, qualified: true }] }, + }) + expect(JSON.stringify(state)).not.toContain(token) + }) +}) diff --git a/backend/cli/test/session/harness-ablation.test.ts b/backend/cli/test/session/harness-ablation.test.ts new file mode 100644 index 00000000..21e2f748 --- /dev/null +++ b/backend/cli/test/session/harness-ablation.test.ts @@ -0,0 +1,561 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { Global } from "../../src/global" +import { HarnessAblation } from "../../src/session/harness/ablation" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" + +const sessions = new Set() +const plans = new Set() +const token = "ablation-evaluator-capability-token-0000000000000000" +const auditor = "ablation-auditor-capability-token-00000000000000000" +const reviewer = "ablation-semantic-capability-token-0000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await Promise.all( + [...plans].map((planID) => + fs.rm(path.join(Global.Path.data, "harness", "ablations", `${planID}.json`), { force: true }), + ), + ) + sessions.clear() + plans.clear() +}) + +async function run(input: { + prefix: string + seed: number + role: "baseline" | "arm" + createdAt: number + model?: string + direction?: "maximize" | "minimize" + factor?: + | "orchestration" + | "search" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" +}) { + const sessionID = `${input.prefix}-${input.seed}-${input.role}` + sessions.add(sessionID) + const contract = await HarnessAdapter.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "matched-ablation-task", + split: "held_out", + evaluator: { name: "official-ablation-evaluator", version: "3", source: "benchmark", token }, + objective: "Measure the isolated effect of conditional orchestration", + profile: input.factor === "search" ? "optimize" : undefined, + search: input.factor === "search" ? (input.role === "arm" ? "adaptive" : "static") : undefined, + orchestration: + input.role === "arm" && input.factor === "orchestration" + ? { topology: "solo", maxWorkers: 1, maxRounds: 1, minIndependentVerifiers: 1 } + : undefined, + evaluatorAudit: + input.factor === "synthesis" || (input.role === "arm" && input.factor === "evaluator_audit") + ? { + token: auditor, + protocol: { + protocolVersion: "evaluator-audit-v1", + auditor: { name: "ablation-meta-evaluator", version: "1", source: "external" }, + suite: { name: "ablation-suite", version: "1", commitmentSHA256: hash("ablation-suite") }, + minCleanCases: 2, + minCasesPerFault: 1, + requiredFaults: + input.factor === "synthesis" ? ["wrong_answer", "unsupported_claim", "data_leakage"] : ["wrong_answer"], + minSensitivity: 0.8, + minSpecificity: 0.8, + minBalancedAccuracy: 0.8, + minFaultRecall: 0.8, + maxBrierScore: 0.15, + }, + } + : undefined, + synthesis: + input.role === "arm" && input.factor === "synthesis" + ? { + protocolVersion: "scientific-synthesis-v1", + querySHA256: hash("ablation-synthesis-query"), + referenceSHA256: hash("ablation-synthesis-reference"), + referenceFactsSHA256: hash("ablation-synthesis-facts"), + referenceFactCount: 2, + cutoff: "2026-01-01", + tools: ["paper_search"], + traceSchemaSHA256: hash("ablation-synthesis-trace"), + filterPolicySHA256: hash("ablation-synthesis-filter"), + maxToolEvents: 20, + decomposer: { + name: "ablation-decomposer", + version: "1", + promptSHA256: hash("ablation-decomposer-prompt"), + configSHA256: hash("ablation-decomposer-config"), + }, + judges: { + precision: { + name: "ablation-precision", + version: "1", + promptSHA256: hash("ablation-precision-prompt"), + configSHA256: hash("ablation-precision-config"), + }, + recall: { + name: "ablation-recall", + version: "1", + promptSHA256: hash("ablation-recall-prompt"), + configSHA256: hash("ablation-recall-config"), + }, + }, + minGeneratedFacts: 2, + minPrecision: 0.4, + minRecall: 0.4, + minF1: 0.4, + cleanRoomRequired: true, + judgeFailurePolicy: "inconclusive", + } + : undefined, + autonomy: + input.role === "arm" && input.factor === "autonomy" + ? { + protocolVersion: "human-ai-autonomy-v1", + claimedLevel: "essentially_autonomous", + recorder: { + name: "ablation-interaction-recorder", + version: "1", + artifactSHA256: hash("ablation-autonomy-recorder"), + source: "evaluator_runtime", + }, + traceSchemaSHA256: hash("ablation-autonomy-trace"), + classificationPolicySHA256: hash("ablation-autonomy-policy"), + maxEvents: 32, + rawRetention: "required", + disclosure: "evaluator_retained", + completeTraceRequired: true, + uncertaintyPolicy: "inconclusive", + } + : undefined, + formalProof: + input.role === "arm" && input.factor === "formal_proof" + ? { + protocolVersion: "formal-proof-v1", + language: "lean4", + tier: "kernel", + relation: "exact_proof", + challengeSHA256: hash("ablation-formal-challenge"), + statementSHA256: hash("ablation-formal-statement"), + declaration: "Ablation.formal", + module: "Ablation.Formal", + leanVersion: "4.33.0", + leanToolchainSHA256: hash("ablation-lean-toolchain"), + lakeManifestSHA256: hash("ablation-lake-manifest"), + dependencyTreeSHA256: hash("ablation-dependency-tree"), + verifiers: [ + { + role: "lean_kernel", + name: "ablation-lean-kernel", + version: "1", + artifactSHA256: hash("ablation-lean-kernel"), + }, + { + role: "source_auditor", + name: "ablation-source-auditor", + version: "1", + artifactSHA256: hash("ablation-source-auditor"), + }, + { + role: "axiom_auditor", + name: "ablation-axiom-auditor", + version: "1", + artifactSHA256: hash("ablation-axiom-auditor"), + }, + ], + forbiddenConstructs: ["sorry", "admit", "debug.skipKernelTC", "native_decide"], + allowedAxioms: ["Classical.choice", "Quot.sound", "propext"].toSorted((a, b) => a.localeCompare(b)), + maxFiles: 32, + completeManifestRequired: true, + warningPolicy: "fail", + semanticPolicy: "formal_statement_only", + } + : undefined, + semanticAudit: + input.role === "arm" && input.factor === "semantic_audit" + ? { + token: reviewer, + protocol: { + protocolVersion: "semantic-audit-v1", + reviewer: { name: "ablation-meaning-review", version: "1", source: "external" }, + scope: { + objectiveSHA256: hash("Measure the isolated effect of conditional orchestration"), + criteria: [{ id: "intent", requirement: "Answer the intended scientific problem" }], + forbiddenShortcuts: [{ id: "vacuity", description: "Do not satisfy only a vacuous interpretation" }], + literature: { cutoff: "2026-08-01", corpusSHA256: hash("ablation-literature") }, + noveltyFloor: "not_required", + }, + minReviewers: 2, + minConfidence: 0.8, + }, + } + : undefined, + replication: + input.role === "arm" && input.factor === "replication" + ? { + protocolVersion: "replicated-evaluation-v1", + validatorSHA256: hash("ablation-replication-validator"), + environmentSHA256: hash("ablation-replication-environment"), + sampling: { + design: "crossed-stratified-cluster-v1", + stratumKind: "task", + clusterKind: "seed", + strata: [{ id: "task-0", commitmentSHA256: hash("ablation-task-0") }], + clusters: [0, 1, 2, 3, 4].map((seed) => ({ + id: `seed-${seed}`, + commitmentSHA256: hash(`ablation-seed-${seed}`), + })), + }, + estimator: "iqm", + interval: { + method: "stratified-bootstrap-percentile-v1", + confidence: 0.95, + resamples: 1_000, + seed: 91, + }, + decision: { + rule: "conservative-bound-v1", + direction: input.direction ?? "maximize", + target: 0.5, + }, + failurePolicy: "fail-closed", + } + : undefined, + metric: { + name: input.factor === "synthesis" ? "factual_f1" : "score", + direction: input.direction ?? "maximize", + target: input.factor === "replication" ? 0.5 : input.factor === "synthesis" ? 0.4 : undefined, + }, + model: { provider: "test", name: input.model ?? "model" }, + tools: input.factor === "synthesis" ? ["read", "bash", "paper_search"] : ["read", "bash"], + skills: [], + budget: { + steps: 30, + tokens: 20_000, + costUSD: 2, + ...(input.factor === "search" ? { candidates: 8 } : {}), + }, + seed: input.seed, + intervention: "autonomous", + contamination: { + policy: "hidden outcomes remain evaluator-private", + hiddenTestsAccessible: false, + publicDataCutoff: input.factor === "synthesis" ? "2026-01-01" : undefined, + }, + createdAt: input.createdAt, + }) + return contract +} + +async function study( + prefix: string, + drift = false, + direction: "maximize" | "minimize" = "maximize", + factor: + | "orchestration" + | "search" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" = "orchestration", +) { + const createdAt = Date.now() + const pairs = await Promise.all( + [1, 2, 3].map(async (seed) => ({ + baseline: await run({ prefix, seed, role: "baseline", createdAt, direction, factor }), + arm: await run({ + prefix, + seed, + role: "arm", + createdAt, + direction, + factor, + model: drift && seed === 2 ? "different-model" : undefined, + }), + })), + ) + const plan = { + schemaVersion: 1 as const, + studyID: `${prefix}-study`, + factor: { kind: factor }, + minEffect: 0.05, + maxPairRegression: 0, + pairs: pairs.map((pair) => ({ + baseline: { sessionID: pair.baseline.sessionID, evaluatorToken: token }, + arm: { sessionID: pair.arm.sessionID, evaluatorToken: token }, + })), + } + const credentials = pairs.flatMap((pair) => [ + { sessionID: pair.baseline.sessionID, evaluatorToken: token }, + { sessionID: pair.arm.sessionID, evaluatorToken: token }, + ]) + return { pairs, plan, credentials } +} + +async function evaluate(contract: HarnessContract.Info, score: number, evaluatedAt: number) { + const checks = HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + return HarnessAdapter.ingest({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + status: "passed", + score, + metrics: { score }, + checks, + evidence: ["official:ablation-result.json"], + evaluatedAt, + }) +} + +async function fail(contract: HarnessContract.Info) { + const checks = HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "failed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + return HarnessAdapter.ingest({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + status: "failed", + metrics: {}, + checks, + evidence: ["official:ablation-failure.json"], + evaluatedAt: Date.now(), + }) +} + +describe("matched scientific ablations", () => { + test("freezes seed-paired contracts before evaluation and derives supported effects", async () => { + const input = await study("ablation-supported") + const app = HarnessRoutes() + const initialized = await app.request("/ablations", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(input.plan), + }) + expect(initialized.status).toBe(200) + const state = (await initialized.json()) as HarnessAblation.State + plans.add(state.plan.planID) + expect(state.plan.pairs.map((pair) => pair.seed)).toEqual([1, 2, 3]) + expect(state.plan.baselineValueSHA256).not.toBe(state.plan.armValueSHA256) + expect(JSON.stringify(state)).not.toContain(token) + + const effects = [0.1, 0.11, 0.09] + await Promise.all( + input.pairs.flatMap((pair, index) => [ + evaluate(pair.baseline, 0.7, Date.now()), + evaluate(pair.arm, 0.7 + effects[index]!, Date.now()), + ]), + ) + const assessed = await app.request(`/ablations/${state.plan.planID}/assessment`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ runs: input.credentials }), + }) + expect(assessed.status).toBe(200) + const result = (await assessed.json()) as HarnessAblation.State + expect(result.receipt).toMatchObject({ + verdict: "supported", + statistics: { pairs: 3, validPairs: 3, regressions: 0 }, + }) + expect(result.receipt?.statistics.meanEffect).toBeCloseTo(0.1) + expect(result.receipt?.statistics.confidence95?.[0]).toBeGreaterThan(0.05) + expect(result.receipt?.pairs.map((pair) => pair.effect)).toEqual([ + 0.09999999999999998, 0.10999999999999999, 0.08999999999999997, + ]) + }) + + test("rejects an arm with model drift outside the declared factor", async () => { + const input = await study("ablation-drift", true) + await expect(HarnessAblation.initialize(input.plan)).rejects.toThrow("differs outside the declared factor") + }) + + test("isolates evaluator qualification as its own ablatable protocol factor", async () => { + const input = await study("ablation-evaluator-audit", false, "maximize", "evaluator_audit") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + expect(initialized.plan.factor.kind).toBe("evaluator_audit") + expect(initialized.plan.baselineValueSHA256).not.toBe(initialized.plan.armValueSHA256) + }) + + test("isolates semantic meaning review as its own ablatable protocol factor", async () => { + const input = await study("ablation-semantic-audit", false, "maximize", "semantic_audit") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + expect(initialized.plan.factor.kind).toBe("semantic_audit") + expect(initialized.plan.baselineValueSHA256).not.toBe(initialized.plan.armValueSHA256) + }) + + test("isolates conservative replicated evaluation as its own ablatable protocol factor", async () => { + const input = await study("ablation-replication", false, "maximize", "replication") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + expect(initialized.plan.factor.kind).toBe("replication") + expect(initialized.plan.baselineValueSHA256).not.toBe(initialized.plan.armValueSHA256) + }) + + test("isolates clean-room scientific synthesis as its own ablatable protocol factor", async () => { + const input = await study("ablation-synthesis", false, "maximize", "synthesis") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + expect(initialized.plan.factor.kind).toBe("synthesis") + expect(initialized.plan.baselineValueSHA256).not.toBe(initialized.plan.armValueSHA256) + }) + + test("isolates human-AI autonomy tracing as its own ablatable protocol factor", async () => { + const input = await study("ablation-autonomy", false, "maximize", "autonomy") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + expect(initialized.plan.factor.kind).toBe("autonomy") + expect(initialized.plan.baselineValueSHA256).not.toBe(initialized.plan.armValueSHA256) + }) + + test("isolates formal proof verification as its own ablatable protocol factor", async () => { + const input = await study("ablation-formal-proof", false, "maximize", "formal_proof") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + expect(initialized.plan.factor.kind).toBe("formal_proof") + expect(initialized.plan.baselineValueSHA256).not.toBe(initialized.plan.armValueSHA256) + }) + + test("isolates the adaptive search controller from the static leased baseline", async () => { + const input = await study("ablation-search", false, "maximize", "search") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + expect(initialized.plan.factor.kind).toBe("search") + expect( + input.pairs.every((pair) => !pair.baseline.search && pair.arm.search?.protocolVersion === "adaptive-search-v1"), + ).toBe(true) + expect(initialized.plan.baselineValueSHA256).not.toBe(initialized.plan.armValueSHA256) + }) + + test("requires predeclaration before any paired outcome is visible", async () => { + const input = await study("ablation-late") + await evaluate(input.pairs[0]!.baseline, 0.7, Date.now()) + await expect(HarnessAblation.initialize(input.plan)).rejects.toThrow("before any paired evaluation") + }) + + test("rejects a precisely measured effect below the practical threshold", async () => { + const input = await study("ablation-null") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + await Promise.all( + input.pairs.flatMap((pair) => [evaluate(pair.baseline, 0.7, Date.now()), evaluate(pair.arm, 0.7, Date.now())]), + ) + const result = await HarnessAblation.assess(initialized.plan.planID, { + runs: input.credentials, + }) + expect(result?.receipt).toMatchObject({ + verdict: "rejected", + statistics: { meanEffect: 0, confidence95: [0, 0], minEffect: 0.05 }, + }) + }) + + test("reverses paired effects for a metric that is minimized", async () => { + const input = await study("ablation-minimize", false, "minimize") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + await Promise.all( + input.pairs.flatMap((pair, index) => [ + evaluate(pair.baseline, 1, Date.now()), + evaluate(pair.arm, 0.9 - index * 0.01, Date.now()), + ]), + ) + const result = await HarnessAblation.assess(initialized.plan.planID, { runs: input.credentials }) + expect(result?.receipt).toMatchObject({ verdict: "supported", statistics: { regressions: 0 } }) + expect(result?.receipt?.statistics.meanEffect).toBeCloseTo(0.11) + }) + + test("rejects an average gain when one seed exceeds the regression tolerance", async () => { + const input = await study("ablation-regression") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + const effects = [0.4, 0.4, -0.01] + await Promise.all( + input.pairs.flatMap((pair, index) => [ + evaluate(pair.baseline, 0.5, Date.now()), + evaluate(pair.arm, 0.5 + effects[index]!, Date.now()), + ]), + ) + const result = await HarnessAblation.assess(initialized.plan.planID, { runs: input.credentials }) + expect(result?.receipt).toMatchObject({ + verdict: "rejected", + statistics: { regressions: 1, maxPairRegression: 0 }, + }) + }) + + test("records failed evaluator outcomes as rejected rather than usable effects", async () => { + const input = await study("ablation-failure") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + await Promise.all( + input.pairs.flatMap((pair, index) => [ + evaluate(pair.baseline, 0.7, Date.now()), + index === 1 ? fail(pair.arm) : evaluate(pair.arm, 0.8, Date.now()), + ]), + ) + const result = await HarnessAblation.assess(initialized.plan.planID, { runs: input.credentials }) + expect(result?.receipt).toMatchObject({ + verdict: "rejected", + statistics: { pairs: 3, validPairs: 2 }, + }) + expect(result?.receipt?.pairs[1]).toMatchObject({ arm: { status: "failed" } }) + expect(result?.receipt?.pairs[1]?.effect).toBeUndefined() + }) + + test("fails closed when persisted plan content is corrupted", async () => { + const input = await study("ablation-tamper") + const initialized = await HarnessAblation.initialize(input.plan) + if (!initialized) throw new Error("Expected an initialized ablation") + plans.add(initialized.plan.planID) + const target = path.join(Global.Path.data, "harness", "ablations", `${initialized.plan.planID}.json`) + const state = JSON.parse(await Bun.file(target).text()) as HarnessAblation.State + await Bun.write(target, JSON.stringify({ ...state, plan: { ...state.plan, minEffect: 0 } })) + expect(await HarnessAblation.read(initialized.plan.planID)).toBeNull() + await expect(HarnessAblation.assess(initialized.plan.planID, { runs: input.credentials })).rejects.toThrow( + "Unknown ablation plan", + ) + }) +}) diff --git a/backend/cli/test/session/harness-adaptation.test.ts b/backend/cli/test/session/harness-adaptation.test.ts new file mode 100644 index 00000000..1c12ea4e --- /dev/null +++ b/backend/cli/test/session/harness-adaptation.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, test } from "bun:test" +import { HarnessAdaptation } from "../../src/session/harness/adaptation" +import { HarnessContract } from "../../src/session/harness/contract" + +const id = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +describe("verified adaptive search controller", () => { + test("implements decayed local improvement and globally normalized reward exactly", () => { + const events: HarnessAdaptation.Event[] = [ + { candidateID: id("a"), island: 0, revision: 1, status: "passed", score: 10 }, + { candidateID: id("b"), island: 1, revision: 2, status: "passed", score: 8 }, + { candidateID: id("c"), island: 0, revision: 3, status: "passed", score: 12 }, + ] + const improved = HarnessAdaptation.derive({ + policy: HarnessContract.adaptiveSearch, + direction: "maximize", + islands: 2, + events, + }) + expect(improved).toMatchObject({ events: 3, stalled: 0, selectedIsland: 1, globalStagnation: false }) + expect(improved.islands[0]).toMatchObject({ + visits: 2, + decayedVisits: 1.9, + improvements: 1, + accumulatedImprovement: 0.004, + decayedReward: 0.2, + }) + expect(improved.islands[0]!.rewardMean).toBeCloseTo(0.2 / 1.9, 12) + + const decayed = HarnessAdaptation.derive({ + policy: HarnessContract.adaptiveSearch, + direction: "maximize", + islands: 2, + events: [...events, { candidateID: id("d"), island: 0, revision: 4, status: "passed", score: 11 }], + }) + expect(decayed.islands[0]!.accumulatedImprovement).toBeCloseTo(0.0036, 12) + expect(decayed.islands[0]!.decayedReward).toBeCloseTo(0.18, 12) + expect(decayed.islands[0]!.decayedVisits).toBeCloseTo(2.71, 12) + expect(decayed.stalled).toBe(1) + }) + + test("handles minimizing negative metrics without inverting improvement", () => { + const result = HarnessAdaptation.derive({ + policy: HarnessContract.adaptiveSearch, + direction: "minimize", + islands: 1, + events: [ + { candidateID: id("negative-baseline"), island: 0, revision: 1, status: "passed", score: -10 }, + { candidateID: id("negative-improvement"), island: 0, revision: 2, status: "passed", score: -12 }, + ], + }) + expect(result.islands[0]).toMatchObject({ + visits: 2, + improvements: 1, + bestID: id("negative-improvement"), + bestFitness: 12, + accumulatedImprovement: 0.004, + decayedReward: 0.2, + }) + }) + + test("decays failed attempts, triggers measured stagnation, and remains deterministic", () => { + const events: HarnessAdaptation.Event[] = [ + { candidateID: id("seed"), island: 0, revision: 1, status: "passed", score: 1 }, + ...Array.from({ length: 5 }, (_, index) => ({ + candidateID: id(`failure-${index}`), + island: 0, + revision: index + 2, + status: "failed" as const, + })), + ] + const summary = HarnessAdaptation.derive({ + policy: HarnessContract.adaptiveSearch, + direction: "maximize", + islands: 1, + events, + }) + expect(summary).toMatchObject({ events: 6, stalled: 5, selectedIsland: 0, globalStagnation: true }) + const input = { + policy: HarnessContract.adaptiveSearch, + direction: "maximize" as const, + islands: 1, + events, + targetIsland: 0, + key: "run:session:revision", + } + expect(HarnessAdaptation.control(input)).toEqual(HarnessAdaptation.control(input)) + expect(HarnessAdaptation.control(input)).toMatchObject({ + eventCount: 6, + stalled: 5, + selectedIsland: 0, + targetIsland: 0, + globalStagnation: true, + }) + }) + + test("fails closed on duplicate candidate events and out-of-range islands", () => { + const event = { candidateID: id("duplicate"), island: 0, revision: 1, status: "passed" as const, score: 1 } + expect(() => + HarnessAdaptation.derive({ + policy: HarnessContract.adaptiveSearch, + direction: "maximize", + islands: 1, + events: [event, { ...event, revision: 2 }], + }), + ).toThrow("at most one final event") + expect(() => + HarnessAdaptation.derive({ + policy: HarnessContract.adaptiveSearch, + direction: "maximize", + islands: 1, + events: [{ ...event, island: 1 }], + }), + ).toThrow("unknown island") + }) +}) diff --git a/backend/cli/test/session/harness-adapter.test.ts b/backend/cli/test/session/harness-adapter.test.ts new file mode 100644 index 00000000..d1278d01 --- /dev/null +++ b/backend/cli/test/session/harness-adapter.test.ts @@ -0,0 +1,197 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" + +const sessions = new Set() +const token = "local-evaluator-capability-token-000000000000000000" + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "search", "reports"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + ), + ), + ) + sessions.clear() +}) + +function task(sessionID: string, overrides: Partial = {}): HarnessAdapter.Task { + sessions.add(sessionID) + return { + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "local-discovery-suite", + title: "Local discovery evaluation", + family: "biology", + task: "Generate and verify a testable biological hypothesis", + version: "1", + taskID: "case-1", + split: "validation", + evaluator: { name: "local-evaluator", version: "1", source: "external", token }, + objective: "Produce a supported and independently checkable discovery", + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + tools: ["read"], + skills: [], + packs: ["statistics", "biology"], + budget: { steps: 20 }, + seed: 7, + intervention: "autonomous", + contamination: { policy: "Evaluator state stays outside the agent process", hiddenTestsAccessible: false }, + createdAt: Date.now(), + ...overrides, + } +} + +function evaluation(contract: HarnessContract.Info): HarnessAdapter.Evaluation { + return { + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + status: "passed", + score: 0.8, + metrics: { score: 0.8 }, + checks: HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed", + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })), + evidence: ["local:evaluation.json"], + evaluatedAt: Date.now(), + } +} + +describe("generic harness adapter", () => { + test("binds an arbitrary caller-owned evaluation identity without a bundled catalog", async () => { + const contract = await HarnessAdapter.bind(task("adapter-generic")) + expect(contract).toMatchObject({ + profile: "react", + benchmark: { + name: "local-discovery-suite", + title: "Local discovery evaluation", + family: "biology", + task: "Generate and verify a testable biological hypothesis", + evaluator: "local-evaluator", + evaluatorSource: "external", + }, + packs: ["statistics", "biology"], + }) + expect(JSON.stringify(contract)).not.toContain(token) + }) + + test("uses neutral defaults and accepts caller-defined identifiers", async () => { + const contract = await HarnessAdapter.bind( + task("adapter-defaults", { + benchmark: "private-suite-v3", + title: undefined, + family: undefined, + task: undefined, + packs: [], + }), + ) + expect(contract.benchmark).toMatchObject({ + name: "private-suite-v3", + title: "private-suite-v3", + family: "custom", + task: "Produce a supported and independently checkable discovery", + }) + expect(contract.packs).toEqual([]) + }) + + test("enforces numeric budgets and routes search and simulation protocols", async () => { + await expect(HarnessAdapter.bind(task("adapter-metric", { metric: { direction: "maximize" } }))).rejects.toThrow( + "declare its metric name", + ) + await expect(HarnessAdapter.bind(task("adapter-budget", { profile: "optimize" }))).rejects.toThrow( + "candidate budget", + ) + await expect(HarnessAdapter.bind(task("adapter-search", { search: "adaptive" }))).rejects.toThrow( + "candidate budget", + ) + const search = await HarnessAdapter.bind( + task("adapter-search-routed", { search: "adaptive", budget: { steps: 20, candidates: 4 } }), + ) + expect(search).toMatchObject({ profile: "optimize", search: HarnessContract.adaptiveSearch }) + const simulation = await HarnessAdapter.bind( + task("adapter-simulation", { + packs: ["statistics"], + simulation: { + kind: "pde", + engine: { + name: "solver", + version: "1", + commandSHA256: "a".repeat(64), + configSHA256: "b".repeat(64), + }, + problemSHA256: "c".repeat(64), + reference: { kind: "analytic", identity: "closed-form", sha256: "d".repeat(64) }, + validation: { + errorNorm: "L2", + minLevels: 3, + expectedOrder: 2, + orderTolerance: 0.2, + maxResidual: 1e-6, + invariantTolerances: { mass: 1e-6 }, + requiredStressTests: ["timestep_sensitivity"], + }, + }, + }), + ) + expect(simulation.profile).toBe("numerical") + expect(simulation.packs).toEqual(["statistics", "physics", "pde"]) + }) + + test("enables adaptive search only for bounded numeric optimization", async () => { + const adaptive = await HarnessAdapter.bind( + task("adapter-adaptive", { profile: "optimize", budget: { steps: 20, candidates: 4 } }), + ) + expect(adaptive.search).toEqual(HarnessContract.adaptiveSearch) + + const fixed = await HarnessAdapter.bind( + task("adapter-static", { + profile: "optimize", + search: "static", + budget: { steps: 20, candidates: 4 }, + }), + ) + expect(fixed.search).toBeUndefined() + }) + + test("authenticates external evaluations and records their evidence", async () => { + const contract = await HarnessAdapter.bind(task("adapter-ingest")) + await expect(HarnessAdapter.ingest({ ...evaluation(contract), evaluatorToken: "x".repeat(48) })).rejects.toThrow( + "capability was rejected", + ) + expect(await HarnessEvaluation.list(contract.sessionID)).toEqual([]) + + const result = await HarnessAdapter.ingest(evaluation(contract)) + expect(result.evaluation).toMatchObject({ + runID: contract.runID, + score: 0.8, + evaluator: { name: "local-evaluator", source: "external" }, + evidence: ["local:evaluation.json"], + }) + }) + + test("keeps a bound evaluator capability immutable", async () => { + const input = task("adapter-immutable") + await HarnessAdapter.bind(input) + await expect( + HarnessAdapter.bind({ + ...input, + evaluator: { ...input.evaluator, token: "replacement-capability-token-0000000000000000" }, + }), + ).rejects.toThrow("immutable once bound") + }) +}) diff --git a/backend/cli/test/session/harness-audit.test.ts b/backend/cli/test/session/harness-audit.test.ts new file mode 100644 index 00000000..1c68bc33 --- /dev/null +++ b/backend/cli/test/session/harness-audit.test.ts @@ -0,0 +1,543 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessAudit } from "../../src/session/harness/audit" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const receipts = new Set() +const token = "active-audit-evaluator-capability-token-000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + fs.rm(path.join(Global.Path.data, "harness", "bindings", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "contracts", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "search", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "evaluations", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "audits", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ]), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "audit-receipts", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + receipts.clear() +}) + +function config(values: Partial = {}): HarnessContract.Audit { + return HarnessContract.Audit.parse({ + mode: "performance", + budget: 3, + minSamples: 2, + noiseVariance: 0.05, + lengthscale: 0.7, + beta: 0, + failureThreshold: 0.5, + tolerance: 0.01, + maxUncertainty: 0.05, + estimationWeight: 0.5, + diversityWeight: 0.3, + coverageWeight: 0.2, + ...values, + }) +} + +async function bind(sessionID: string, audit = config()) { + sessions.add(sessionID) + return HarnessAdapter.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "active-audit", + split: "validation", + evaluator: { name: "official-evaluator", version: "1", source: "benchmark", token }, + objective: "Estimate held-out loss and discover diverse failures", + audit, + metric: { name: "loss", direction: "minimize" }, + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { steps: 20 }, + seed: 7, + intervention: "autonomous", + contamination: { policy: "hidden probes remain external", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const line = Array.from( + { length: 5 }, + (_, index): HarnessAudit.Probe => ({ + id: `probe-${index}`, + commitment: hash(`hidden-probe-${index}`), + features: [index - 2], + stratum: index < 3 ? "left" : "right", + weight: 1, + priorLoss: 0.5, + }), +) + +const history = Array.from( + { length: 5 }, + (_, index): HarnessAudit.TransferProbe => ({ + id: `history-${index}`, + commitment: hash(`hidden-history-${index}`), + sourceLosses: [0.1 + index * 0.05, 0.2 + index * 0.05, 0.3 + index * 0.05], + stratum: index < 3 ? "left" : "right", + weight: 1, + }), +) + +function proactive(values: Partial = {}) { + return config({ + budget: 3, + minSamples: 2, + tolerance: 1, + maxUncertainty: 1, + transfer: { + protocolVersion: "score-history-prior-v1", + poolSHA256: hash(JSON.stringify(history)), + sourceManifestSHA256: hash( + JSON.stringify({ + sourceModels: ["source-a", "source-b", "source-c"], + scores: history.map((probe) => ({ id: probe.id, sourceLosses: probe.sourceLosses })), + }), + ), + selectionSHA256: hash("pca-gmm-selection"), + selectionMethod: "pca-gmm-profile-v1", + sourceModels: ["source-a", "source-b", "source-c"], + calibrationSamples: 2, + maxCalibrationMAE: 0.2, + }, + ...values, + }) +} + +function access(sessionID: string): HarnessAudit.Access { + return { sessionID, evaluatorToken: token } +} + +describe("active committed-probe audit", () => { + test("preserves legacy contract fingerprints unless proactive promotion is explicit", () => { + const legacy = { mode: "performance" as const, budget: 3, minSamples: 2 } + const parsed = HarnessContract.Audit.parse(legacy) + expect(parsed.promotionRequired).toBeUndefined() + expect(JSON.stringify(parsed)).not.toContain("promotionRequired") + expect(() => HarnessContract.Audit.parse({ ...legacy, promotionRequired: true })).toThrow("transfer-qualified") + expect(() => HarnessContract.Audit.parse({ ...proactive(), mode: "failure", promotionRequired: true })).toThrow( + "failure-only", + ) + }) + + test("binds an opaque pool to the evaluator capability and audited artifact", async () => { + const contract = await bind("audit-bind") + const input: HarnessAudit.Initialize = { + ...access(contract.sessionID), + subject: { type: "run", id: contract.runID, artifactSHA256: hash("frozen-run-artifact") }, + probes: line, + } + await expect(HarnessAudit.initialize({ ...input, evaluatorToken: "x".repeat(40) })).rejects.toThrow( + "capability was rejected", + ) + const state = await HarnessAudit.initialize(input) + expect(state).toMatchObject({ + protocolVersion: "active-audit-v1", + status: "active", + revision: 0, + estimate: { observed: 0, failures: 0, abstain: true }, + }) + expect(state.order).toEqual(line.map((probe) => probe.id).toSorted()) + expect(JSON.stringify(state)).not.toContain(token) + expect(JSON.stringify(state)).not.toContain("hidden-probe") + expect(await HarnessAudit.initialize(input)).toEqual(state) + await expect( + HarnessAudit.status(state.auditID, { ...access(contract.sessionID), evaluatorToken: "z".repeat(40) }), + ).rejects.toThrow("capability was rejected") + const file = path.join( + Global.Path.data, + "harness", + "audits", + encodeURIComponent(contract.sessionID), + `${encodeURIComponent(state.auditID)}.json`, + ) + const tampered = JSON.parse(await fs.readFile(file, "utf8")) + tampered.pool["probe-0"].features[0] = 999 + await fs.writeFile(file, JSON.stringify(tampered)) + await expect(HarnessAudit.status(state.auditID, access(contract.sessionID))).rejects.toThrow( + "failed its commitment", + ) + }) + + test("selects high-leverage probes, decreases uncertainty, and is restart-idempotent", async () => { + const contract = await bind("audit-performance") + const state = await HarnessAudit.initialize({ + ...access(contract.sessionID), + subject: { type: "run", id: contract.runID, artifactSHA256: hash("performance-artifact") }, + probes: line, + }) + const selected = await HarnessAudit.select(state.auditID, access(contract.sessionID)) + expect(selected.probeID).toBe("probe-2") + expect(selected.acquisition.varianceReduction).toBeGreaterThan(0) + expect(await HarnessAudit.select(state.auditID, access(contract.sessionID))).toEqual(selected) + + await expect( + HarnessAudit.observe(state.auditID, { + ...access(contract.sessionID), + probeID: "probe-0", + loss: 0, + failure: false, + evidence: ["receipt://unselected"], + }), + ).rejects.toThrow("selected before observation") + const observed = await HarnessAudit.observe(state.auditID, { + ...access(contract.sessionID), + probeID: selected.probeID, + loss: 0.4, + failure: false, + evidence: ["receipt://probe-2"], + }) + expect(observed.estimate.observed).toBe(1) + expect(observed.estimate.standardDeviation).toBeLessThan(state.estimate.standardDeviation) + expect(observed.estimate.abstain).toBe(true) + expect(observed.pool[selected.probeID]!.observation!.evaluatedAt).toBeGreaterThan(0) + expect(await HarnessAudit.status(state.auditID, access(contract.sessionID))).toEqual(observed) + }) + + test("uses failure diversity and coverage to leave a discovered neighborhood", async () => { + const audit = config({ mode: "failure", budget: 4, minSamples: 2, targetFailures: 2, lengthscale: 0.2 }) + const contract = await bind("audit-failure", audit) + const probes: HarnessAudit.Probe[] = [ + { id: "a-1", commitment: hash("a-1"), features: [0, 0], stratum: "a", weight: 1, priorLoss: 0.95 }, + { id: "a-2", commitment: hash("a-2"), features: [0.05, 0], stratum: "a", weight: 1, priorLoss: 0.9 }, + { id: "b-1", commitment: hash("b-1"), features: [10, 10], stratum: "b", weight: 1, priorLoss: 0.85 }, + { id: "b-2", commitment: hash("b-2"), features: [10.05, 10], stratum: "b", weight: 1, priorLoss: 0.8 }, + ] + const state = await HarnessAudit.initialize({ + ...access(contract.sessionID), + subject: { type: "run", id: contract.runID, artifactSHA256: hash("failure-artifact") }, + probes, + }) + const first = await HarnessAudit.select(state.auditID, access(contract.sessionID)) + expect(first.probeID).toBe("a-1") + await HarnessAudit.observe(state.auditID, { + ...access(contract.sessionID), + probeID: first.probeID, + loss: 1, + failure: true, + evidence: ["receipt://failure-a"], + }) + const second = await HarnessAudit.select(state.auditID, access(contract.sessionID)) + expect(second.probeID).toBe("b-1") + expect(second.acquisition.diversity).toBeGreaterThan(0.9) + const completed = await HarnessAudit.observe(state.auditID, { + ...access(contract.sessionID), + probeID: second.probeID, + loss: 0.9, + failure: true, + evidence: ["receipt://failure-b"], + }) + expect(completed).toMatchObject({ + status: "completed", + stopReason: "failure_target_reached", + estimate: { observed: 2, failures: 2, stratumCoverage: 1 }, + }) + }) + + test("rejects contradictory and mutable evaluator outcomes", async () => { + const contract = await bind("audit-outcome") + const state = await HarnessAudit.initialize({ + ...access(contract.sessionID), + subject: { type: "run", id: contract.runID, artifactSHA256: hash("outcome-artifact") }, + probes: line, + }) + const selected = await HarnessAudit.select(state.auditID, access(contract.sessionID)) + await expect( + HarnessAudit.observe(state.auditID, { + ...access(contract.sessionID), + probeID: selected.probeID, + loss: 0.9, + failure: false, + evidence: ["receipt://contradiction"], + }), + ).rejects.toThrow("threshold") + const input: HarnessAudit.Observe = { + ...access(contract.sessionID), + probeID: selected.probeID, + loss: 0.9, + failure: true, + evidence: ["receipt://failure"], + } + const observed = await HarnessAudit.observe(state.auditID, input) + const repeated = await HarnessAudit.observe(state.auditID, input) + expect(repeated).toEqual(observed) + await expect(HarnessAudit.observe(state.auditID, { ...input, loss: 0.8 })).rejects.toThrow("immutable") + }) + + test("binds candidate audits to the candidate's exact artifact hash", async () => { + const sessionID = "audit-candidate" + sessions.add(sessionID) + const contract = await HarnessAdapter.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "mle", + version: "2026.08", + taskID: "candidate-audit", + split: "validation", + evaluator: { name: "official-evaluator", version: "1", source: "benchmark", token }, + objective: "Optimize and actively audit a frozen candidate", + audit: config(), + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { steps: 20, candidates: 2 }, + seed: 7, + intervention: "autonomous", + contamination: { policy: "hidden probes remain external", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) + const search = await HarnessSearch.initialize({ sessionID }) + const recommendation = HarnessSearch.recommend(search) + const added = await HarnessSearch.add({ + sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "baseline", + proposal: "frozen baseline", + artifact: { uri: "artifact://candidate", sha256: hash("candidate-artifact") }, + }) + const input: HarnessAudit.Initialize = { + ...access(sessionID), + subject: { type: "candidate", id: added.id, artifactSHA256: hash("wrong-artifact") }, + probes: line, + } + await expect(HarnessAudit.initialize(input)).rejects.toThrow("does not match") + const state = await HarnessAudit.initialize({ + ...input, + subject: { ...input.subject, artifactSHA256: hash("candidate-artifact") }, + }) + expect(state.subject).toEqual({ + type: "candidate", + id: added.id, + artifactSHA256: hash("candidate-artifact"), + }) + expect(state.contractFingerprint).toBe(HarnessContract.fingerprint(contract)) + }) + + test("derives score-history priors and keeps calibration selection independent of outcomes", async () => { + const left = await bind( + "audit-transfer-left", + proactive({ transfer: { ...proactive().transfer!, maxCalibrationMAE: 1 } }), + ) + const right = await bind( + "audit-transfer-right", + proactive({ transfer: { ...proactive().transfer!, maxCalibrationMAE: 1 } }), + ) + const initialize = async (contract: HarnessContract.Info) => + HarnessAudit.initialize({ + ...access(contract.sessionID), + subject: { type: "run", id: contract.runID, artifactSHA256: hash("same-transfer-artifact") }, + probes: history, + }) + await expect( + HarnessAudit.initialize({ + ...access(left.sessionID), + subject: { type: "run", id: left.runID, artifactSHA256: hash("same-transfer-artifact") }, + probes: line, + }), + ).rejects.toThrow() + const leftState = await initialize(left) + const rightState = await initialize(right) + expect(leftState.protocolVersion).toBe("proactive-audit-v2") + expect(leftState.estimate.transfer).toMatchObject({ status: "calibrating", observed: 0, required: 2 }) + const entry = leftState.pool["history-0"]! + expect(entry.priorLoss).toBeCloseTo(0.2) + expect(entry.features.reduce((sum, value) => sum + value, 0)).toBeCloseTo(0) + expect(entry.sourceLosses).toEqual(history[0]!.sourceLosses) + await expect( + HarnessAudit.initialize({ + ...access(left.sessionID), + subject: leftState.subject, + probes: history.map((probe, index) => + index === 0 ? { ...probe, sourceLosses: [0, ...probe.sourceLosses.slice(1)] } : probe, + ), + }), + ).rejects.toThrow("frozen transfer-pool commitment") + + const leftFirst = await HarnessAudit.select(leftState.auditID, access(left.sessionID)) + const rightFirst = await HarnessAudit.select(rightState.auditID, access(right.sessionID)) + expect(leftFirst.probeID).toBe(rightFirst.probeID) + expect(leftFirst.phase).toBe("calibration") + await HarnessAudit.observe(leftState.auditID, { + ...access(left.sessionID), + probeID: leftFirst.probeID, + loss: 0, + failure: false, + evidence: ["receipt://left-calibration"], + }) + await HarnessAudit.observe(rightState.auditID, { + ...access(right.sessionID), + probeID: rightFirst.probeID, + loss: 1, + failure: true, + evidence: ["receipt://right-calibration"], + }) + const leftSecond = await HarnessAudit.select(leftState.auditID, access(left.sessionID)) + const rightSecond = await HarnessAudit.select(rightState.auditID, access(right.sessionID)) + expect(leftSecond.probeID).toBe(rightSecond.probeID) + expect(leftSecond.phase).toBe("calibration") + }) + + test("rejects negative transfer, permanently abstains, and seals an unqualified receipt", async () => { + const contract = await bind( + "audit-transfer-reject", + proactive({ transfer: { ...proactive().transfer!, maxCalibrationMAE: 0.01 } }), + ) + const state = await HarnessAudit.initialize({ + ...access(contract.sessionID), + subject: { type: "run", id: contract.runID, artifactSHA256: hash("rejected-transfer-artifact") }, + probes: history, + }) + await expect(HarnessAudit.seal(state.auditID, access(contract.sessionID))).rejects.toThrow("terminal") + for (const loss of [1, 1, 1]) { + const selected = await HarnessAudit.select(state.auditID, access(contract.sessionID)) + const observed = await HarnessAudit.observe(state.auditID, { + ...access(contract.sessionID), + probeID: selected.probeID, + loss, + failure: true, + evidence: [`receipt://rejected-${selected.round}`], + }) + if (selected.round === 2) { + expect(observed.estimate.transfer.status).toBe("rejected") + expect(observed.estimate.abstain).toBe(true) + } + if (selected.round === 3) expect(selected.phase).toBe("fallback") + } + const completed = await HarnessAudit.status(state.auditID, access(contract.sessionID)) + expect(completed).toMatchObject({ + status: "completed", + stopReason: "budget_exhausted", + estimate: { abstain: true, transfer: { status: "rejected" } }, + }) + const receipt = await HarnessAudit.seal(state.auditID, access(contract.sessionID)) + receipts.add(receipt.receiptID) + expect(receipt.qualified).toBe(false) + expect(await HarnessAudit.seal(state.auditID, access(contract.sessionID))).toEqual(receipt) + await expect( + HarnessAudit.assert({ + contract, + receiptID: receipt.receiptID, + subject: { type: "run", id: contract.runID }, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + requireQualified: true, + }), + ).rejects.toThrow("qualified non-abstaining") + await expect( + HarnessAudit.assert({ + contract, + receiptID: receipt.receiptID, + subject: { type: "run", id: "different-run" }, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + requireQualified: false, + }), + ).rejects.toThrow("different evaluation subject") + await expect( + HarnessAudit.assert({ + contract, + receiptID: receipt.receiptID, + subject: { type: "run", id: contract.runID }, + evaluatedAt: receipt.completedAt - 1, + recordedAt: Date.now(), + requireQualified: false, + }), + ).rejects.toThrow("predates") + const stateFile = path.join( + Global.Path.data, + "harness", + "audits", + encodeURIComponent(contract.sessionID), + `${state.auditID}.json`, + ) + const original = await fs.readFile(stateFile, "utf8") + const changed = JSON.parse(original) + changed.estimate.meanLoss = 0 + await fs.writeFile(stateFile, JSON.stringify(changed)) + expect(await HarnessAudit.readReceipt(receipt.receiptID)).toBeNull() + await fs.writeFile(stateFile, original) + const receiptFile = path.join(Global.Path.data, "harness", "audit-receipts", `${receipt.receiptID}.json`) + const tampered = JSON.parse(await fs.readFile(receiptFile, "utf8")) + tampered.estimate.meanLoss = 0 + await fs.writeFile(receiptFile, JSON.stringify(tampered)) + expect(await HarnessAudit.readReceipt(receipt.receiptID)).toBeNull() + }) + + test("requires a matching qualified audit receipt before final promotion", async () => { + const contract = await bind("audit-promotion", proactive({ promotionRequired: true })) + const state = await HarnessAudit.initialize({ + ...access(contract.sessionID), + subject: { type: "run", id: contract.runID, artifactSHA256: hash("promoted-transfer-artifact") }, + probes: history, + }) + for (let round = 0; round < 2; round++) { + const selected = await HarnessAudit.select(state.auditID, access(contract.sessionID)) + const prior = (await HarnessAudit.status(state.auditID, access(contract.sessionID))).pool[selected.probeID]! + .priorLoss + await HarnessAudit.observe(state.auditID, { + ...access(contract.sessionID), + probeID: selected.probeID, + loss: prior, + failure: false, + evidence: [`receipt://accepted-${round}`], + }) + } + const receipt = await HarnessAudit.seal(state.auditID, access(contract.sessionID)) + receipts.add(receipt.receiptID) + expect(receipt.qualified).toBe(true) + const evaluation: HarnessEvaluation.Info = { + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status: "passed", + score: 0.2, + metrics: { loss: 0.2 }, + checks: ["estimand", "assumptions", "effect-size", "uncertainty", "multiplicity", "stat-replay"].map((id) => ({ + id, + status: "passed" as const, + blocking: true, + evidence: [`receipt://${id}`], + })), + evidence: ["receipt://official-score"], + evaluatedAt: Math.max(Date.now(), receipt.sealedAt), + } + await expect(HarnessEvaluation.record(evaluation)).rejects.toThrow("qualified active audit receipt") + const recorded = await HarnessEvaluation.record({ ...evaluation, auditReceiptID: receipt.receiptID }) + expect(recorded.auditReceiptID).toBe(receipt.receiptID) + }) +}) diff --git a/backend/cli/test/session/harness-autonomy.test.ts b/backend/cli/test/session/harness-autonomy.test.ts new file mode 100644 index 00000000..15268f6a --- /dev/null +++ b/backend/cli/test/session/harness-autonomy.test.ts @@ -0,0 +1,382 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessAutonomy } from "../../src/session/harness/autonomy" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const receipts = new Set() +const evaluator = "human-ai-autonomy-evaluator-token-000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +function protocol(claimedLevel: HarnessContract.AutonomyLevel = "essentially_autonomous") { + return HarnessContract.HumanAIAutonomy.parse({ + protocolVersion: "human-ai-autonomy-v1", + claimedLevel, + recorder: { + name: "evaluator-interaction-recorder", + version: "1", + artifactSHA256: hash("recorder-binary"), + source: "evaluator_runtime", + }, + traceSchemaSHA256: hash("interaction-trace-schema"), + classificationPolicySHA256: hash("contribution-classification-policy"), + maxEvents: 32, + rawRetention: "required", + disclosure: "evaluator_retained", + completeTraceRequired: true, + uncertaintyPolicy: "inconclusive", + }) +} + +function task( + sessionID: string, + claimedLevel: HarnessContract.AutonomyLevel = "essentially_autonomous", + intervention: "autonomous" | "human_reprompted" = claimedLevel === "essentially_autonomous" + ? "autonomous" + : "human_reprompted", +): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "human-ai-autonomy", + split: "validation", + evaluator: { name: "official-autonomy-evaluator", version: "1", source: "benchmark", token: evaluator }, + autonomy: protocol(claimedLevel), + objective: "Solve the scientific task under a predeclared human-AI contribution level", + metric: { name: "accuracy", direction: "maximize", target: 0.8 }, + model: { provider: "test", name: "research-agent" }, + tools: ["read"], + skills: [{ name: "record-human-ai-autonomy" }], + budget: { steps: 20, ...(claimedLevel === "essentially_autonomous" ? {} : { candidates: 2 }) }, + profile: claimedLevel === "essentially_autonomous" ? "react" : "optimize", + seed: 43, + intervention, + contamination: { policy: "hidden benchmark material stays evaluator-private", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +function events( + contract: HarnessContract.Info, + artifactSHA256: string, + input: { + human?: HarnessAutonomy.Contribution + agent?: HarnessAutonomy.Contribution + humanKind?: HarnessAutonomy.Kind + } = {}, +): HarnessAutonomy.Submit["trace"]["events"] { + const startedAt = contract.createdAt + const endedAt = Math.max(Date.now(), startedAt) + return [ + { + sequence: 1, + at: startedAt, + actor: "benchmark", + kind: "problem_statement", + contribution: "problem", + contentSHA256: hash(`${contract.sessionID}-problem`), + evidence: ["trace://problem"], + }, + ...(input.human + ? [ + { + sequence: 2, + at: endedAt, + actor: "human" as const, + kind: input.humanKind ?? ("strategy" as const), + contribution: input.human, + contentSHA256: hash(`${contract.sessionID}-human`), + evidence: ["trace://human"], + }, + ] + : []), + { + sequence: input.human ? 3 : 2, + at: endedAt, + actor: "agent", + kind: "artifact_edit", + contribution: input.agent ?? "core", + contentSHA256: hash(`${contract.sessionID}-agent`), + artifactAfterSHA256: artifactSHA256, + evidence: ["trace://agent-artifact"], + }, + ] +} + +function submit( + contract: HarnessContract.Info, + input: { + subject?: HarnessAutonomy.Subject + artifactSHA256?: string + human?: HarnessAutonomy.Contribution + agent?: HarnessAutonomy.Contribution + humanKind?: HarnessAutonomy.Kind + } = {}, +) { + if (!contract.autonomy) throw new Error("Expected autonomy protocol") + const artifactSHA256 = input.artifactSHA256 ?? hash(`${contract.sessionID}-artifact`) + const trace = events(contract, artifactSHA256, input) + return HarnessAutonomy.Submit.parse({ + sessionID: contract.sessionID, + evaluatorToken: evaluator, + subject: input.subject ?? { type: "run", id: contract.runID }, + artifactSHA256, + trace: { + owner: "evaluator_runtime", + complete: true, + recorderArtifactSHA256: contract.autonomy.recorder.artifactSHA256, + schemaSHA256: contract.autonomy.traceSchemaSHA256, + classificationPolicySHA256: contract.autonomy.classificationPolicySHA256, + rawLogSHA256: hash(`${contract.sessionID}-raw-log`), + startedAt: contract.createdAt, + endedAt: trace.at(-1)!.at, + events: trace, + }, + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`evidence://${check.id}`], + })) + +function evaluation(contract: HarnessContract.Info, receipt?: HarnessAutonomy.Receipt) { + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: evaluator, + autonomyReceiptID: receipt?.receiptID, + status: "passed", + score: 0.9, + metrics: { accuracy: 0.9 }, + checks: checks(contract), + evidence: ["official://autonomy-result"], + evaluatedAt: Math.max(Date.now(), receipt?.endedAt ?? contract.createdAt), + }) +} + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + ...["bindings", "contracts", "evaluations", "reports", "search", "retrospectives"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + fs.rm(path.join(Global.Path.data, "harness", "autonomy", "subjects", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ]), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "autonomy", "receipts", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + receipts.clear() +}) + +describe("human-AI autonomy receipts", () => { + test("derives essentially autonomous work and gates the final report", async () => { + const contract = await HarnessAdapter.bind(task("autonomy-pass")) + const policy = HarnessAutonomy.prompt(contract) + expect(policy).toContain("essentially_autonomous") + expect(policy).not.toContain(contract.autonomy!.traceSchemaSHA256) + await expect(HarnessAdapter.ingest(evaluation(contract))).rejects.toThrow("human-AI autonomy receipt") + + const receipt = await HarnessAutonomy.record(submit(contract), contract) + receipts.add(receipt.receiptID) + expect(receipt).toMatchObject({ + status: "passed", + claimedLevel: "essentially_autonomous", + derivedLevel: "essentially_autonomous", + metrics: { problemEvents: 1, humanSubstantiveEvents: 0, agentSubstantiveEvents: 1 }, + }) + expect(receipt.events[1]?.priorEventID).toBe(receipt.events[0]?.eventID) + expect(JSON.stringify(receipt)).not.toContain(evaluator) + + await expect( + HarnessAdapter.ingest({ ...evaluation(contract, receipt), evaluatedAt: receipt.recordedAt - 1 }), + ).rejects.toThrow("predates") + + const result = await HarnessAdapter.ingest(evaluation(contract, receipt)) + const report = HarnessReport.compile({ contract, evaluations: [result.evaluation], autonomy: receipt }) + expect(report.execution.autonomy).toEqual({ + claimedLevel: "essentially_autonomous", + derivedLevel: "essentially_autonomous", + status: "passed", + }) + expect(report.quality.autonomyReceiptID).toBe(receipt.receiptID) + const collaborative = HarnessContract.Info.parse({ + ...contract, + runID: `${contract.runID}-collaborative`, + sessionID: `${contract.sessionID}-collaborative`, + intervention: "human_reprompted", + autonomy: { ...contract.autonomy!, claimedLevel: "human_ai_collaboration" }, + }) + const other = HarnessReport.compile({ contract: collaborative, evaluations: [] }) + expect(report.comparisonKey).not.toBe(other.comparisonKey) + }) + + test("downgrades essential human help while preserving auxiliary problem and exposition work", async () => { + const autonomous = await HarnessAdapter.bind( + task("autonomy-laundered", "essentially_autonomous", "human_reprompted"), + ) + const laundered = await HarnessAutonomy.record( + submit(autonomous, { human: "essential", humanKind: "strategy" }), + autonomous, + ) + receipts.add(laundered.receiptID) + expect(laundered).toMatchObject({ status: "failed", derivedLevel: "human_ai_collaboration" }) + await expect(HarnessAdapter.ingest(evaluation(autonomous, laundered))).rejects.toThrow( + "passing human-AI autonomy receipt", + ) + + const auxiliary = await HarnessAdapter.bind( + task("autonomy-auxiliary", "essentially_autonomous", "human_reprompted"), + ) + const retained = await HarnessAutonomy.record( + submit(auxiliary, { human: "auxiliary", humanKind: "exposition" }), + auxiliary, + ) + receipts.add(retained.receiptID) + expect(retained).toMatchObject({ status: "passed", derivedLevel: "essentially_autonomous" }) + + const edited = await HarnessAdapter.bind(task("autonomy-post-edit")) + const post = submit(edited) + const final = post.artifactSHA256 + post.trace.events.push({ + sequence: 3, + at: post.trace.endedAt, + actor: "agent", + kind: "artifact_edit", + contribution: "auxiliary", + contentSHA256: hash("post-final-edit"), + artifactBeforeSHA256: final, + artifactAfterSHA256: hash("unbound-post-final-artifact"), + evidence: ["trace://post-final-edit"], + }) + const changed = await HarnessAutonomy.record(post, edited) + receipts.add(changed.receiptID) + expect(changed.status).toBe("failed") + expect(changed.failures).toContain("last interaction artifact transition does not bind the final artifact") + + const collaborative = await HarnessAdapter.bind(task("autonomy-collaboration", "human_ai_collaboration")) + const collaboration = await HarnessAutonomy.record(submit(collaborative, { human: "essential" }), collaborative) + receipts.add(collaboration.receiptID) + expect(collaboration).toMatchObject({ status: "passed", derivedLevel: "human_ai_collaboration" }) + + const primarily = await HarnessAdapter.bind(task("autonomy-human", "primarily_human")) + const human = await HarnessAutonomy.record( + submit(primarily, { human: "core", agent: "auxiliary", humanKind: "artifact_edit" }), + primarily, + ) + receipts.add(human.receiptID) + expect(human).toMatchObject({ status: "passed", derivedLevel: "primarily_human" }) + + await expect( + HarnessAdapter.bind(task("autonomy-invalid-label", "human_ai_collaboration", "autonomous")), + ).rejects.toThrow("human_reprompted") + }) + + test("makes ambiguous classifications inconclusive and rejects trace or candidate laundering", async () => { + const contract = await HarnessAdapter.bind(task("autonomy-adversarial")) + const uncertain = await HarnessAutonomy.record( + submit(contract, { human: "unclear", humanKind: "technical_correction" }), + contract, + ) + receipts.add(uncertain.receiptID) + expect(uncertain.status).toBe("inconclusive") + expect(uncertain.derivedLevel).toBeUndefined() + + const changed = task("autonomy-trace-drift") + changed.profile = "optimize" + changed.budget = { ...changed.budget, candidates: 2 } + const searchContract = await HarnessAdapter.bind(changed) + await HarnessSearch.initialize({ sessionID: searchContract.sessionID }) + const recommendation = HarnessSearch.recommend(await HarnessSearch.read(searchContract.sessionID)) + const artifactSHA256 = hash("registered-candidate-artifact") + const candidate = await HarnessSearch.add({ + sessionID: searchContract.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "autonomy", + proposal: "candidate with an evaluator-owned interaction trace", + artifact: { uri: "candidate://autonomy", sha256: artifactSHA256 }, + }) + await expect( + HarnessAutonomy.record( + submit(searchContract, { + subject: { type: "candidate", id: candidate.id }, + artifactSHA256: hash("substituted-candidate-artifact"), + }), + searchContract, + ), + ).rejects.toThrow("changed the candidate artifact") + + const valid = submit(searchContract, { + subject: { type: "candidate", id: candidate.id }, + artifactSHA256, + }) + const gap = structuredClone(valid) + gap.trace.events[1]!.sequence = 3 + await expect(HarnessAutonomy.record(gap, searchContract)).rejects.toThrow("contiguous") + const late = structuredClone(valid) + late.trace.startedAt += 1 + late.trace.events[0]!.at = late.trace.startedAt + await expect(HarnessAutonomy.record(late, searchContract)).rejects.toThrow("run interval") + const future = structuredClone(valid) + future.trace.endedAt = Date.now() + 60_000 + future.trace.events[1]!.at = future.trace.endedAt + await expect(HarnessAutonomy.record(future, searchContract)).rejects.toThrow("run interval") + + const receipt = await HarnessAutonomy.record(valid, searchContract) + receipts.add(receipt.receiptID) + const replacement = structuredClone(valid) + replacement.trace.rawLogSHA256 = hash("replacement-log") + await expect(HarnessAutonomy.record(replacement, searchContract)).rejects.toThrow("canonical receipt") + }) + + test("protects receipt routes with the evaluator capability and fails closed on disk tampering", async () => { + const contract = await HarnessAdapter.bind(task("autonomy-route")) + const app = HarnessRoutes() + const response = await app.request("/autonomy/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(submit(contract)), + }) + expect(response.status).toBe(200) + const receipt = HarnessAutonomy.Receipt.parse(await response.json()) + receipts.add(receipt.receiptID) + + await expect( + HarnessAdapter.authorize(contract.sessionID, "wrong-human-ai-autonomy-token-000000000000000"), + ).rejects.toThrow("capability was rejected") + const read = await app.request(`/autonomy/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: evaluator }), + }) + expect(read.status).toBe(200) + + const file = path.join(Global.Path.data, "harness", "autonomy", "receipts", `${receipt.receiptID}.json`) + await Bun.write(file, JSON.stringify({ ...receipt, derivedLevel: "primarily_human" })) + expect(await HarnessAutonomy.readReceipt(receipt.receiptID)).toBeNull() + }) +}) diff --git a/backend/cli/test/session/harness-blueprint.test.ts b/backend/cli/test/session/harness-blueprint.test.ts new file mode 100644 index 00000000..ea69c373 --- /dev/null +++ b/backend/cli/test/session/harness-blueprint.test.ts @@ -0,0 +1,377 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessBlueprint } from "../../src/session/harness/blueprint" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessReport } from "../../src/session/harness/report" + +const sessions = new Set() +const evaluator = "proof-blueprint-evaluator-token-000000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +function protocol(input: { attempts?: number; refinements?: number; parallel?: number; leaseMs?: number } = {}) { + const kernel = hash("blueprint-lean-kernel") + return HarnessContract.FormalProof.parse({ + protocolVersion: "formal-proof-v1", + language: "lean4", + tier: "kernel", + relation: "exact_proof", + challengeSHA256: hash("blueprint-challenge"), + statementSHA256: hash("blueprint-root-statement"), + declaration: "OpenScience.root", + module: "OpenScience.Blueprint", + leanVersion: "4.33.0", + leanToolchainSHA256: hash("blueprint-toolchain"), + lakeManifestSHA256: hash("blueprint-lake-manifest"), + dependencyTreeSHA256: hash("blueprint-dependency-tree"), + verifiers: [ + { role: "lean_kernel", name: "lean", version: "4.33.0", artifactSHA256: kernel }, + { role: "source_auditor", name: "source", version: "1", artifactSHA256: hash("blueprint-source") }, + { role: "axiom_auditor", name: "axioms", version: "1", artifactSHA256: hash("blueprint-axioms") }, + ], + forbiddenConstructs: HarnessContract.FormalForbidden.options, + allowedAxioms: ["Classical.choice"], + maxFiles: 32, + completeManifestRequired: true, + warningPolicy: "fail", + semanticPolicy: "formal_statement_only", + blueprint: { + protocolVersion: "proof-blueprint-v1", + graphSchemaSHA256: hash("proof-blueprint-schema-v1"), + compilerArtifactSHA256: kernel, + sketchValidatorArtifactSHA256: hash("blueprint-sketch-validator"), + reviewerArtifactSHA256: hash("blueprint-reviewer"), + reviewerPromptSHA256: hash("blueprint-review-rubric"), + nodePolicy: "and-or-monotone-v1", + failurePolicy: "preserve-and-refine", + memoization: "goal-sha256", + finalAuthority: "formal-proof-v1", + directAttemptFirst: true, + verifiedSketchRequired: true, + completeFailureHistoryRequired: true, + maxNodes: 16, + maxDepth: 4, + maxParallel: input.parallel ?? 4, + maxAttemptsPerGoal: input.attempts ?? 2, + maxRefinementsPerGoal: input.refinements ?? 1, + leaseDurationMs: input.leaseMs ?? 60_000, + }, + }) +} + +function task( + sessionID: string, + input: { attempts?: number; refinements?: number; parallel?: number; leaseMs?: number } = {}, +) { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "proof-blueprint", + split: "validation", + evaluator: { name: "official-blueprint-evaluator", version: "1", source: "benchmark", token: evaluator }, + formalProof: protocol(input), + objective: "Prove the exact frozen Lean declaration", + metric: { name: "accuracy", direction: "maximize", target: 0.8 }, + model: { provider: "test", name: "blueprint-agent" }, + tools: ["read", "bash"], + skills: [{ name: "operate-proof-blueprint" }], + budget: { steps: 20 }, + profile: "react", + seed: 71, + intervention: "autonomous", + contamination: { policy: "trusted challenge remains evaluator-owned", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const child = (name: string) => ({ + statementSHA256: hash(`statement-${name}`), + declaration: `OpenScience.${name}`, + module: "OpenScience.Blueprint", +}) + +function verification(contract: HarnessContract.Info, lease: HarnessBlueprint.Lease, name: string, exitCode = 0) { + return { + compilerArtifactSHA256: contract.formalProof!.blueprint!.compilerArtifactSHA256, + statementMatched: true, + exitCode, + warnings: 0, + transcriptSHA256: hash(`compiler-${name}`), + feedbackSHA256: hash(`feedback-${name}`), + startedAt: lease.issuedAt, + endedAt: Date.now(), + } +} + +function direct( + contract: HarnessContract.Info, + lease: HarnessBlueprint.Lease, + name: string, + input: { claim?: "proof" | "refutation" | "failure"; exitCode?: number } = {}, +) { + return HarnessBlueprint.DirectSubmit.parse({ + sessionID: contract.sessionID, + evaluatorToken: evaluator, + kind: "direct", + leaseID: lease.id, + artifactSHA256: hash(`artifact-${name}`), + claim: input.claim ?? "proof", + verification: verification(contract, lease, name, input.exitCode), + }) +} + +function decompose( + contract: HarnessContract.Info, + lease: HarnessBlueprint.Lease, + name: string, + children: HarnessBlueprint.GoalSpec[], + review: { relevant?: boolean; easier?: boolean; plausible?: boolean } = {}, +) { + const blueprint = contract.formalProof!.blueprint! + return HarnessBlueprint.DecompositionSubmit.parse({ + sessionID: contract.sessionID, + evaluatorToken: evaluator, + kind: "decomposition", + leaseID: lease.id, + informalPlanSHA256: hash(`plan-${name}`), + artifactSHA256: hash(`sketch-${name}`), + children, + verification: { + ...verification(contract, lease, name), + validatorArtifactSHA256: blueprint.sketchValidatorArtifactSHA256, + placeholderDeclarations: children.map((item) => item.declaration).toSorted((a, b) => a.localeCompare(b)), + validatorTranscriptSHA256: hash(`validator-${name}`), + }, + review: { + reviewerArtifactSHA256: blueprint.reviewerArtifactSHA256, + promptSHA256: blueprint.reviewerPromptSHA256, + relevant: review.relevant ?? true, + easier: review.easier ?? true, + plausible: review.plausible ?? true, + transcriptSHA256: hash(`review-${name}`), + }, + }) +} + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "reports", "blueprints"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + ), + ), + ) + sessions.clear() +}) + +describe("formal proof blueprints", () => { + test("closes a compiler-grounded AND branch while formal receipt authority stays separate", async () => { + const contract = await HarnessAdapter.bind(task("blueprint-close")) + const initial = await HarnessBlueprint.initialize(contract) + expect(initial).toMatchObject({ summary: { status: "open", goals: 1, attempts: 0 } }) + expect(await HarnessBlueprint.initialize(contract)).toEqual(initial) + + const root = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + await expect( + HarnessBlueprint.record(decompose(contract, root, "premature", [child("left")]), contract), + ).rejects.toThrow("direct proof attempt") + const failed = await HarnessBlueprint.record(direct(contract, root, "root-fail", { exitCode: 1 }), contract) + expect(failed.state.attempts[0]).toMatchObject({ + result: "failed", + failures: ["Lean compiler rejected the artifact"], + }) + + const decompositionLease = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + const branch = await HarnessBlueprint.record( + decompose(contract, decompositionLease, "root-split", [child("left"), child("right")]), + contract, + ) + expect(branch.state).toMatchObject({ + summary: { status: "open", goals: 3, decompositions: 1, attempts: 2 }, + }) + expect(branch.decompositionID).toMatch(/^[a-f0-9]{64}$/) + + const work = (await HarnessBlueprint.lease(contract, 4)).leases + expect(work).toHaveLength(2) + expect(new Set(work.map((item) => item.goalID)).size).toBe(2) + const closed = await Promise.all( + work.map((lease, index) => HarnessBlueprint.record(direct(contract, lease, `child-${index}`), contract)), + ) + expect(closed.at(-1)!.state.summary).toMatchObject({ status: "proved", proved: 3, goals: 3 }) + + const report = HarnessReport.compile({ + contract, + evaluations: [], + blueprint: closed.at(-1)!.state.summary, + }) + expect(report.execution.formal?.blueprint?.status).toBe("proved") + expect(report.execution.formal?.status).toBeUndefined() + const foreign = HarnessContract.Info.parse({ + ...contract, + runID: `${contract.runID}-foreign`, + sessionID: `${contract.sessionID}-foreign`, + }) + expect(() => + HarnessReport.compile({ contract: foreign, evaluations: [], blueprint: closed.at(-1)!.state.summary }), + ).toThrow("different harness contract") + expect(await HarnessBlueprint.context(contract.sessionID)).toContain( + "never replaces the canonical formal-proof-v1 receipt", + ) + }) + + test("retains rejected decompositions and derives exhaustion from frozen budgets", async () => { + const contract = await HarnessAdapter.bind(task("blueprint-reject", { attempts: 1, refinements: 0 })) + await HarnessBlueprint.initialize(contract) + const root = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + await HarnessBlueprint.record(direct(contract, root, "root-fail", { claim: "failure" }), contract) + const lease = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + const placeholders = decompose(contract, lease, "wrong-placeholders", [child("harder")]) + placeholders.verification.placeholderDeclarations = ["OpenScience.substituted"] + await expect(HarnessBlueprint.record(placeholders, contract)).rejects.toThrow("exactly equal") + const rejected = await HarnessBlueprint.record( + decompose(contract, lease, "bad-split", [child("harder")], { easier: false }), + contract, + ) + expect(rejected.decompositionID).toBeUndefined() + expect(rejected.state).toMatchObject({ + summary: { status: "exhausted", attempts: 2, rejected: 2, decompositions: 0 }, + }) + expect(rejected.state.attempts[1]).toMatchObject({ + result: "rejected", + failures: ["reviewer rejected decomposition difficulty reduction"], + }) + expect((await HarnessBlueprint.lease(contract, 1)).leases).toHaveLength(0) + }) + + test("reuses shared goals and refines a blocked branch without mutating it", async () => { + const contract = await HarnessAdapter.bind(task("blueprint-refine", { attempts: 1, refinements: 1 })) + await HarnessBlueprint.initialize(contract) + const root = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + await HarnessBlueprint.record(direct(contract, root, "root-fail", { claim: "failure" }), contract) + const firstLease = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + const first = await HarnessBlueprint.record( + decompose(contract, firstLease, "first-branch", [child("shared"), child("false-helper")]), + contract, + ) + const work = (await HarnessBlueprint.lease(contract, 4)).leases + const falseID = first.state.goals.find((item) => item.declaration === "OpenScience.false-helper")!.id + const sharedID = first.state.goals.find((item) => item.declaration === "OpenScience.shared")!.id + const falseLease = work.find((item) => item.goalID === falseID)! + const sharedLease = work.find((item) => item.goalID === sharedID)! + await HarnessBlueprint.record(direct(contract, falseLease, "false-helper", { claim: "refutation" }), contract) + + const refineLease = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + const refined = await HarnessBlueprint.record( + decompose(contract, refineLease, "second-branch", [child("shared"), child("good-helper")]), + contract, + ) + expect(refined.state.summary).toMatchObject({ goals: 4, decompositions: 2, refinements: 1 }) + const uses = refined.state.decompositions.filter((item) => item.childIDs.includes(sharedID)) + expect(uses).toHaveLength(2) + + const goodLease = (await HarnessBlueprint.lease(contract, 4)).leases[0]! + await Promise.all([ + HarnessBlueprint.record(direct(contract, sharedLease, "shared"), contract), + HarnessBlueprint.record(direct(contract, goodLease, "good-helper"), contract), + ]) + const closed = await HarnessBlueprint.read(contract.sessionID) + expect(closed.summary).toMatchObject({ status: "proved", goals: 4, proved: 3, refuted: 1, refinements: 1 }) + expect(closed.decompositions.map((item) => item.status).toSorted()).toEqual(["blocked", "closed"]) + }) + + test("rejects cycles, substituted verifiers, consumed leases, and persisted tampering", async () => { + const contract = await HarnessAdapter.bind(task("blueprint-adversarial", { attempts: 1, refinements: 1 })) + await HarnessBlueprint.initialize(contract) + const root = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + const changed = direct(contract, root, "changed-compiler", { claim: "failure" }) + changed.verification.compilerArtifactSHA256 = hash("substituted-compiler") + await expect(HarnessBlueprint.record(changed, contract)).rejects.toThrow("frozen Lean compiler") + await HarnessBlueprint.record(direct(contract, root, "root-fail", { claim: "failure" }), contract) + await expect(HarnessBlueprint.record(direct(contract, root, "replay"), contract)).rejects.toThrow("consumed") + + const split = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + await HarnessBlueprint.record(decompose(contract, split, "root-child", [child("cycle")]), contract) + const cycle = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + await HarnessBlueprint.record(direct(contract, cycle, "cycle-fail", { claim: "failure" }), contract) + const cycleLease = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + const rootSpec = { + statementSHA256: contract.formalProof!.statementSHA256, + declaration: contract.formalProof!.declaration, + module: contract.formalProof!.module, + } + const rejected = await HarnessBlueprint.record(decompose(contract, cycleLease, "cycle-root", [rootSpec]), contract) + expect(rejected.decompositionID).toBeUndefined() + expect(rejected.state.attempts.at(-1)).toMatchObject({ + result: "rejected", + failures: ["decomposition would make the proof blueprint cyclic"], + }) + + const current = await HarnessBlueprint.state(contract.sessionID) + const attempt = Object.values(current.attempts)[0]! + const file = path.join(Global.Path.data, "harness", "blueprints", `${encodeURIComponent(contract.sessionID)}.json`) + await Bun.write( + file, + JSON.stringify({ + ...current, + attempts: { ...current.attempts, [attempt.id]: { ...attempt, artifactSHA256: hash("tampered-artifact") } }, + }), + ) + await expect(HarnessBlueprint.read(contract.sessionID)).rejects.toThrow("attempt identity") + + expect(() => + HarnessContract.FormalProof.parse({ + ...protocol(), + blueprint: { ...protocol().blueprint!, compilerArtifactSHA256: hash("another-compiler") }, + }), + ).toThrow("frozen Lean kernel") + }) + + test("serializes competing leases and protects the blueprint API with evaluator capability", async () => { + const contract = await HarnessAdapter.bind(task("blueprint-routes", { parallel: 2 })) + const app = HarnessRoutes() + const init = await app.request("/proofs/blueprints", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: evaluator }), + }) + expect(init.status).toBe(200) + expect(HarnessBlueprint.View.parse(await init.json()).summary.goals).toBe(1) + + const competing = await Promise.all(Array.from({ length: 6 }, () => HarnessBlueprint.lease(contract, 2))) + const leases = competing.flatMap((item) => item.leases) + expect(leases).toHaveLength(1) + expect(new Set(leases.map((item) => item.id)).size).toBe(1) + await expect( + HarnessAdapter.authorize(contract.sessionID, "wrong-blueprint-evaluator-token-00000000000000"), + ).rejects.toThrow("capability was rejected") + + const status = await app.request("/proofs/blueprints/status", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: evaluator }), + }) + expect(status.status).toBe(200) + expect(HarnessBlueprint.View.parse(await status.json()).summary.openLeases).toBe(1) + }) + + test("expires abandoned work and issues a fresh revision-bound lease", async () => { + const contract = await HarnessAdapter.bind(task("blueprint-expiry", { leaseMs: 1_000 })) + await HarnessBlueprint.initialize(contract) + const stale = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + await Bun.sleep(1_050) + await expect(HarnessBlueprint.record(direct(contract, stale, "too-late"), contract)).rejects.toThrow("expired") + const fresh = (await HarnessBlueprint.lease(contract, 1)).leases[0]! + expect(fresh.id).not.toBe(stale.id) + const state = await HarnessBlueprint.read(contract.sessionID) + expect(state.leases.map((item) => item.status)).toEqual(["expired", "open"]) + }) +}) diff --git a/backend/cli/test/session/harness-claims.test.ts b/backend/cli/test/session/harness-claims.test.ts new file mode 100644 index 00000000..918f5e0b --- /dev/null +++ b/backend/cli/test/session/harness-claims.test.ts @@ -0,0 +1,361 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessClaims } from "../../src/session/harness/claims" +import { HarnessContract } from "../../src/session/harness/contract" +import { ClaimTool } from "../../src/tool/claim" + +const sessions = new Set() +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + fs.rm(path.join(Global.Path.data, "harness", "contracts", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "claims", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "verifications", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ]), + ) + sessions.clear() +}) + +async function bind(sessionID: string) { + sessions.add(sessionID) + return HarnessContract.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective: "Produce a defensible scientific conclusion", + benchmark: { + name: "claim-test", + title: "Claim evaluation", + family: "custom", + task: "Produce a defensible scientific conclusion", + version: "1", + taskID: sessionID, + split: "held_out", + evaluator: "official-evaluator", + metric: "score", + direction: "maximize", + }, + profile: "reproduce", + model: { provider: "test", name: "model" }, + tools: ["claim"], + skills: [], + budget: { steps: 20 }, + seed: 2, + intervention: "autonomous", + contamination: { policy: "hidden outputs stay hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +async function declare( + sessionID: string, + kind: HarnessClaims.Kind = "descriptive", + input?: { text?: string; independentSources?: number; checks?: string[] }, +) { + await bind(sessionID) + return HarnessClaims.declare({ + sessionID, + actor: "producer", + messageID: "message", + text: input?.text ?? "The result is scientifically supported", + kind, + importance: "headline", + subject: { uri: `artifact://${sessionID}`, sha256: hash(sessionID) }, + requirements: { independentSources: input?.independentSources, checks: input?.checks }, + }) +} + +function verification( + claim: HarnessClaims.Claim, + input?: { + mode?: HarnessClaims.Mode + actor?: string + sessionID?: string + status?: "passed" | "failed" | "inconclusive" + checks?: string[] + isolation?: Partial<{ + freshProcess: boolean + cleanWorkspace: boolean + outputWithheld: boolean + codeIndependent: boolean + }> + }, +): Omit { + const mode = input?.mode ?? "heldout_evaluator" + const clean = ["clean_replay", "independent_implementation", "independent_derivation"].includes(mode) + const independent = ["independent_implementation", "independent_derivation"].includes(mode) + const status = input?.status ?? "passed" + const ids = input?.checks ?? (claim.requirements.checks.length ? claim.requirements.checks : ["direct-check"]) + return { + schemaVersion: 1, + runID: claim.runID, + sessionID: claim.sessionID, + claimID: claim.id, + mode, + producer: { actor: "producer", sessionID: claim.sessionID }, + verifier: { + actor: input?.actor ?? "verifier", + sessionID: input?.sessionID ?? (clean ? `${claim.sessionID}-verification` : claim.sessionID), + model: "independent-model", + environment: "clean-test-environment", + }, + isolation: { + freshProcess: input?.isolation?.freshProcess ?? clean, + cleanWorkspace: input?.isolation?.cleanWorkspace ?? clean, + outputWithheld: input?.isolation?.outputWithheld ?? (independent || mode === "heldout_evaluator"), + codeIndependent: input?.isolation?.codeIndependent ?? independent, + hiddenTestsAccessible: false, + }, + source: { + uri: `verification://${input?.actor ?? "verifier"}/${mode}`, + evaluator: mode === "heldout_evaluator" ? "official-evaluator" : undefined, + sha256: clean ? hash(`${claim.id}:${mode}`) : undefined, + }, + status, + summary: `${mode} ${status}`, + checks: ids.map((id) => ({ + id, + status, + blocking: true, + evidence: [`check:${id}`], + })), + evidence: [`report:${claim.id}`], + metrics: { score: status === "passed" ? 1 : 0 }, + evaluatedAt: Date.now(), + } +} + +describe("scientific claim reconciliation", () => { + test("applies claim-kind requirements and permits only stronger overrides", async () => { + const claim = await declare("claims-defaults", "statistical", { independentSources: 1, checks: ["robustness"] }) + expect(claim.requirements).toEqual({ + independentSources: 1, + checks: ["estimand", "assumptions", "multiplicity", "robustness"], + }) + }) + + test("keeps supporting agent observations provisional", async () => { + const claim = await declare("claims-observed") + await HarnessClaims.observe({ + sessionID: claim.sessionID, + claimID: claim.id, + actor: "producer", + kind: "measurement", + stance: "supports", + summary: "the local metric increased", + source: { uri: "notebook://cell-1" }, + metrics: { score: 100 }, + }) + expect(await HarnessClaims.get(claim.sessionID, claim.id)).toMatchObject({ + status: "provisional", + independentSources: 0, + }) + }) + + test("does not let provisional refutations masquerade as verified rejection", async () => { + const claim = await declare("claims-observed-refute") + await HarnessClaims.observe({ + sessionID: claim.sessionID, + claimID: claim.id, + actor: "producer", + kind: "review", + stance: "refutes", + summary: "possible concern", + source: { uri: "review://draft" }, + }) + expect((await HarnessClaims.get(claim.sessionID, claim.id))?.status).toBe("provisional") + }) + + test("rejects a verifier who is also the claim producer", async () => { + const claim = await declare("claims-self-verify") + expect(() => HarnessClaims.VerificationInfo.parse(verification(claim, { actor: "producer" }))).toThrow( + "Verifier must differ", + ) + }) + + test("requires a separate clean session, process, and workspace for replay", async () => { + const claim = await declare("claims-clean-replay") + await expect( + HarnessClaims.stage( + verification(claim, { + mode: "clean_replay", + sessionID: claim.sessionID, + isolation: { freshProcess: false, cleanWorkspace: false }, + }), + ), + ).rejects.toThrow() + }) + + test("binds held-out support to the contract's exact evaluator", async () => { + const claim = await declare("claims-evaluator") + const input = verification(claim) + input.source.evaluator = "unbound-evaluator" + await expect(HarnessClaims.stage(input)).rejects.toThrow("bound benchmark evaluator") + }) + + test("requires withheld outputs and independent code for independent implementation", async () => { + const claim = await declare("claims-independent-code") + await expect( + HarnessClaims.stage( + verification(claim, { + mode: "independent_implementation", + isolation: { outputWithheld: false, codeIndependent: false }, + }), + ), + ).rejects.toThrow("Independent verification") + }) + + test("accepts an exact-byte clean replay from a separate verifier session", async () => { + const claim = await declare("claims-valid-replay") + const result = await HarnessClaims.verify(verification(claim, { mode: "clean_replay" })) + expect(result.claim).toMatchObject({ status: "supported", independentSources: 1 }) + expect(result.verification.source.sha256).toHaveLength(64) + }) + + test("requires immutable bytes for headline performance claims", async () => { + await bind("claims-performance-hash") + await expect( + HarnessClaims.declare({ + sessionID: "claims-performance-hash", + actor: "producer", + text: "The model beats the benchmark", + kind: "performance", + importance: "headline", + subject: { uri: "artifact://mutable-model" }, + }), + ).rejects.toThrow("immutable subject SHA-256") + }) + + test("supports a descriptive claim after one valid backend verification", async () => { + const claim = await declare("claims-supported") + const result = await HarnessClaims.verify(verification(claim)) + expect(result.claim).toMatchObject({ status: "supported", independentSources: 1, missingChecks: [] }) + expect(result.claim?.evidence[0]).toMatchObject({ origin: "verified", stance: "supports" }) + }) + + test("keeps a verified claim inconclusive while required checks are missing", async () => { + const claim = await declare("claims-missing", "performance") + await HarnessClaims.verify(verification(claim, { checks: ["held-out"] })) + expect(await HarnessClaims.get(claim.sessionID, claim.id)).toMatchObject({ + status: "inconclusive", + missingChecks: ["baseline", "budget"], + }) + }) + + test("supports performance only after held-out, baseline, and budget checks", async () => { + const claim = await declare("claims-performance", "performance") + await HarnessClaims.verify(verification(claim)) + expect((await HarnessClaims.get(claim.sessionID, claim.id))?.status).toBe("supported") + }) + + test("requires two genuinely independent sources for causal claims", async () => { + const claim = await declare("claims-causal", "causal") + await HarnessClaims.verify(verification(claim, { actor: "verifier-a", sessionID: "verification-a" })) + expect((await HarnessClaims.get(claim.sessionID, claim.id))?.status).toBe("inconclusive") + await HarnessClaims.verify(verification(claim, { actor: "verifier-b", sessionID: "verification-b" })) + expect(await HarnessClaims.get(claim.sessionID, claim.id)).toMatchObject({ + status: "supported", + independentSources: 2, + }) + }) + + test("does not count repeat reports from one verifier as independent", async () => { + const claim = await declare("claims-repeat", "causal") + await HarnessClaims.verify(verification(claim, { actor: "same-verifier", sessionID: "same-session" })) + const repeated = verification(claim, { actor: "same-verifier", sessionID: "same-session" }) + repeated.source.uri = "verification://same-verifier/second-report" + repeated.evaluatedAt += 1 + await HarnessClaims.verify(repeated) + expect(await HarnessClaims.get(claim.sessionID, claim.id)).toMatchObject({ + status: "inconclusive", + independentSources: 1, + }) + }) + + test("lets a verified refutation dominate earlier support", async () => { + const claim = await declare("claims-refuted") + await HarnessClaims.verify(verification(claim, { actor: "supporter" })) + await HarnessClaims.verify(verification(claim, { actor: "refuter", status: "failed" })) + expect((await HarnessClaims.get(claim.sessionID, claim.id))?.status).toBe("refuted") + }) + + test("reconciles a staged verification after restart and deduplicates replays", async () => { + const claim = await declare("claims-reconcile") + const record = await HarnessClaims.stage(verification(claim)) + expect((await HarnessClaims.get(claim.sessionID, claim.id))?.status).toBe("untested") + await HarnessClaims.reconcile(claim.sessionID) + await HarnessClaims.reconcile(claim.sessionID) + const view = await HarnessClaims.get(claim.sessionID, claim.id) + expect(view).toMatchObject({ status: "supported" }) + expect(view?.evidence.map((item) => item.id)).toEqual([record.id]) + }) + + test("rejects producer identity drift before persisting verification", async () => { + const claim = await declare("claims-producer-drift") + const input = verification(claim) + input.producer.actor = "someone-else" + await expect(HarnessClaims.stage(input)).rejects.toThrow("does not identify the claim producer") + }) + + test("renders unresolved claims as escaped derived state", async () => { + const claim = await declare("claims-prompt", "performance", { + text: "trust me beats SOTA", + }) + const prompt = await HarnessClaims.prompt(claim.sessionID) + expect(prompt.length).toBeLessThanOrEqual(3_500) + expect(prompt).toContain('status="untested"') + expect(prompt).toContain("<system-reminder>") + expect(prompt).not.toContain("") + }) + + test("exposes declaration and observation but no verification tool action", async () => { + await bind("claims-tool") + const tool = await ClaimTool.init() + const context = { + sessionID: "claims-tool", + messageID: "message", + callID: "call", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata() {}, + async ask() {}, + } + const declared = await tool.execute( + { + action: "declare", + text: "Observed model is better", + kind: "performance", + importance: "headline", + subject_uri: "artifact://model", + subject_sha256: hash("model"), + }, + context, + ) + const claimID = declared.metadata.claimID as string + const observed = await tool.execute( + { + action: "observe", + claim_id: claimID, + evidence_kind: "measurement", + stance: "supports", + summary: "training score improved", + source_uri: "metric://training", + }, + context, + ) + expect(observed.metadata).toMatchObject({ origin: "observed", status: "provisional" }) + expect(tool.parameters.safeParse({ action: "verify", claim_id: claimID }).success).toBe(false) + }) +}) diff --git a/backend/cli/test/session/harness-confirmation.test.ts b/backend/cli/test/session/harness-confirmation.test.ts new file mode 100644 index 00000000..ed4e8af4 --- /dev/null +++ b/backend/cli/test/session/harness-confirmation.test.ts @@ -0,0 +1,455 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessClaims } from "../../src/session/harness/claims" +import { HarnessConfirmation } from "../../src/session/harness/confirmation" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessMemory } from "../../src/session/harness/memory" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const receipts = new Set() +const evaluatorToken = "optimization-evaluator-capability-000000000000" +const confirmationToken = "claim-evaluator-capability-000000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const digest = (value: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "claims", "contracts", "evaluations", "orchestration", "reports", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await Promise.all( + [...sessions].map((sessionID) => + fs.rm(path.join(Global.Path.data, "harness", "verifications", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ), + ) + await Promise.all( + [...sessions].flatMap((sessionID) => { + const base = `statistics\0confirmation-v1\0confirmation-${sessionID}\0optimization-evaluator` + const sealed = hash(JSON.stringify(protocol())) + return [ + fs.rm(path.join(Global.Path.data, "harness", "confirmations", "sessions", `${digest(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "retrospectives", `${hash(`${base}\0${sealed}`)}.json`), { + force: true, + }), + ] + }), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "confirmations", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + receipts.clear() +}) + +function protocol(input: { target?: number; manifestsEqual?: boolean } = {}) { + return HarnessContract.Confirmation.parse({ + protocolVersion: "sealed-confirmation-v1", + optimization: { + split: "validation", + manifestSHA256: hash("optimization-manifest"), + }, + claim: { + taskID: "official-hidden-confirmation", + split: "held_out", + manifestSHA256: hash(input.manifestsEqual ? "optimization-manifest" : "claim-manifest"), + validatorSHA256: hash("claim-validator"), + environmentSHA256: hash("claim-environment"), + evaluator: { name: "claim-evaluator", version: "2", source: "benchmark" }, + metric: "score", + direction: "maximize", + target: input.target ?? 0.8, + }, + selection: { rule: "terminal-verified-best-v1", subjects: 1 }, + exposure: { policy: "terminal-receipt-only", searchFeedback: false, memoryCapture: false }, + failurePolicy: "fail-closed", + }) +} + +function task( + sessionID: string, + input: { + protocol?: HarnessContract.Confirmation + split?: HarnessContract.Split + evaluatorToken?: string + confirmationToken?: string + } = {}, +) { + sessions.add(sessionID) + const confirmation = input.protocol ?? protocol() + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "confirmation-v1", + taskID: `confirmation-${sessionID}`, + split: input.split ?? "validation", + evaluator: { + name: "optimization-evaluator", + version: "1", + source: "benchmark", + token: input.evaluatorToken ?? evaluatorToken, + }, + confirmation: { + protocol: confirmation, + token: input.confirmationToken ?? confirmationToken, + }, + objective: "Optimize on validation, then confirm exactly one terminal winner on untouched hidden data", + profile: "optimize", + metric: { name: "score", direction: "maximize", target: confirmation.claim.target }, + model: { provider: "test", name: "model" }, + tools: ["read", "bash"], + skills: [], + budget: { steps: 20, candidates: 2 }, + seed: 19, + intervention: "autonomous", + contamination: { policy: "claim split remains evaluator-only", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +function checks(contract: HarnessContract.Info) { + return HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`optimization:${check.id}.json`], + })) +} + +async function finish(sessionID: string, score = 0.9) { + const contract = await HarnessAdapter.bind(task(sessionID)) + const state = await HarnessSearch.initialize({ sessionID }) + const recommendation = HarnessSearch.recommend(state) + const added = await HarnessSearch.add({ + sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "candidate", + proposal: "Candidate selected only from optimization-split evidence", + artifact: { uri: `candidate://${sessionID}`, sha256: hash(`candidate-${sessionID}`) }, + }) + const result = await HarnessAdapter.ingest({ + schemaVersion: 1, + runID: contract.runID, + sessionID, + evaluatorToken, + candidateID: added.id, + status: "passed", + score, + metrics: { score }, + checks: checks(contract), + evidence: ["optimization:official-result.json"], + evaluatedAt: Date.now(), + }) + const search = await HarnessSearch.read(sessionID) + expect(search).toMatchObject({ status: "completed", bestID: added.id, stopReason: "objective_met" }) + return { contract, candidate: search.candidates[added.id]!, evaluation: result.evaluation, search } +} + +async function submission( + contract: HarnessContract.Info, + input: { score?: number; outcome?: "completed" | "failed" | "inconclusive" } = {}, +) { + const selection = await HarnessConfirmation.select(contract) + const outcome = input.outcome ?? "completed" + const score = input.score ?? 0.85 + return HarnessConfirmation.Submit.parse({ + schemaVersion: 1, + sessionID: contract.sessionID, + confirmationToken, + candidateSHA256: selection.candidateArtifact.sha256, + manifestSHA256: contract.confirmation!.claim.manifestSHA256, + validatorSHA256: contract.confirmation!.claim.validatorSHA256, + environmentSHA256: contract.confirmation!.claim.environmentSHA256, + outcome, + score: outcome === "completed" ? score : undefined, + metrics: outcome === "completed" ? { score } : {}, + checks: HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`claim:${check.id}.json`], + })), + evidence: ["claim:result.json"], + usage: { wallTimeMs: 500, costUSD: 0.02 }, + outputSHA256: hash(`${contract.sessionID}:${outcome}:${score}`), + evaluatedAt: Math.max(Date.now(), selection.selectedAt), + }) +} + +describe("sealed post-search confirmation", () => { + test("freezes distinct optimization and claim splits, manifests, identities, and capabilities", async () => { + expect(() => protocol({ manifestsEqual: true })).toThrow("manifests must be distinct") + expect(() => task("confirmation-token-alias", { confirmationToken: evaluatorToken })).toThrow( + "capabilities must differ", + ) + const drift = task("confirmation-split-drift", { split: "development" }) + await expect(HarnessAdapter.bind(drift)).rejects.toThrow("optimization split must match") + const same = protocol() + const identity = { + ...same, + claim: { + ...same.claim, + evaluator: { name: "optimization-evaluator", version: "1", source: "benchmark" as const }, + }, + } + await expect(HarnessAdapter.bind(task("confirmation-identity-alias", { protocol: identity }))).rejects.toThrow( + "evaluator identity distinct", + ) + }) + + test("selects only the backend winner after search is terminal and rejects caller substitution", async () => { + const contract = await HarnessAdapter.bind(task("confirmation-selection")) + await HarnessSearch.initialize({ sessionID: contract.sessionID }) + await expect(HarnessConfirmation.select(contract)).rejects.toThrow("terminal search") + + const done = await finish("confirmation-selected") + const selection = await HarnessConfirmation.select(done.contract) + expect(selection).toMatchObject({ + candidateID: done.candidate.id, + candidateArtifact: done.candidate.artifact, + stopReason: "objective_met", + }) + const input = await submission(done.contract) + expect(() => HarnessConfirmation.Submit.parse({ ...input, candidateID: hash("other") })).toThrow() + await expect( + HarnessConfirmation.record( + { ...(await submission(done.contract)), candidateSHA256: hash("alternate-candidate") }, + done.contract, + ), + ).rejects.toThrow("server-selected candidate artifact") + await expect( + HarnessConfirmation.record( + { ...(await submission(done.contract)), evaluatedAt: selection.selectedAt - 1 }, + done.contract, + ), + ).rejects.toThrow("terminal selection interval") + }) + + test("keeps optimization provisional and makes sealed evidence the sole report quality source", async () => { + const done = await finish("confirmation-report", 0.99) + const before = HarnessReport.compile({ + contract: done.contract, + evaluations: [done.evaluation], + search: done.search, + generatedAt: Date.now(), + }) + expect(before.quality).toMatchObject({ + source: "sealed_confirmation", + provisional: true, + targetReached: false, + evaluator: "claim-evaluator", + }) + expect(before.quality.status).toBeUndefined() + expect(before.quality.score).toBeUndefined() + expect(HarnessReport.frontier([before])).toEqual([]) + expect(() => HarnessReport.compare([before], before.runID)).toThrow("Provisional optimization results") + + const search = await HarnessSearch.read(done.contract.sessionID) + const memory = await HarnessMemory.retrieve({ + sessionID: done.contract.sessionID, + query: "Candidate selected only from optimization-split evidence", + }) + const receipt = await HarnessConfirmation.record(await submission(done.contract, { score: 0.85 }), done.contract) + receipts.add(receipt.receiptID) + expect(await HarnessSearch.read(done.contract.sessionID)).toEqual(search) + expect( + await HarnessMemory.retrieve({ + sessionID: done.contract.sessionID, + query: "Candidate selected only from optimization-split evidence", + }), + ).toEqual(memory) + expect(JSON.stringify(receipt)).not.toContain(confirmationToken) + + const report = HarnessReport.compile({ + contract: done.contract, + evaluations: [done.evaluation], + search: done.search, + confirmation: receipt, + generatedAt: Date.now(), + }) + expect(report.quality).toMatchObject({ + source: "sealed_confirmation", + provisional: false, + status: "passed", + score: 0.85, + targetReached: true, + evaluator: "claim-evaluator", + evaluatorVersion: "2", + confirmationReceiptID: receipt.receiptID, + }) + expect(report.quality.score).not.toBe(done.evaluation.score) + }) + + test("freezes one canonical holdout receipt with exact concurrent retry idempotency", async () => { + const done = await finish("confirmation-single-shot") + const input = await submission(done.contract) + const [first, concurrent] = await Promise.all([ + HarnessConfirmation.record(input, done.contract), + HarnessConfirmation.record(input, done.contract), + ]) + receipts.add(first.receiptID) + receipts.add(concurrent.receiptID) + expect(concurrent.receiptID).toBe(first.receiptID) + expect((await HarnessConfirmation.record(input, done.contract)).receiptID).toBe(first.receiptID) + await expect( + HarnessConfirmation.record({ ...input, outputSHA256: hash("changed-holdout-output") }, done.contract), + ).rejects.toThrow("holdout retries are forbidden") + }) + + test("derives the final claim verdict instead of trusting a completed evaluator status", async () => { + const weak = await finish("confirmation-below-target") + const failed = await HarnessConfirmation.record(await submission(weak.contract, { score: 0.7 }), weak.contract) + receipts.add(failed.receiptID) + expect(failed.status).toBe("failed") + expect(failed.failures).toContainEqual(expect.stringContaining("does not satisfy")) + + const uncertain = await finish("confirmation-inconclusive") + const inconclusive = await HarnessConfirmation.record( + await submission(uncertain.contract, { outcome: "inconclusive" }), + uncertain.contract, + ) + receipts.add(inconclusive.receiptID) + expect(inconclusive.status).toBe("inconclusive") + expect(inconclusive.score).toBeUndefined() + }) + + test("requires every claim-side domain gate before deriving a pass", async () => { + const done = await finish("confirmation-domain-gates") + await expect( + HarnessConfirmation.record( + { + ...(await submission(done.contract)), + checks: [ + { + id: "official-claim-gate", + status: "passed", + blocking: true, + evidence: ["claim:gate.json"], + }, + ], + }, + done.contract, + ), + ).rejects.toThrow("Domain verification pack failed") + }) + + test("prevents provisional optimization evidence from supporting a performance claim", async () => { + const done = await finish("confirmation-claim-ledger") + const receipt = await HarnessConfirmation.record(await submission(done.contract), done.contract) + receipts.add(receipt.receiptID) + const claim = await HarnessClaims.declare({ + sessionID: done.contract.sessionID, + actor: "producer", + text: "The terminal candidate clears the official hidden benchmark target", + kind: "performance", + importance: "headline", + subject: done.candidate.artifact, + }) + const base = { + schemaVersion: 1 as const, + runID: done.contract.runID, + sessionID: done.contract.sessionID, + claimID: claim.id, + mode: "heldout_evaluator" as const, + producer: { actor: "producer", sessionID: done.contract.sessionID }, + verifier: { actor: "claim-evaluator", sessionID: "claim-evaluator-session" }, + isolation: { + freshProcess: true, + cleanWorkspace: true, + outputWithheld: true, + codeIndependent: false, + hiddenTestsAccessible: false as const, + }, + source: { + uri: `confirmation://${receipt.receiptID}`, + evaluator: "claim-evaluator", + sha256: done.candidate.artifact.sha256, + }, + status: receipt.status, + summary: "Canonical sealed confirmation result", + checks: ["held-out", "baseline", "budget"].map((id) => ({ + id, + status: "passed" as const, + blocking: true, + evidence: [`claim:${id}.json`], + })), + evidence: [`confirmation:${receipt.receiptID}`], + metrics: { score: receipt.score! }, + evaluatedAt: Math.max(Date.now(), receipt.recordedAt), + } + await expect( + HarnessClaims.verify({ ...base, source: { ...base.source, evaluator: "optimization-evaluator" } }), + ).rejects.toThrow("bound benchmark evaluator") + await expect(HarnessClaims.verify(base)).rejects.toThrow("canonical claim receipt") + const forged = HarnessClaims.Verification.parse({ ...base, id: digest(base) }) + const folder = path.join(Global.Path.data, "harness", "verifications", encodeURIComponent(done.contract.sessionID)) + await fs.mkdir(folder, { recursive: true }) + await Bun.write(path.join(folder, `${forged.id}.json`), JSON.stringify(forged)) + await expect(HarnessClaims.reconcile(done.contract.sessionID)).rejects.toThrow("canonical claim receipt") + await fs.rm(path.join(folder, `${forged.id}.json`)) + const verified = await HarnessClaims.verify({ + ...base, + source: { ...base.source, confirmationReceiptID: receipt.receiptID }, + }) + expect(verified.claim).toMatchObject({ status: "supported", independentSources: 1 }) + }) + + test("exposes selection and receipts only through the claim evaluator capability", async () => { + const done = await finish("confirmation-route") + const app = HarnessRoutes() + const denied = await app.request("/confirmations/selection", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: done.contract.sessionID, confirmationToken: evaluatorToken }), + }) + expect(denied.status).not.toBe(200) + const selected = await app.request("/confirmations/selection", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: done.contract.sessionID, confirmationToken }), + }) + expect(selected.status).toBe(200) + + const recorded = await app.request("/confirmations/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(await submission(done.contract)), + }) + expect(recorded.status).toBe(200) + const receipt = (await recorded.json()) as HarnessConfirmation.Receipt + receipts.add(receipt.receiptID) + const hidden = await app.request(`/confirmations/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: done.contract.sessionID, confirmationToken: evaluatorToken }), + }) + expect(hidden.status).not.toBe(200) + const read = await app.request(`/confirmations/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: done.contract.sessionID, confirmationToken }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, status: "passed" }) + }) +}) diff --git a/backend/cli/test/session/harness-contract.test.ts b/backend/cli/test/session/harness-contract.test.ts new file mode 100644 index 00000000..3e4fbb6b --- /dev/null +++ b/backend/cli/test/session/harness-contract.test.ts @@ -0,0 +1,120 @@ +import { afterEach, describe, expect, test } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { Global } from "../../src/global" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" + +const sessions = new Set() + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + fs.rm(path.join(Global.Path.data, "harness", "contracts", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "evaluations", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + ]), + ) + sessions.clear() +}) + +function contract(sessionID: string): HarnessContract.Info { + sessions.add(sessionID) + return { + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective: "Maximize the official held-out score without accessing hidden tests", + benchmark: { + name: "example-bench", + title: "Example evaluation", + family: "custom", + task: "Produce a verified held-out result", + version: "2026.08", + taskID: "task-1", + split: "held_out", + evaluator: "official-evaluator", + metric: "score", + direction: "maximize", + }, + profile: "optimize", + model: { provider: "openai", name: "gpt-test", effort: "high" }, + tools: ["bash", "read"], + skills: [{ name: "statistics", version: "1.0.0" }], + budget: { wallTimeMs: 60_000, steps: 50, costUSD: 10 }, + seed: 17, + intervention: "autonomous", + contamination: { policy: "official", hiddenTestsAccessible: false }, + createdAt: Date.now(), + } +} + +function evaluation(sessionID: string): HarnessEvaluation.Info { + return { + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + evaluator: { name: "official-evaluator", version: "1.0.0", source: "benchmark" }, + status: "passed", + score: 0.82, + metrics: { score: 0.82 }, + checks: [{ id: "official-score", status: "passed", blocking: true, evidence: ["metric:score"] }], + evidence: ["artifact:submission.csv"], + evaluatedAt: Date.now(), + } +} + +describe("harness contract", () => { + test("persists an immutable-shaped benchmark run contract", async () => { + const input = contract("session-contract") + await HarnessContract.bind(input) + expect(await HarnessContract.read(input.sessionID)).toEqual(input) + expect(HarnessContract.fingerprint(input)).toHaveLength(64) + expect(HarnessContract.fingerprint(input)).toBe(HarnessContract.fingerprint(structuredClone(input))) + await expect(HarnessContract.bind({ ...input, objective: "changed after binding" })).rejects.toThrow("immutable") + }) + + test("rejects a passed evaluation with a failed blocking gate", () => { + const input = evaluation("session-invalid") + input.checks[0]!.status = "failed" + expect(() => HarnessEvaluation.Info.parse(input)).toThrow("non-passing blocking check") + }) + + test("records only the evaluator and run bound by the contract", async () => { + const input = contract("session-evaluation") + await HarnessContract.bind(input) + const submitted = { ...evaluation(input.sessionID), recordedAt: 1 } + const before = Date.now() + const result = await HarnessEvaluation.record(submitted) + expect(HarnessEvaluation.verified(result)).toBe(true) + expect(result.recordedAt).toBeGreaterThanOrEqual(before) + expect(await HarnessEvaluation.read(input.sessionID)).toEqual(result) + expect(await HarnessEvaluation.record(submitted)).toEqual(result) + + const mismatch = evaluation(input.sessionID) + mismatch.runID = "different-run" + await expect(HarnessEvaluation.record(mismatch)).rejects.toThrow("does not match contract run") + }) + + test("pins evaluator version and source when the adapter declares them", async () => { + const input = contract("session-evaluator-version") + input.benchmark.evaluatorVersion = "1.0.0" + input.benchmark.evaluatorSource = "benchmark" + await HarnessContract.bind(input) + await expect( + HarnessEvaluation.record({ + ...evaluation(input.sessionID), + evaluator: { name: "official-evaluator", version: "2.0.0", source: "benchmark" }, + }), + ).rejects.toThrow("does not match contract evaluator version") + await expect( + HarnessEvaluation.record({ + ...evaluation(input.sessionID), + evaluator: { name: "official-evaluator", version: "1.0.0", source: "external" }, + }), + ).rejects.toThrow("does not match contract source") + }) +}) diff --git a/backend/cli/test/session/harness-domain.test.ts b/backend/cli/test/session/harness-domain.test.ts new file mode 100644 index 00000000..132ddb4e --- /dev/null +++ b/backend/cli/test/session/harness-domain.test.ts @@ -0,0 +1,211 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessPack } from "../../src/session/harness/pack" + +const sessions = new Set() + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["contracts", "evaluations"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + sessions.clear() +}) + +async function bind(sessionID: string, packs?: HarnessPack.Id[]) { + sessions.add(sessionID) + return HarnessContract.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective: "Pass a domain benchmark without skipping methodology", + benchmark: { + name: "domain-test", + title: "Domain evaluation", + family: "custom", + task: "Apply every declared methodology check", + version: "1", + taskID: sessionID, + split: "held_out", + evaluator: "official-evaluator", + metric: "score", + direction: "maximize", + }, + profile: "optimize", + packs, + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { steps: 10 }, + seed: 1, + intervention: "autonomous", + contamination: { policy: "hidden tests stay hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const checks = (packs: HarnessPack.Id[]): HarnessDomain.Actual[] => + HarnessDomain.compose(packs).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`evidence:${check.id}`], + })) + +function evaluation( + sessionID: string, + input: { status?: HarnessEvaluation.Status; checks: HarnessEvaluation.Check[] }, +): HarnessEvaluation.Info { + const status = input.status ?? "passed" + return { + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status, + score: status === "passed" ? 1 : 0, + metrics: { score: status === "passed" ? 1 : 0 }, + checks: input.checks, + evidence: ["report:domain"], + evaluatedAt: Date.now(), + } +} + +describe("domain verification packs", () => { + test("publishes every typed domain pack", () => { + expect(Object.keys(HarnessDomain.catalog).toSorted()).toEqual(HarnessPack.Id.options.toSorted()) + for (const pack of Object.values(HarnessDomain.catalog)) expect(HarnessDomain.Info.parse(pack)).toEqual(pack) + }) + + test("gives every pack at least one blocking methodology gate", () => { + for (const pack of Object.values(HarnessDomain.catalog)) { + expect(pack.checks.some((check) => check.severity === "blocking")).toBe(true) + } + }) + + test("composes shared ML and forecast gates without collisions", () => { + const combined = HarnessDomain.compose(["ml", "forecast"]) + expect(combined.filter((check) => check.id === "held-out")).toHaveLength(1) + expect(combined.filter((check) => check.id === "baseline")).toHaveLength(1) + expect(new Set(combined.map((check) => check.id)).size).toBe(combined.length) + }) + + test("reports every missing blocking check", () => { + const audit = HarnessDomain.audit(["statistics"], []) + expect(audit.missing.map((check) => check.id)).toEqual( + expect.arrayContaining(["estimand", "assumptions", "effect-size", "uncertainty", "multiplicity", "stat-replay"]), + ) + expect(audit.advisory.map((check) => check.id)).toEqual(["sensitivity"]) + }) + + test("rejects present checks that fail or remain inconclusive", () => { + const actual = checks(["pde"]) + actual.find((check) => check.id === "pde-convergence")!.status = "failed" + expect(HarnessDomain.audit(["pde"], actual).failed).toEqual([ + expect.objectContaining({ reason: "status:failed", check: expect.objectContaining({ id: "pde-convergence" }) }), + ]) + }) + + test("requires evidence and blocking posture for contract gates", () => { + const actual = checks(["chemistry"]) + actual.find((check) => check.id === "chem-identity")!.evidence = [] + actual.find((check) => check.id === "chem-valence")!.blocking = false + expect(HarnessDomain.audit(["chemistry"], actual).failed.map((item) => item.reason)).toEqual( + expect.arrayContaining(["missing-evidence", "not-marked-blocking"]), + ) + }) + + test("fails closed on duplicate evaluator check IDs", () => { + const actual = checks(["physics"]) + actual.push(structuredClone(actual[0]!)) + expect(() => HarnessDomain.assert(["physics"], actual)).toThrow("duplicate") + }) + + test("passes a complete evidence-backed composed portfolio", () => { + const actual = checks(["biology", "statistics"]) + const audit = HarnessDomain.assert(["biology", "statistics"], actual) + expect(audit.missing).toEqual([]) + expect(audit.failed).toEqual([]) + }) + + test("rejects a passed external evaluation with an omitted domain gate", async () => { + await bind("domain-eval-missing", ["ml"]) + const actual = checks(["ml"]).filter((check) => check.id !== "ml-leakage") + await expect(HarnessEvaluation.record(evaluation("domain-eval-missing", { checks: actual }))).rejects.toThrow( + "ml-leakage:missing", + ) + }) + + test("records failed evaluations even when the run stopped before every gate", async () => { + await bind("domain-eval-failed", ["pde"]) + const result = await HarnessEvaluation.record( + evaluation("domain-eval-failed", { + status: "failed", + checks: [{ id: "pde-stability", status: "failed", blocking: true, evidence: ["solver:diverged"] }], + }), + ) + expect(result.status).toBe("failed") + }) + + test("keeps contracts without packs backward compatible", async () => { + await bind("domain-eval-generic") + const result = await HarnessEvaluation.record( + evaluation("domain-eval-generic", { + checks: [{ id: "generic", status: "passed", blocking: true, evidence: ["report:generic"] }], + }), + ) + expect(result.status).toBe("passed") + }) + + test.each([ + ["research", "react", "Compute a chi-square test and effect size for this table.", ["statistics"]], + ["biology", "react", "Analyze this gene expression cohort and compare groups.", ["biology"]], + ["physics", "theory", "Derive the theoretical field equations.", ["physics"]], + ["physics", "numerical", "Simulate this PDE with finite elements.", ["physics", "pde"]], + ["research", "react", "Model molecular properties for these compounds.", ["chemistry"]], + ["ml", "training", "Train this model with SFT.", ["ml"]], + ["ml", "forecast", "Evaluate this weather forecast model.", ["ml", "forecast"]], + ] as const)("recommends bounded packs for %s/%s", (agent, profile, text, expected) => { + expect(HarnessDomain.recommend({ agent, profile, text })).toEqual([...expected]) + }) + + test("does not turn simple definitions or lookups into methodology workflows", () => { + expect( + HarnessDomain.recommend({ agent: "research", profile: "react", text: "What is a chi-square test?" }), + ).toEqual([]) + expect(HarnessDomain.recommend({ agent: "biology", profile: "react", text: "Look up TP53." })).toEqual([]) + }) + + test("uses immutable contract packs instead of heuristic recommendations", async () => { + await bind("domain-contract-route", ["chemistry"]) + expect( + await HarnessDomain.resolve({ + sessionID: "domain-contract-route", + agent: "ml", + profile: "forecast", + text: "Evaluate a weather forecast model.", + }), + ).toEqual({ ids: ["chemistry"], source: "contract" }) + }) + + test("renders a complete bounded checklist with stable IDs", () => { + const prompt = HarnessDomain.prompt({ ids: HarnessPack.Id.options, source: "contract" }) + expect(prompt.length).toBeLessThan(12_000) + expect(prompt).toContain("pde-convergence") + expect(prompt).toContain("chem-split") + expect(prompt).toContain("forecast-leads") + expect(prompt.endsWith("")).toBe(true) + }) + + test("rejects duplicate packs in the immutable contract", async () => { + await expect(bind("domain-duplicate-pack", ["ml", "ml"])).rejects.toThrow("packs must be unique") + }) +}) diff --git a/backend/cli/test/session/harness-evolution.test.ts b/backend/cli/test/session/harness-evolution.test.ts new file mode 100644 index 00000000..1169c6ed --- /dev/null +++ b/backend/cli/test/session/harness-evolution.test.ts @@ -0,0 +1,402 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessEvolution } from "../../src/session/harness/evolution" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const token = "evolution-evaluator-capability-token-00000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "evolution", "reports", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await fs.rm(path.join(Global.Path.data, "harness", "retrospectives"), { recursive: true, force: true }) + sessions.clear() +}) + +function protocol() { + return HarnessContract.Evolution.parse({ + protocolVersion: "evolution-trace-v1", + validatorSHA256: hash("trace-evolutionary-candidate.py:v1"), + manifestSchemaSHA256: hash("evolution-source-manifest:v1"), + lineAlgorithm: "sha256-exact-line-v1", + roots: ["src"], + extensions: [".ts"], + exclude: ["src/generated"], + maxFiles: 100, + maxFileBytes: 100_000, + maxTotalBytes: 1_000_000, + maxSourceLines: 10_000, + maxChangedLines: 1_000, + }) +} + +function task(sessionID: string): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "mle", + version: "2026.08", + taskID: "evolution-task-1", + split: "validation", + evaluator: { name: "official-evolution-evaluator", version: "7", source: "benchmark", token }, + objective: "Improve the official score while retaining evaluator-owned replayable source lineage", + search: "static", + evolution: protocol(), + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "research-agent" }, + tools: ["read", "bash"], + skills: [{ name: "trace-evolutionary-candidate", version: "1", sha256: protocol().validatorSHA256 }], + budget: { steps: 40, candidates: 4 }, + seed: 43, + intervention: "autonomous", + contamination: { policy: "hidden tests remain outside the candidate process", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + +function snapshot(contract: HarnessContract.Info, name: string, content: string) { + const evolution = contract.evolution + if (!evolution) throw new Error("Expected an evolution trace protocol") + const bytes = new TextEncoder().encode(content) + const files = HarnessEvolution.Files.parse([ + { + path: "src/main.ts", + sha256: hash(content), + bytes: bytes.byteLength, + lineHashes: content.split("\n").flatMap((line) => (line ? [hash(line)] : [])), + }, + ]) + return HarnessEvolution.Snapshot.parse({ + artifact: { uri: `artifact:${name}.manifest.json`, sha256: HarnessEvolution.manifestSHA256(evolution, files) }, + schemaSHA256: evolution.manifestSchemaSHA256, + files, + }) +} + +function submit( + contract: HarnessContract.Info, + candidate: HarnessSearch.Candidate, + source: ReturnType, + parents: HarnessEvolution.Info[] = [], +) { + const evolution = contract.evolution + if (!evolution) throw new Error("Expected an evolution trace protocol") + const subject = { type: "candidate" as const, id: candidate.id, artifact: candidate.artifact } + const evaluatedAt = Math.max(candidate.createdAt, ...parents.map((parent) => parent.evaluatedAt), Date.now()) + 1 + return HarnessEvolution.Submit.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + protocol: evolution, + subject, + snapshot: source, + parents: parents + .map((parent) => ({ + id: parent.subject.id, + artifact: parent.subject.artifact, + receiptID: parent.receiptID, + snapshotSHA256: parent.snapshot.artifact.sha256, + delta: { + uri: `artifact:${candidate.id}-${parent.subject.id}.delta.json`, + sha256: HarnessEvolution.deltaSHA256({ subject, snapshot: source, parent }), + }, + })) + .toSorted((left, right) => left.id.localeCompare(right.id)), + validator: { + name: "trace-evolutionary-candidate", + version: 1, + scriptSHA256: evolution.validatorSHA256, + }, + evidence: [`artifact:${candidate.id}-trace-report.json`], + evaluatedAt, + }) +} + +function evaluation(contract: HarnessContract.Info, candidateID: string, receiptID?: string, score = 0.8) { + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + candidateID, + evolutionReceiptID: receiptID, + status: "passed", + score, + metrics: { score }, + checks: checks(contract), + evidence: ["official:score.json"], + evaluatedAt: Date.now() + 10, + }) +} + +async function add(contract: HarnessContract.Info, branch: string, artifact: string) { + const state = await HarnessSearch.read(contract.sessionID) + const recommendation = HarnessSearch.recommend(state) + const result = await HarnessSearch.add({ + sessionID: contract.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch, + proposal: `Evaluate ${branch} without trusting producer-authored lineage`, + artifact: { uri: `artifact:${artifact}`, sha256: hash(artifact) }, + }) + return result.state.candidates[result.id]! +} + +describe("evaluator-owned evolutionary provenance", () => { + test("derives ancestral source reintroductions while leaving fitness under evaluator authority", async () => { + const contract = await HarnessAdapter.bind(task("evolution-cycle")) + await HarnessSearch.initialize({ sessionID: contract.sessionID, candidates: 4 }) + + const root = await add(contract, "root", "root-artifact") + const rootTrace = await HarnessEvolution.record( + submit(contract, root, snapshot(contract, "root", "alpha\nold\n")), + contract, + ) + expect(rootTrace.diagnostics).toMatchObject({ depth: 0, ancestors: 0, addedLines: 0, cycleDetected: false }) + expect((await HarnessSearch.read(contract.sessionID)).candidates[root.id]!.result).toBeUndefined() + await expect(HarnessAdapter.ingest(evaluation(contract, root.id))).rejects.toThrow("must reference") + await HarnessAdapter.ingest(evaluation(contract, root.id, rootTrace.receiptID, 0.7)) + + const scout = await add(contract, "independent-scout", "scout-artifact") + const scoutTrace = await HarnessEvolution.record( + submit(contract, scout, snapshot(contract, "scout", "alpha\nscout\n")), + contract, + ) + await HarnessAdapter.ingest(evaluation(contract, scout.id, scoutTrace.receiptID, 0.6)) + + const child = await add(contract, "replace-old", "child-artifact") + const childTrace = await HarnessEvolution.record( + submit(contract, child, snapshot(contract, "child", "alpha\nnew\n"), [rootTrace]), + contract, + ) + expect(childTrace.diagnostics).toMatchObject({ + depth: 1, + ancestors: 1, + addedLines: 1, + deletedLines: 1, + reintroducedLines: 0, + cycleDetected: false, + }) + await expect(HarnessAdapter.ingest(evaluation(contract, child.id, rootTrace.receiptID, 0.75))).rejects.toThrow( + "does not match the evaluated candidate", + ) + await HarnessAdapter.ingest(evaluation(contract, child.id, childTrace.receiptID, 0.75)) + + const grandchild = await add(contract, "reintroduce-old", "grandchild-artifact") + const grandchildTrace = await HarnessEvolution.record( + submit(contract, grandchild, snapshot(contract, "grandchild", "alpha\nold\nfresh\n"), [childTrace]), + contract, + ) + expect(grandchildTrace.diagnostics).toMatchObject({ + depth: 2, + ancestors: 2, + addedLines: 2, + deletedLines: 1, + ancestralDeletedLines: 1, + reintroducedLines: 1, + reintroducedHashes: 1, + reintroducedFraction: 0.5, + novelLines: 1, + sourceChanged: true, + cycleDetected: true, + }) + const result = await HarnessAdapter.ingest(evaluation(contract, grandchild.id, grandchildTrace.receiptID, 0.9)) + expect(result.search?.candidates[grandchild.id]!.result?.evolution).toMatchObject({ + receiptID: grandchildTrace.receiptID, + reintroducedLines: 1, + cycleDetected: true, + }) + expect(result.search?.bestID).toBe(grandchild.id) + const report = HarnessReport.compile({ contract, evaluations: [result.evaluation], search: result.search }) + expect(report.quality.evolutionReceiptID).toBe(grandchildTrace.receiptID) + }) + + test("rejects candidate, parent, validator, protocol, delta, and temporal substitution", async () => { + const contract = await HarnessAdapter.bind(task("evolution-substitution")) + await HarnessSearch.initialize({ sessionID: contract.sessionID, candidates: 3 }) + const root = await add(contract, "root", "substitution-root") + const rootInput = submit(contract, root, snapshot(contract, "substitution-root", "alpha\nold\n")) + await expect( + HarnessEvolution.record( + { + ...rootInput, + subject: { ...rootInput.subject, artifact: { ...rootInput.subject.artifact, sha256: hash("x") } }, + }, + contract, + ), + ).rejects.toThrow("artifact does not match") + await expect( + HarnessEvolution.record( + { ...rootInput, validator: { ...rootInput.validator, scriptSHA256: hash("substituted-validator") } }, + contract, + ), + ).rejects.toThrow("validator does not match") + await expect( + HarnessEvolution.record({ ...rootInput, protocol: { ...rootInput.protocol, maxChangedLines: 999 } }, contract), + ).rejects.toThrow("immutable harness contract") + await expect(HarnessEvolution.record({ ...rootInput, evaluatedAt: root.createdAt - 1 }, contract)).rejects.toThrow( + "predates the candidate", + ) + + const rootTrace = await HarnessEvolution.record(rootInput, contract) + await HarnessAdapter.ingest(evaluation(contract, root.id, rootTrace.receiptID, 0.7)) + const scout = await add(contract, "scout", "substitution-scout") + const scoutTrace = await HarnessEvolution.record( + submit(contract, scout, snapshot(contract, "substitution-scout", "alpha\nscout\n")), + contract, + ) + await HarnessAdapter.ingest(evaluation(contract, scout.id, scoutTrace.receiptID, 0.6)) + const child = await add(contract, "child", "substitution-child") + const childInput = submit(contract, child, snapshot(contract, "substitution-child", "alpha\nnew\n"), [rootTrace]) + await expect( + HarnessEvolution.record( + { + ...childInput, + parents: childInput.parents.map((parent) => ({ + ...parent, + delta: { ...parent.delta, sha256: hash("substituted-delta") }, + })), + }, + contract, + ), + ).rejects.toThrow("delta content hash is invalid") + await expect( + HarnessEvolution.record( + { + ...childInput, + parents: childInput.parents.map((parent) => ({ ...parent, receiptID: hash("missing-receipt") })), + }, + contract, + ), + ).rejects.toThrow("does not exist") + await expect( + HarnessEvolution.record( + { ...childInput, parents: childInput.parents.map((parent) => ({ ...parent, id: hash("other-parent") })) }, + contract, + ), + ).rejects.toThrow("do not match the candidate lineage") + const childTrace = await HarnessEvolution.record(childInput, contract) + await expect( + HarnessEvolution.record({ ...childInput, evidence: ["artifact:different-report.json"] }, contract), + ).rejects.toThrow("immutable once recorded") + expect(childTrace.parents[0]!.receiptID).toBe(rootTrace.receiptID) + }) + + test("derives fusion novelty against the union of both exact parent snapshots", async () => { + const contract = await HarnessAdapter.bind(task("evolution-fusion")) + await HarnessSearch.initialize({ sessionID: contract.sessionID, candidates: 3, stall: 1 }) + const left = await add(contract, "left-root", "left-artifact") + const leftTrace = await HarnessEvolution.record( + submit(contract, left, snapshot(contract, "left", "common\nleft\nold\n")), + contract, + ) + await HarnessAdapter.ingest(evaluation(contract, left.id, leftTrace.receiptID, 0.8)) + const right = await add(contract, "right-root", "right-artifact") + const rightTrace = await HarnessEvolution.record( + submit(contract, right, snapshot(contract, "right", "common\nright\nold\n")), + contract, + ) + await HarnessAdapter.ingest(evaluation(contract, right.id, rightTrace.receiptID, 0.7)) + + const fused = await add(contract, "fusion", "fused-artifact") + expect(fused.parentIDs.toSorted()).toEqual([left.id, right.id].toSorted()) + const trace = await HarnessEvolution.record( + submit(contract, fused, snapshot(contract, "fused", "common\nleft\nright\n"), [leftTrace, rightTrace]), + contract, + ) + expect(trace.diagnostics).toMatchObject({ + depth: 1, + ancestors: 2, + addedLines: 0, + deletedLines: 1, + reintroducedLines: 0, + reintroducedFraction: 0, + novelLines: 0, + sourceChanged: true, + cycleDetected: false, + }) + expect(trace.diagnostics.parents).toEqual([ + expect.objectContaining({ addedLines: 1, deletedLines: 1, filesChanged: 1 }), + expect.objectContaining({ addedLines: 1, deletedLines: 1, filesChanged: 1 }), + ]) + }) + + test("protects receipt routes and fails closed when derived diagnostics are edited", async () => { + const contract = await HarnessAdapter.bind(task("evolution-route")) + await HarnessSearch.initialize({ sessionID: contract.sessionID, candidates: 1 }) + const root = await add(contract, "route-root", "route-root") + const input = submit(contract, root, snapshot(contract, "route-root", "alpha\n")) + const app = HarnessRoutes() + const recorded = await app.request("/evolution/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(input), + }) + expect(recorded.status).toBe(200) + const receipt = (await recorded.json()) as HarnessEvolution.Info + expect(JSON.stringify(receipt)).not.toContain(token) + + const denied = await app.request(`/evolution/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: "x".repeat(48) }), + }) + expect(denied.status).not.toBe(200) + const read = await app.request(`/evolution/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: token }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, subject: { id: root.id } }) + + const target = path.join(Global.Path.data, "harness", "evolution", `${encodeURIComponent(contract.sessionID)}.json`) + const state = (await Bun.file(target).json()) as { items: Record } + state.items[receipt.receiptID]!.diagnostics.cycleDetected = true + await Bun.write(target, JSON.stringify(state)) + await expect(HarnessEvolution.list(contract.sessionID)).rejects.toThrow("content hash is invalid") + }) + + test("keeps the protocol optional and restricts it to authenticated optimization", async () => { + const legacy = task("evolution-legacy") + delete legacy.evolution + expect((await HarnessAdapter.bind(legacy)).evolution).toBeUndefined() + const external = await HarnessAdapter.bind({ + ...task("evolution-external"), + evaluator: { name: "external-evaluator", version: "1", source: "external", token }, + }) + expect(() => HarnessContract.Info.parse({ ...external, profile: "theory" })).toThrow("optimize profile") + expect(() => + HarnessContract.Info.parse({ + ...external, + benchmark: { ...external.benchmark, evaluator: "human", evaluatorSource: "human" }, + }), + ).toThrow("capability-authenticated") + }) +}) diff --git a/backend/cli/test/session/harness-failure.test.ts b/backend/cli/test/session/harness-failure.test.ts new file mode 100644 index 00000000..6d1b019d --- /dev/null +++ b/backend/cli/test/session/harness-failure.test.ts @@ -0,0 +1,610 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessAudit } from "../../src/session/harness/audit" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessFailure } from "../../src/session/harness/failure" + +const sessions = new Set() +const auditReceipts = new Set() +const failureReceipts = new Set() +const token = "topic-aware-failure-evaluator-capability-token-0000000000" +const digest = (value: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex") + +const probes = Array.from( + { length: 4 }, + (_, index): HarnessAudit.Probe => ({ + id: `probe-${index}`, + commitment: digest(`hidden-probe-${index}`), + features: [index - 1.5], + stratum: index < 2 ? "left" : "right", + weight: 1, + priorLoss: 0.5, + }), +) + +const identity = (name: string) => ({ + name, + version: "1", + promptSHA256: digest(`${name}-prompt`), + configSHA256: digest(`${name}-config`), +}) + +function audit(): HarnessContract.Audit { + return HarnessContract.Audit.parse({ + mode: "failure", + budget: 3, + minSamples: 2, + noiseVariance: 0.05, + lengthscale: 0.7, + beta: 1.96, + failureThreshold: 0.5, + tolerance: 0.01, + maxUncertainty: 0.05, + estimationWeight: 0.5, + diversityWeight: 0.2, + coverageWeight: 0.2, + }) +} + +function config(values: Partial = {}) { + return HarnessContract.FailureDiscovery.parse({ + protocolVersion: "topic-aware-failure-v1", + sourcePoolSHA256: digest(probes), + topicModel: { kind: "predefined", identity: identity("topic-model") }, + topics: ["alpha", "beta", "gamma"].map((id) => ({ id, commitment: digest(`topic-${id}`) })), + generator: identity("generator"), + validators: HarnessContract.FailureValidatorKind.options.map((kind) => ({ + kind, + identity: identity(`${kind}-validator`), + })), + embedding: { identity: identity("embedding-model"), dimensions: 2, regularization: 1e-6 }, + budget: 4, + anchorsPerAttempt: 2, + exploration: Math.SQRT2, + failureThreshold: 0.5, + ...values, + }) +} + +async function bind(sessionID: string, failureDiscovery = config()) { + sessions.add(sessionID) + return HarnessAdapter.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "topic-aware-failure-discovery", + split: "validation", + evaluator: { name: "official-evaluator", version: "1", source: "benchmark", token }, + objective: "Discover diverse validated failures without changing the official score", + audit: audit(), + failureDiscovery, + metric: { name: "loss", direction: "minimize" }, + model: { provider: "test", name: "target-model" }, + tools: [], + skills: [], + budget: { steps: 30 }, + seed: 7, + intervention: "autonomous", + contamination: { policy: "generated cases stay outside the population estimate", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +async function source(contract: HarnessContract.Info) { + const subject = { + type: "run" as const, + id: contract.runID, + artifactSHA256: digest(`${contract.sessionID}-artifact`), + } + const state = await HarnessAudit.initialize({ + sessionID: contract.sessionID, + evaluatorToken: token, + subject, + probes, + }) + for (const index of [0, 1, 2]) { + const selection = await HarnessAudit.select(state.auditID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + await HarnessAudit.observe(state.auditID, { + sessionID: contract.sessionID, + evaluatorToken: token, + probeID: selection.probeID, + loss: 0.7 + index * 0.1, + failure: true, + evidence: [`evidence://audit-failure-${index}`], + }) + } + const receipt = await HarnessAudit.seal(state.auditID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + auditReceipts.add(receipt.receiptID) + const stream = await HarnessFailure.initialize({ + sessionID: contract.sessionID, + evaluatorToken: token, + subject, + auditReceiptID: receipt.receiptID, + }) + return { subject, audit: state, receipt, stream } +} + +const validations = (status: "passed" | "failed" | "inconclusive" = "passed") => + HarnessContract.FailureValidatorKind.options.map((kind) => ({ + kind, + status, + evidence: [`evidence://${kind}`], + })) + +function generated(index: number, embedding: number[] = [1, 0]): HarnessFailure.Generation { + return { + status: "generated", + caseSHA256: digest(`generated-case-${index}`), + outputSHA256: digest(`generator-output-${index}`), + embedding, + evidence: [`evidence://generation-${index}`], + } +} + +function outcome(index: number, failure = true): HarnessFailure.Outcome { + return { + loss: failure ? 0.9 : 0.1, + failure, + outputSHA256: digest(`target-output-${index}`), + evidence: [`evidence://target-${index}`], + } +} + +async function record( + state: HarnessFailure.State, + index: number, + options: { + generation?: HarnessFailure.Generation + validations?: HarnessFailure.Validation[] + outcome?: HarnessFailure.Outcome + } = {}, +) { + const selection = await HarnessFailure.next(state.streamID, { + sessionID: state.sessionID, + evaluatorToken: token, + }) + return HarnessFailure.observe(state.streamID, { + sessionID: state.sessionID, + evaluatorToken: token, + selectionID: selection.selectionID, + generation: options.generation ?? generated(index), + validations: options.validations ?? validations(), + outcome: options.outcome ?? outcome(index), + evaluatedAt: Math.max(Date.now(), selection.selectedAt), + }) +} + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + fs.rm(path.join(Global.Path.data, "harness", "bindings", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "contracts", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "evaluations", `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "audits", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + fs.rm(path.join(Global.Path.data, "harness", "failures", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ]), + ) + await Promise.all( + [...auditReceipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "audit-receipts", `${receiptID}.json`), { force: true }), + ), + ) + await Promise.all( + [...failureReceipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "failure-receipts", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + auditReceipts.clear() + failureReceipts.clear() +}) + +describe("topic-aware adversarial failure discovery", () => { + test("requires canonical independent validators and enough budget to initialize every arm", () => { + expect(() => config({ budget: 2 })).toThrow("initialize every topic") + const valid = config() + expect(() => + HarnessContract.FailureDiscovery.parse({ + ...valid, + topics: [{ ...valid.topics[0]!, id: "__proto__" }, ...valid.topics.slice(1)], + }), + ).toThrow() + expect(() => + HarnessContract.FailureDiscovery.parse({ ...valid, validators: valid.validators.toReversed() }), + ).toThrow("canonical kind order") + expect(() => + HarnessContract.FailureDiscovery.parse({ + ...valid, + validators: valid.validators.map((item, index) => + index ? item : { ...item, identity: { ...valid.generator, name: "relabeled-generator" } }, + ), + }), + ).toThrow("distinct prompt/config commitments") + expect(() => + HarnessContract.Info.parse({ + schemaVersion: 1, + runID: "run", + sessionID: "session", + objective: "test", + benchmark: { + name: "statistics", + version: "1", + taskID: "task", + split: "validation", + evaluator: "evaluator", + evaluatorVersion: "1", + evaluatorSource: "benchmark", + }, + profile: "react", + audit: audit(), + failureDiscovery: { ...valid, failureThreshold: 0.7 }, + packs: ["statistics"], + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { steps: 10 }, + seed: 1, + intervention: "autonomous", + contamination: { policy: "hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }), + ).toThrow("must use the active audit failure threshold") + }) + + test("forces every topic once, then replays UCB1 from validated immutable rewards", async () => { + const contract = await bind("failure-ucb") + const initialized = await source(contract) + const before = await HarnessAudit.status(initialized.audit.auditID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + + const [alpha, retry] = await Promise.all( + Array.from({ length: 2 }, () => + HarnessFailure.next(initialized.stream.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }), + ), + ) + expect(retry).toEqual(alpha) + expect(alpha.topic.id).toBe("alpha") + expect(alpha.allocation.phase).toBe("initialization") + const first = await HarnessFailure.observe(initialized.stream.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: alpha.selectionID, + generation: { status: "failed", mode: "generator_error", evidence: ["evidence://generator-error"] }, + validations: [], + evaluatedAt: Math.max(Date.now(), alpha.selectedAt), + }) + const beta = await HarnessFailure.next(first.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + expect(beta.topic.id).toBe("beta") + const second = await HarnessFailure.observe(first.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: beta.selectionID, + generation: generated(1), + validations: validations().toReversed(), + outcome: outcome(1, true), + evaluatedAt: Math.max(Date.now(), beta.selectedAt), + }) + const third = await record(second, 2, { outcome: outcome(2, false), generation: generated(2, [0, 1]) }) + expect(third.attempts[2]!.selection.topic.id).toBe("gamma") + const selected = await HarnessFailure.next(third.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + expect(selected.topic.id).toBe("beta") + expect(selected.allocation).toMatchObject({ phase: "ucb1", pulls: 1, rewards: 1 }) + const completed = await HarnessFailure.observe(third.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: selected.selectionID, + generation: generated(3, [Math.SQRT1_2, Math.SQRT1_2]), + validations: validations(), + outcome: outcome(3, true), + evaluatedAt: Math.max(Date.now(), selected.selectedAt), + }) + expect(completed).toMatchObject({ + status: "completed", + stopReason: "budget_exhausted", + statistics: { + attempts: 4, + generated: 3, + admissible: 3, + failures: 2, + invalid: 1, + samplesToFirstFailure: 2, + failureRate: 2 / 3, + }, + }) + expect(completed.statistics.topicEntropy).toBe(0) + expect(completed.statistics.embeddingLogDet).toBeFinite() + expect(completed.statistics.topics.beta).toEqual({ pulls: 2, rewards: 2, rate: 1 }) + const receipt = await HarnessFailure.seal(completed.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + failureReceipts.add(receipt.receiptID) + expect( + await HarnessFailure.seal(completed.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }), + ).toEqual(receipt) + const after = await HarnessAudit.status(initialized.audit.auditID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + expect(after).toEqual(before) + + const evaluation = HarnessEvaluation.Info.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + failureDiscoveryReceiptID: receipt.receiptID, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status: "passed", + score: 0.2, + metrics: { loss: 0.2 }, + checks: ["estimand", "assumptions", "effect-size", "uncertainty", "multiplicity", "stat-replay"].map((id) => ({ + id, + status: "passed" as const, + blocking: true, + evidence: [`evidence://${id}`], + })), + evidence: ["evidence://official-score"], + evaluatedAt: Math.max(Date.now(), receipt.completedAt), + }) + const recorded = await HarnessEvaluation.record(evaluation) + expect(recorded.score).toBe(0.2) + expect(recorded.metrics).toEqual({ loss: 0.2 }) + expect(recorded.failureDiscoveryReceiptID).toBe(receipt.receiptID) + }) + + test("does not let a low failure target bypass topic initialization", async () => { + const contract = await bind("failure-target-initialization", config({ targetFailures: 1 })) + const initialized = await source(contract) + const first = await record(initialized.stream, 0) + expect(first).toMatchObject({ status: "active", statistics: { attempts: 1, failures: 1 } }) + expect(first.attempts[0]!.selection.topic.id).toBe("alpha") + expect( + await HarnessFailure.initialize({ + sessionID: contract.sessionID, + evaluatorToken: token, + subject: initialized.subject, + auditReceiptID: initialized.receipt.receiptID, + }), + ).toEqual(first) + const second = await record(first, 1, { generation: generated(1, [0, 1]) }) + expect(second).toMatchObject({ status: "active", statistics: { attempts: 2, failures: 2 } }) + expect(second.attempts[1]!.selection.topic.id).toBe("beta") + const completed = await record(second, 2, { + generation: generated(2, [Math.SQRT1_2, Math.SQRT1_2]), + }) + expect(completed).toMatchObject({ + status: "completed", + stopReason: "failure_target_reached", + statistics: { attempts: 3, failures: 3 }, + }) + expect(completed.attempts[2]!.selection.topic.id).toBe("gamma") + }) + + test("rejects substituted selections, malformed embeddings, and duplicate reward inflation", async () => { + const contract = await bind("failure-adversarial") + const initialized = await source(contract) + const selection = await HarnessFailure.next(initialized.stream.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + await expect( + HarnessFailure.observe(initialized.stream.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: digest("attacker-selected-topic"), + generation: generated(0), + validations: validations(), + outcome: outcome(0), + evaluatedAt: Math.max(Date.now(), selection.selectedAt), + }), + ).rejects.toThrow("does not match the server-selected") + await expect( + HarnessFailure.observe(initialized.stream.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: selection.selectionID, + generation: generated(0, [1, 1]), + validations: validations(), + outcome: outcome(0), + evaluatedAt: Math.max(Date.now(), selection.selectedAt), + }), + ).rejects.toThrow("L2-normalized") + const first = await HarnessFailure.observe(initialized.stream.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: selection.selectionID, + generation: generated(0), + validations: validations(), + outcome: outcome(0), + evaluatedAt: Math.max(Date.now(), selection.selectedAt), + }) + const next = await HarnessFailure.next(first.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + await expect( + HarnessFailure.observe(first.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: next.selectionID, + generation: generated(0), + validations: validations(), + outcome: outcome(1), + evaluatedAt: Math.max(Date.now(), next.selectedAt), + }), + ).rejects.toThrow("exact duplicate") + const invalid = validations().map((item) => + item.kind === "novelty" ? { ...item, status: "failed" as const } : item, + ) + const accepted = await HarnessFailure.observe(first.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: next.selectionID, + generation: generated(0), + validations: invalid, + evaluatedAt: Math.max(Date.now(), next.selectedAt), + }) + expect(accepted.statistics).toMatchObject({ attempts: 2, admissible: 1, failures: 1, invalid: 1 }) + expect(accepted.attempts[1]).toMatchObject({ admissible: false, reward: 0 }) + await expect( + HarnessFailure.observe(first.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + selectionID: next.selectionID, + generation: generated(2), + validations: validations(), + outcome: outcome(2), + evaluatedAt: Math.max(Date.now(), next.selectedAt), + }), + ).rejects.toThrow("immutable") + }) + + test("invalidates receipts after state or receipt tampering", async () => { + const contract = await bind("failure-tamper", config({ budget: 3 })) + const initialized = await source(contract) + const first = await record(initialized.stream, 0) + const second = await record(first, 1, { generation: generated(1, [0, 1]) }) + const completed = await record(second, 2, { + generation: generated(2, [Math.SQRT1_2, Math.SQRT1_2]), + outcome: outcome(2, false), + }) + const receipt = await HarnessFailure.seal(completed.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }) + failureReceipts.add(receipt.receiptID) + const stateFile = path.join( + Global.Path.data, + "harness", + "failures", + encodeURIComponent(contract.sessionID), + `${completed.streamID}.json`, + ) + const original = await fs.readFile(stateFile, "utf8") + const semantic = JSON.parse(original) + const attempt = semantic.attempts.at(-1) + attempt.outcome.loss = 0.1 + attempt.outcome.failure = true + attempt.attemptID = digest({ + selection: attempt.selection, + generation: attempt.generation, + validations: attempt.validations, + outcome: attempt.outcome, + admissible: attempt.admissible, + reward: attempt.reward, + evaluatedAt: attempt.evaluatedAt, + recordedAt: attempt.recordedAt, + }) + await fs.writeFile(stateFile, JSON.stringify(semantic)) + await expect( + HarnessFailure.status(completed.streamID, { + sessionID: contract.sessionID, + evaluatorToken: token, + }), + ).rejects.toThrow("cannot be replayed") + await fs.writeFile(stateFile, original) + const changed = JSON.parse(original) + changed.statistics.failures = 0 + await fs.writeFile(stateFile, JSON.stringify(changed)) + expect(await HarnessFailure.readReceipt(receipt.receiptID)).toBeNull() + await fs.writeFile(stateFile, original) + const auditFile = path.join( + Global.Path.data, + "harness", + "audits", + encodeURIComponent(contract.sessionID), + `${initialized.audit.auditID}.json`, + ) + const auditState = await fs.readFile(auditFile, "utf8") + const corruptedAudit = JSON.parse(auditState) + corruptedAudit.estimate.failures = 0 + await fs.writeFile(auditFile, JSON.stringify(corruptedAudit)) + expect(await HarnessFailure.readReceipt(receipt.receiptID)).toBeNull() + await fs.writeFile(auditFile, auditState) + const receiptFile = path.join(Global.Path.data, "harness", "failure-receipts", `${receipt.receiptID}.json`) + const saved = await fs.readFile(receiptFile, "utf8") + const forged = JSON.parse(saved) + forged.statistics.failureRate = 0 + await fs.writeFile(receiptFile, JSON.stringify(forged)) + expect(await HarnessFailure.readReceipt(receipt.receiptID)).toBeNull() + }) + + test("exposes stream initialization and selection only through the evaluator capability", async () => { + const contract = await bind("failure-route") + const initialized = await source(contract) + const app = HarnessRoutes() + const denied = await app.request("/failure-streams", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID: contract.sessionID, + evaluatorToken: "wrong-topic-aware-capability-token-0000000000000", + subject: initialized.subject, + auditReceiptID: initialized.receipt.receiptID, + }), + }) + expect(denied.status).not.toBe(200) + const response = await app.request("/failure-streams", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID: contract.sessionID, + evaluatorToken: token, + subject: initialized.subject, + auditReceiptID: initialized.receipt.receiptID, + }), + }) + expect(response.status).toBe(200) + const state = (await response.json()) as HarnessFailure.State + expect(state.streamID).toBe(initialized.stream.streamID) + const selected = await app.request(`/failure-streams/${state.streamID}/selection`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: token }), + }) + expect(selected.status).toBe(200) + const selection = (await selected.json()) as HarnessFailure.Selection + expect(selection).toMatchObject({ round: 1, topic: { id: "alpha" } }) + expect(JSON.stringify(selection)).not.toContain("hidden-probe") + expect(JSON.stringify(selection)).not.toContain(token) + }) +}) diff --git a/backend/cli/test/session/harness-formal.test.ts b/backend/cli/test/session/harness-formal.test.ts new file mode 100644 index 00000000..89a1da31 --- /dev/null +++ b/backend/cli/test/session/harness-formal.test.ts @@ -0,0 +1,445 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessFormal } from "../../src/session/harness/formal" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const receipts = new Set() +const evaluator = "formal-proof-evaluator-token-000000000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +const verifier = (role: HarnessContract.FormalVerifierRole) => ({ + role, + name: `test-${role}`, + version: "1", + artifactSHA256: hash(`verifier-${role}`), +}) + +function protocol( + tier: HarnessContract.FormalTier = "kernel", + relation: HarnessContract.FormalRelation = "exact_proof", +) { + const roles = + tier === "kernel" + ? (["lean_kernel", "source_auditor", "axiom_auditor"] as const) + : tier === "fresh_recheck" + ? (["lean_kernel", "source_auditor", "axiom_auditor", "fresh_rechecker"] as const) + : HarnessContract.FormalVerifierRole.options + return HarnessContract.FormalProof.parse({ + protocolVersion: "formal-proof-v1", + language: "lean4", + tier, + relation, + challengeSHA256: hash(`challenge-${relation}`), + statementSHA256: hash(`statement-${relation}`), + declaration: `OpenScience.${relation}`, + module: "OpenScience.Proof", + leanVersion: "4.33.0", + leanToolchainSHA256: hash("lean-toolchain"), + lakeManifestSHA256: hash("lake-manifest"), + dependencyTreeSHA256: hash("dependency-tree"), + verifiers: roles.map(verifier), + ...(tier === "external_crosscheck" ? { sandboxImageSHA256: hash("formal-sandbox") } : {}), + forbiddenConstructs: HarnessContract.FormalForbidden.options, + allowedAxioms: ["Classical.choice", "Quot.sound", "propext"].toSorted((a, b) => a.localeCompare(b)), + maxFiles: 32, + completeManifestRequired: true, + warningPolicy: "fail", + semanticPolicy: "formal_statement_only", + }) +} + +function task( + sessionID: string, + tier: HarnessContract.FormalTier = "kernel", + relation: HarnessContract.FormalRelation = "exact_proof", +): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "formal-proof", + split: "validation", + evaluator: { name: "official-formal-evaluator", version: "1", source: "benchmark", token: evaluator }, + formalProof: protocol(tier, relation), + objective: "Produce the exact frozen Lean result under the declared proof relation", + metric: { name: "accuracy", direction: "maximize", target: 0.8 }, + model: { provider: "test", name: "formal-agent" }, + tools: ["read", "bash"], + skills: [{ name: "verify-formal-proof" }], + budget: { steps: 20 }, + profile: "react", + seed: 47, + intervention: "autonomous", + contamination: { policy: "trusted challenge remains evaluator-owned", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const bound = (contract: HarnessContract.Info, role: HarnessContract.FormalVerifierRole) => + contract.formalProof!.verifiers.find((item) => item.role === role)!.artifactSHA256 + +function submit( + contract: HarnessContract.Info, + input: { + subject?: HarnessFormal.Subject + artifactSHA256?: string + startedAt?: number + warnings?: number + sourceComplete?: boolean + findings?: { construct: HarnessContract.FormalForbidden; path: string; line: number }[] + observed?: string[] + complete?: boolean + typesTraversed?: boolean + fresh?: boolean + freshExitCode?: number + sandboxed?: boolean + challengeMatched?: boolean + externalAccepted?: boolean + } = {}, +) { + if (!contract.formalProof) throw new Error("Expected formal proof protocol") + const artifactSHA256 = input.artifactSHA256 ?? hash(`${contract.sessionID}-proof`) + const files = [ + { path: "challenge.lean", role: "challenge" as const, sha256: contract.formalProof.challengeSHA256 }, + { path: "deps.json", role: "dependency_tree" as const, sha256: contract.formalProof.dependencyTreeSHA256 }, + { path: "lake-manifest.json", role: "lake_manifest" as const, sha256: contract.formalProof.lakeManifestSHA256 }, + { + path: "lean-toolchain", + role: "lean_toolchain" as const, + sha256: contract.formalProof.leanToolchainSHA256, + }, + { path: "proof.lean", role: "proof" as const, sha256: artifactSHA256 }, + { path: "statement.lean", role: "statement" as const, sha256: contract.formalProof.statementSHA256 }, + ] + const startedAt = input.startedAt ?? contract.createdAt + const endedAt = Math.max(Date.now(), startedAt) + const tier = contract.formalProof.tier + return HarnessFormal.Submit.parse({ + sessionID: contract.sessionID, + evaluatorToken: evaluator, + subject: input.subject ?? { type: "run", id: contract.runID }, + artifactSHA256, + relation: contract.formalProof.relation, + challengeSHA256: contract.formalProof.challengeSHA256, + statementSHA256: contract.formalProof.statementSHA256, + declaration: contract.formalProof.declaration, + module: contract.formalProof.module, + environment: { + leanVersion: contract.formalProof.leanVersion, + leanToolchainSHA256: contract.formalProof.leanToolchainSHA256, + lakeManifestSHA256: contract.formalProof.lakeManifestSHA256, + dependencyTreeSHA256: contract.formalProof.dependencyTreeSHA256, + }, + manifest: { complete: input.complete ?? true, files }, + verification: { + startedAt, + endedAt, + build: { + verifierArtifactSHA256: bound(contract, "lean_kernel"), + exitCode: 0, + warnings: input.warnings ?? 0, + transcriptSHA256: hash(`${contract.sessionID}-build`), + }, + source: { + verifierArtifactSHA256: bound(contract, "source_auditor"), + complete: input.sourceComplete ?? true, + findings: input.findings ?? [], + transcriptSHA256: hash(`${contract.sessionID}-source`), + }, + axioms: { + verifierArtifactSHA256: bound(contract, "axiom_auditor"), + complete: true, + typesTraversed: input.typesTraversed ?? true, + observed: input.observed ?? contract.formalProof.allowedAxioms, + transcriptSHA256: hash(`${contract.sessionID}-axioms`), + }, + ...(tier === "kernel" + ? {} + : { + fresh: { + verifierArtifactSHA256: bound(contract, "fresh_rechecker"), + fresh: input.fresh ?? true, + exitCode: input.freshExitCode ?? 0, + transcriptSHA256: hash(`${contract.sessionID}-fresh`), + }, + }), + ...(tier === "external_crosscheck" + ? { + external: { + comparatorArtifactSHA256: bound(contract, "sandbox_comparator"), + sandboxImageSHA256: contract.formalProof.sandboxImageSHA256, + sandboxed: input.sandboxed ?? true, + challengeMatched: input.challengeMatched ?? true, + proofTermSHA256: hash(`${contract.sessionID}-proof-term`), + transcriptSHA256: hash(`${contract.sessionID}-comparator`), + checks: [ + { + role: "lean_kernel", + verifierArtifactSHA256: bound(contract, "lean_kernel"), + accepted: true, + transcriptSHA256: hash(`${contract.sessionID}-external-lean`), + }, + { + role: "external_checker", + verifierArtifactSHA256: bound(contract, "external_checker"), + accepted: input.externalAccepted ?? true, + transcriptSHA256: hash(`${contract.sessionID}-external-independent`), + }, + ], + }, + } + : {}), + }, + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`evidence://${check.id}`], + })) + +function evaluation(contract: HarnessContract.Info, receipt?: HarnessFormal.Receipt) { + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: evaluator, + proofReceiptID: receipt?.receiptID, + status: "passed", + score: 0.9, + metrics: { accuracy: 0.9 }, + checks: checks(contract), + evidence: ["official://formal-proof-result"], + evaluatedAt: Math.max(Date.now(), receipt?.recordedAt ?? contract.createdAt), + }) +} + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + ...["bindings", "contracts", "evaluations", "reports", "search", "retrospectives"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + fs.rm(path.join(Global.Path.data, "harness", "formal", "subjects", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ]), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "formal", "receipts", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + receipts.clear() +}) + +describe("formal proof receipts", () => { + test("binds an exact kernel proof and gates final reporting", async () => { + const contract = await HarnessAdapter.bind(task("formal-pass")) + expect(contract.packs).toContain("formal") + expect(HarnessFormal.prompt(contract)).toContain("exact_proof") + expect(HarnessFormal.prompt(contract)).not.toContain(contract.formalProof!.challengeSHA256) + await expect(HarnessAdapter.ingest(evaluation(contract))).rejects.toThrow("formal proof receipt") + + const receipt = await HarnessFormal.record(submit(contract), contract) + receipts.add(receipt.receiptID) + expect(receipt).toMatchObject({ + status: "passed", + tier: "kernel", + relation: "exact_proof", + metrics: { buildAccepted: true, sourceAuditAccepted: true, axiomAuditAccepted: true, files: 6 }, + }) + expect(JSON.stringify(receipt)).not.toContain(evaluator) + await expect( + HarnessAdapter.ingest({ ...evaluation(contract, receipt), evaluatedAt: receipt.recordedAt - 1 }), + ).rejects.toThrow("predates") + + const result = await HarnessAdapter.ingest(evaluation(contract, receipt)) + const report = HarnessReport.compile({ contract, evaluations: [result.evaluation], formal: receipt }) + expect(report.execution.formal).toEqual({ tier: "kernel", relation: "exact_proof", status: "passed" }) + expect(report.quality.proofReceiptID).toBe(receipt.receiptID) + const stronger = HarnessContract.Info.parse({ + ...contract, + runID: `${contract.runID}-fresh`, + sessionID: `${contract.sessionID}-fresh`, + formalProof: protocol("fresh_recheck"), + }) + expect(report.comparisonKey).not.toBe(HarnessReport.compile({ contract: stronger, evaluations: [] }).comparisonKey) + }) + + test("fails warnings, unchecked source constructs, incomplete axiom traversal, and disallowed axioms", async () => { + const warningContract = await HarnessAdapter.bind(task("formal-warning")) + const warning = await HarnessFormal.record(submit(warningContract, { warnings: 1 }), warningContract) + receipts.add(warning.receiptID) + expect(warning.status).toBe("failed") + expect(warning.failures).toContain("Lean build failed or emitted warnings") + + const sourceContract = await HarnessAdapter.bind(task("formal-source-audit")) + const source = await HarnessFormal.record( + submit(sourceContract, { + findings: [{ construct: "debug.skipKernelTC", path: "proof.lean", line: 7 }], + }), + sourceContract, + ) + receipts.add(source.receiptID) + expect(source.metrics.sourceAuditAccepted).toBe(false) + expect(source.failures).toContain("forbidden construct debug.skipKernelTC at proof.lean:7") + + const traversalContract = await HarnessAdapter.bind(task("formal-axiom-types")) + const traversal = await HarnessFormal.record( + submit(traversalContract, { typesTraversed: false }), + traversalContract, + ) + receipts.add(traversal.receiptID) + expect(traversal.failures).toContain("axiom audit did not traverse axiom types") + + const sorryContract = await HarnessAdapter.bind(task("formal-sorry")) + const sorry = await HarnessFormal.record( + submit(sorryContract, { + observed: [...sorryContract.formalProof!.allowedAxioms, "sorryAx"].toSorted((a, b) => a.localeCompare(b)), + }), + sorryContract, + ) + receipts.add(sorry.receiptID) + expect(sorry.failures).toContain("disallowed axiom: sorryAx") + await expect(HarnessAdapter.ingest(evaluation(sorryContract, sorry))).rejects.toThrow( + "passing formal proof receipt", + ) + + expect(() => + HarnessContract.FormalProof.parse({ ...protocol(), allowedAxioms: ["Classical.choice", "sorryAx"] }), + ).toThrow("never allow sorryAx") + expect(() => + HarnessContract.FormalProof.parse({ ...protocol(), forbiddenConstructs: ["sorry", "admit"] }), + ).toThrow() + }) + + test("enforces fresh replay and sandboxed independent checker tiers", async () => { + const freshContract = await HarnessAdapter.bind(task("formal-fresh", "fresh_recheck")) + const failedFresh = await HarnessFormal.record(submit(freshContract, { fresh: false }), freshContract) + receipts.add(failedFresh.receiptID) + expect(failedFresh.failures).toContain("fresh kernel replay failed") + + const missing = submit(freshContract) + delete missing.verification.fresh + await expect(HarnessFormal.record(missing, freshContract)).rejects.toThrow("fresh-recheck tier") + + const externalContract = await HarnessAdapter.bind(task("formal-external", "external_crosscheck")) + const failedExternal = await HarnessFormal.record( + submit(externalContract, { challengeMatched: false, externalAccepted: false }), + externalContract, + ) + receipts.add(failedExternal.receiptID) + expect(failedExternal.failures).toContain("external comparator did not match the trusted challenge") + expect(failedExternal.failures).toContain("sandboxed independent cross-check failed") + + const swapped = submit(externalContract) + swapped.verification.external!.checks[1]!.verifierArtifactSHA256 = hash("substituted-external-checker") + await expect(HarnessFormal.record(swapped, externalContract)).rejects.toThrow("external_checker verifier") + + const passContract = await HarnessAdapter.bind(task("formal-external-pass", "external_crosscheck")) + const passed = await HarnessFormal.record(submit(passContract), passContract) + receipts.add(passed.receiptID) + expect(passed).toMatchObject({ status: "passed", tier: "external_crosscheck" }) + }) + + test("rejects relation, environment, manifest, candidate, and canonical-receipt laundering", async () => { + const changed = task("formal-candidate") + changed.profile = "optimize" + changed.budget = { ...changed.budget, candidates: 2 } + const contract = await HarnessAdapter.bind(changed) + await HarnessSearch.initialize({ sessionID: contract.sessionID }) + const recommendation = HarnessSearch.recommend(await HarnessSearch.read(contract.sessionID)) + const artifactSHA256 = hash("formal-registered-candidate") + const candidate = await HarnessSearch.add({ + sessionID: contract.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "formal", + proposal: "candidate Lean proof", + artifact: { uri: "candidate://formal-proof", sha256: artifactSHA256 }, + }) + const subject = { type: "candidate" as const, id: candidate.id } + const startedAt = Date.now() + await expect( + HarnessFormal.record( + submit(contract, { subject, artifactSHA256: hash("substituted-proof"), startedAt }), + contract, + ), + ).rejects.toThrow("changed the candidate artifact") + + const relation = submit(contract, { subject, artifactSHA256, startedAt }) + relation.relation = "repaired_proof" + await expect(HarnessFormal.record(relation, contract)).rejects.toThrow("claim relation") + const environment = submit(contract, { subject, artifactSHA256, startedAt }) + environment.environment.leanVersion = "4.34.0" + await expect(HarnessFormal.record(environment, contract)).rejects.toThrow("Lean environment") + const manifest = submit(contract, { subject, artifactSHA256, startedAt }) + manifest.manifest.files[4]!.sha256 = hash("manifest-proof-swap") + await expect(HarnessFormal.record(manifest, contract)).rejects.toThrow("manifest does not bind") + const auditor = submit(contract, { subject, artifactSHA256, startedAt }) + auditor.verification.source.verifierArtifactSHA256 = hash("substituted-source-auditor") + await expect(HarnessFormal.record(auditor, contract)).rejects.toThrow("source audit used an unbound verifier") + const outside = submit(contract, { + subject, + artifactSHA256, + startedAt, + findings: [{ construct: "sorry", path: "unlisted.lean", line: 1 }], + }) + await expect(HarnessFormal.record(outside, contract)).rejects.toThrow("outside the complete manifest") + + const receipt = await HarnessFormal.record(submit(contract, { subject, artifactSHA256, startedAt }), contract) + receipts.add(receipt.receiptID) + const replacement = submit(contract, { + subject, + artifactSHA256, + observed: [], + startedAt, + }) + await expect(HarnessFormal.record(replacement, contract)).rejects.toThrow("canonical receipt") + }) + + test("protects proof routes with evaluator capability and fails closed on disk tampering", async () => { + const contract = await HarnessAdapter.bind(task("formal-route")) + const app = HarnessRoutes() + const response = await app.request("/proofs/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(submit(contract)), + }) + expect(response.status).toBe(200) + const receipt = HarnessFormal.Receipt.parse(await response.json()) + receipts.add(receipt.receiptID) + + await expect( + HarnessAdapter.authorize(contract.sessionID, "wrong-formal-proof-token-000000000000000000"), + ).rejects.toThrow("capability was rejected") + const read = await app.request(`/proofs/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: evaluator }), + }) + expect(read.status).toBe(200) + + const file = path.join(Global.Path.data, "harness", "formal", "receipts", `${receipt.receiptID}.json`) + await Bun.write(file, JSON.stringify({ ...receipt, status: "failed" })) + expect(await HarnessFormal.readReceipt(receipt.receiptID)).toBeNull() + }) +}) diff --git a/backend/cli/test/session/harness-integrity.test.ts b/backend/cli/test/session/harness-integrity.test.ts new file mode 100644 index 00000000..02f58874 --- /dev/null +++ b/backend/cli/test/session/harness-integrity.test.ts @@ -0,0 +1,347 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessIntegrity } from "../../src/session/harness/integrity" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const token = "integrity-evaluator-capability-token-00000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "integrity", "reports", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await fs.rm(path.join(Global.Path.data, "harness", "retrospectives"), { recursive: true, force: true }) + sessions.clear() +}) + +function protocol() { + return HarnessContract.Integrity.parse({ + protocolVersion: "benchmark-integrity-v1", + validatorSHA256: hash("verify-benchmark-integrity.py:v1"), + traceSchemaSHA256: hash("normalized-trace-schema:v1"), + minEvents: 4, + minCoverage: 0.95, + assignedModel: { + name: "assigned-base-model", + baseArtifactSHA256: hash("assigned-base-weights"), + configSHA256: hash("assigned-base-config"), + }, + forbiddenModelArtifacts: [hash("forbidden-instruct-model")], + policy: { + testItemDerivation: "forbidden", + unapprovedExternalModels: "forbidden", + benchmarkLookup: "forbidden", + }, + auditors: HarnessContract.IntegrityAuditKind.options.map((kind) => ({ + kind, + name: `${kind}-auditor`, + version: "2026.08", + promptSHA256: hash(`${kind}-prompt:v1`), + })), + hiddenCanaryManifestSHA256: hash("hidden-canaries:v1"), + minHiddenCanaries: 2, + }) +} + +function task(sessionID: string): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "mle", + version: "2026.08", + taskID: "integrity-task-1", + split: "validation", + evaluator: { name: "official-integrity-evaluator", version: "5", source: "benchmark", token }, + objective: "Improve the benchmark while proving the execution obeyed its hidden-data and model-use policy", + integrity: protocol(), + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "research-agent" }, + tools: ["read", "bash"], + skills: [], + budget: { steps: 30, candidates: 2 }, + seed: 41, + intervention: "autonomous", + contamination: { policy: "specific test items cannot inform training data", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +function submit(contract: HarnessContract.Info, subject?: HarnessIntegrity.Submit["subject"]) { + const integrity = contract.integrity + if (!integrity) throw new Error("Expected a runtime integrity protocol") + return HarnessIntegrity.Submit.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + protocol: integrity, + subject: subject ?? { + type: "run", + id: contract.runID, + artifact: { uri: "artifact:run-output", sha256: hash("run-output") }, + }, + trace: { + artifact: { uri: "artifact:normalized-trace.jsonl", sha256: hash("normalized-trace") }, + schemaSHA256: integrity.traceSchemaSHA256, + events: 100, + dropped: 0, + startedAt: contract.createdAt + 1, + endedAt: contract.createdAt + 2, + }, + model: { + name: integrity.assignedModel.name, + baseArtifactSHA256: integrity.assignedModel.baseArtifactSHA256, + configSHA256: integrity.assignedModel.configSHA256, + outputArtifactSHA256: hash("fine-tuned-output"), + lineageVerified: true, + }, + audits: integrity.auditors.map((auditor) => ({ + ...auditor, + decision: "clean" as const, + confidence: 0.99, + evidence: [`artifact:${auditor.kind}-audit.json`], + })), + activity: { + unapprovedExternalModelCalls: 0, + benchmarkLookupEvents: 0, + hiddenCanaryManifestSHA256: integrity.hiddenCanaryManifestSHA256, + hiddenCanariesTested: 3, + hiddenCanaryViolations: 0, + }, + validator: { + name: "verify-benchmark-integrity", + version: 1, + scriptSHA256: integrity.validatorSHA256, + }, + evidence: ["artifact:integrity-report.json"], + evaluatedAt: contract.createdAt + 3, + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + +function evaluation(contract: HarnessContract.Info, receiptID?: string, candidateID?: string) { + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + candidateID, + integrityReceiptID: receiptID, + status: "passed", + score: 0.84, + metrics: { score: 0.84 }, + checks: checks(contract), + evidence: ["official:score.json"], + evaluatedAt: contract.createdAt + 4, + }) +} + +describe("trace-backed benchmark runtime integrity", () => { + test("gates final success on a subject-matched, backend-derived passing receipt", async () => { + const contract = await HarnessAdapter.bind(task("integrity-gate")) + await expect(HarnessAdapter.ingest(evaluation(contract))).rejects.toThrow("must reference") + + const receipt = await HarnessIntegrity.record(submit(contract), contract) + expect(receipt).toMatchObject({ + status: "passed", + traceCoverage: 1, + checks: { + traceCompleteness: true, + modelIdentity: true, + testItemContamination: true, + externalModelUse: true, + benchmarkLookup: true, + hiddenCanary: true, + }, + failures: [], + }) + expect(JSON.stringify(receipt)).not.toContain(token) + expect((await HarnessIntegrity.record(submit(contract), contract)).receiptID).toBe(receipt.receiptID) + + const result = await HarnessAdapter.ingest(evaluation(contract, receipt.receiptID)) + expect(result.evaluation).toMatchObject({ status: "passed", integrityReceiptID: receipt.receiptID }) + const report = HarnessReport.compile({ contract, evaluations: [result.evaluation] }) + expect(report.quality.integrityReceiptID).toBe(receipt.receiptID) + const changed = HarnessContract.Info.parse({ + ...contract, + integrity: { ...contract.integrity!, minCoverage: 0.99 }, + }) + expect(HarnessReport.compile({ contract: changed, evaluations: [] }).comparisonKey).not.toBe(report.comparisonKey) + }) + + test("derives every integrity failure instead of accepting a candidate verdict", async () => { + const contract = await HarnessAdapter.bind(task("integrity-failures")) + const input = submit(contract) + const receipt = await HarnessIntegrity.record( + { + ...input, + trace: { ...input.trace, schemaSHA256: hash("substituted-schema"), events: 1, dropped: 3 }, + model: { + ...input.model, + name: "substituted-model", + baseArtifactSHA256: hash("substituted-base"), + configSHA256: hash("substituted-config"), + outputArtifactSHA256: contract.integrity!.forbiddenModelArtifacts[0]!, + lineageVerified: false, + }, + audits: input.audits.map((audit) => ({ ...audit, decision: "flagged" as const })), + activity: { + unapprovedExternalModelCalls: 2, + benchmarkLookupEvents: 1, + hiddenCanaryManifestSHA256: hash("substituted-canaries"), + hiddenCanariesTested: 1, + hiddenCanaryViolations: 1, + }, + }, + contract, + ) + expect(receipt.status).toBe("failed") + expect(receipt.traceCoverage).toBe(0.25) + expect(receipt.failures.toSorted()).toEqual(HarnessIntegrity.Failure.options.toSorted()) + expect(Object.values(receipt.checks).every((value) => !value)).toBe(true) + await expect(HarnessAdapter.ingest(evaluation(contract, receipt.receiptID))).rejects.toThrow( + "requires a passing runtime integrity receipt", + ) + }) + + test("rejects validator, protocol, auditor, and temporal substitution", async () => { + const contract = await HarnessAdapter.bind(task("integrity-substitution")) + const input = submit(contract) + await expect( + HarnessIntegrity.record( + { ...input, validator: { ...input.validator, scriptSHA256: hash("substituted-validator") } }, + contract, + ), + ).rejects.toThrow("validator does not match") + await expect( + HarnessIntegrity.record({ ...input, protocol: { ...input.protocol, minCoverage: 0.9 } }, contract), + ).rejects.toThrow("immutable harness contract") + await expect( + HarnessIntegrity.record( + { + ...input, + audits: input.audits.map((audit) => + audit.kind === "benchmark_lookup" ? { ...audit, name: "substituted-auditor" } : audit, + ), + }, + contract, + ), + ).rejects.toThrow("auditors do not match") + await expect(HarnessIntegrity.record({ ...input, evaluatedAt: input.trace.endedAt - 1 }, contract)).rejects.toThrow( + "predates the trace end", + ) + expect(() => + HarnessContract.Integrity.parse({ + ...protocol(), + auditors: protocol().auditors.map((auditor) => ({ + ...auditor, + name: "one-auditor", + version: "one-version", + promptSHA256: hash("one-prompt"), + })), + }), + ).toThrow("identities must be distinct") + await expect( + HarnessAdapter.bind({ + ...task("integrity-score-evaluator-reuse"), + integrity: { + ...protocol(), + auditors: protocol().auditors.map((auditor, index) => + index ? auditor : { ...auditor, name: "official-integrity-evaluator", version: "5" }, + ), + }, + }), + ).rejects.toThrow("distinct from the score evaluator") + expect(await HarnessIntegrity.list(contract.sessionID)).toEqual([]) + }) + + test("binds candidate receipts to exact artifacts and prevents cross-subject reuse", async () => { + const contract = await HarnessAdapter.bind(task("integrity-candidate")) + const search = await HarnessSearch.initialize({ sessionID: contract.sessionID, candidates: 2 }) + const recommendation = HarnessSearch.recommend(search) + const added = await HarnessSearch.add({ + sessionID: contract.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "lineage-safe", + proposal: "Use only the assigned base model and public training corpus", + artifact: { uri: "artifact:candidate", sha256: hash("candidate") }, + }) + const subject = { + type: "candidate" as const, + id: added.id, + artifact: added.state.candidates[added.id]!.artifact, + } + await expect( + HarnessIntegrity.record( + { + ...submit(contract, subject), + subject: { ...subject, artifact: { ...subject.artifact, sha256: hash("other-candidate") } }, + }, + contract, + ), + ).rejects.toThrow("does not match the candidate artifact") + + const run = await HarnessIntegrity.record(submit(contract), contract) + await expect(HarnessAdapter.ingest(evaluation(contract, run.receiptID, added.id))).rejects.toThrow( + "does not match the evaluated subject", + ) + const receipt = await HarnessIntegrity.record(submit(contract, subject), contract) + const result = await HarnessAdapter.ingest(evaluation(contract, receipt.receiptID, added.id)) + expect(result.search?.bestID).toBe(added.id) + }) + + test("protects route access and fails closed when journal outcomes are edited", async () => { + const contract = await HarnessAdapter.bind(task("integrity-route")) + const app = HarnessRoutes() + const recorded = await app.request("/integrity/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(submit(contract)), + }) + expect(recorded.status).toBe(200) + const receipt = (await recorded.json()) as HarnessIntegrity.Info + + const denied = await app.request(`/integrity/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: "x".repeat(48) }), + }) + expect(denied.status).not.toBe(200) + const read = await app.request(`/integrity/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: token }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, status: "passed" }) + + const target = path.join(Global.Path.data, "harness", "integrity", `${encodeURIComponent(contract.sessionID)}.json`) + const state = (await Bun.file(target).json()) as { items: Record } + state.items[receipt.receiptID]!.status = "failed" + await Bun.write(target, JSON.stringify(state)) + await expect(HarnessIntegrity.list(contract.sessionID)).rejects.toThrow("content hash is invalid") + }) +}) diff --git a/backend/cli/test/session/harness-intervention.test.ts b/backend/cli/test/session/harness-intervention.test.ts new file mode 100644 index 00000000..658b6904 --- /dev/null +++ b/backend/cli/test/session/harness-intervention.test.ts @@ -0,0 +1,548 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessEvolution } from "../../src/session/harness/evolution" +import { HarnessIntervention } from "../../src/session/harness/intervention" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const token = "intervention-evaluator-capability-token-000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const stable = (input: unknown): unknown => { + if (Array.isArray(input)) return input.map(stable) + if (!input || typeof input !== "object") return input + return Object.fromEntries( + Object.entries(input as Record) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => [key, stable(value)]), + ) +} +const digest = (input: unknown) => hash(JSON.stringify(stable(input))) + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + ...["bindings", "contracts", "evaluations", "evolution", "reports", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + fs.rm(path.join(Global.Path.data, "harness", "interventions", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ]), + ) + await fs.rm(path.join(Global.Path.data, "harness", "retrospectives"), { recursive: true, force: true }) + sessions.clear() +}) + +function evolution() { + return HarnessContract.Evolution.parse({ + protocolVersion: "evolution-trace-v1", + validatorSHA256: hash("trace-evolutionary-candidate.py:v1"), + manifestSchemaSHA256: hash("evolution-source-manifest:v1"), + lineAlgorithm: "sha256-exact-line-v1", + roots: ["src"], + extensions: [".ts"], + exclude: [], + maxFiles: 100, + maxFileBytes: 100_000, + maxTotalBytes: 1_000_000, + maxSourceLines: 10_000, + maxChangedLines: 1_000, + }) +} + +function interventions() { + return HarnessContract.Interventions.parse({ + protocolVersion: "intervention-study-v1", + validatorSHA256: hash("design-replay-interventions.py:v1"), + requiredForPromotion: true, + minPairs: 3, + maxPairs: 4, + maxTotalPairs: 12, + confidence: 0.95, + required: ["model_transfer", "replay", "retune"], + rules: [ + { family: "model_transfer", mode: "max_regression", threshold: 0.05 }, + { family: "replay", mode: "max_absolute_effect", threshold: 0.01 }, + { family: "retune", mode: "min_effect", threshold: 0.1 }, + ], + }) +} + +function task(sessionID: string): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "mle", + version: "2026.08", + taskID: "controlled-intervention-task", + split: "held_out", + evaluator: { name: "official-runner", version: "3", source: "benchmark", token }, + objective: "Distinguish stable structural improvement from tuning and evaluator coupling", + evolution: evolution(), + interventions: interventions(), + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "research-agent" }, + tools: ["read", "bash"], + skills: [ + { name: "trace-evolutionary-candidate", version: "1", sha256: evolution().validatorSHA256 }, + { name: "design-replay-interventions", version: "1", sha256: interventions().validatorSHA256 }, + ], + budget: { steps: 80, candidates: 2 }, + seed: 53, + intervention: "autonomous", + contamination: { policy: "hidden evaluator remains external", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + +function snapshot(contract: HarnessContract.Info, content: string) { + const protocol = contract.evolution! + const files = HarnessEvolution.Files.parse([ + { + path: "src/main.ts", + sha256: hash(content), + bytes: new TextEncoder().encode(content).byteLength, + lineHashes: content.split("\n").flatMap((line) => (line ? [hash(line)] : [])), + }, + ]) + return HarnessEvolution.Snapshot.parse({ + artifact: { uri: "artifact:candidate-manifest.json", sha256: HarnessEvolution.manifestSHA256(protocol, files) }, + schemaSHA256: protocol.manifestSchemaSHA256, + files, + }) +} + +async function candidate(contract: HarnessContract.Info) { + const state = await HarnessSearch.read(contract.sessionID) + const recommendation = HarnessSearch.recommend(state) + const result = await HarnessSearch.add({ + sessionID: contract.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "structural-winner", + proposal: "Evaluate a replayable structural candidate under controlled interventions", + artifact: { uri: "artifact:structural-winner.tar.zst", sha256: hash("structural-winner") }, + }) + return result.state.candidates[result.id]! +} + +async function trace(contract: HarnessContract.Info, item: HarnessSearch.Candidate) { + const source = snapshot(contract, "export function solve() { return 42 }\n") + return HarnessEvolution.record( + { + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + protocol: contract.evolution!, + subject: { type: "candidate", id: item.id, artifact: item.artifact }, + snapshot: source, + parents: [], + validator: { + name: "trace-evolutionary-candidate", + version: 1, + scriptSHA256: contract.evolution!.validatorSHA256, + }, + evidence: ["artifact:trace-report.json"], + evaluatedAt: Date.now(), + }, + contract, + ) +} + +function artifact(name: string) { + return { uri: `artifact:${name}`, sha256: hash(name) } +} + +function condition(seed: number): HarnessIntervention.Condition { + return HarnessIntervention.Condition.parse({ + seed, + model: { provider: "test", name: "primary-model", version: "1" }, + context: artifact("primary-context"), + evaluator: { name: "official-runner", version: "3", source: "benchmark" }, + split: { name: "held_out", manifest: artifact("held-out-split") }, + environment: artifact("locked-environment"), + budget: artifact("matched-budget"), + }) +} + +function pairs(item: HarnessSearch.Candidate) { + return [0, 1, 2].flatMap((index) => { + const base = condition(100 + index) + const subject = { artifact: item.artifact, condition: base } + return [ + { + family: "model_transfer" as const, + index, + control: subject, + arm: { + artifact: item.artifact, + condition: { ...base, model: { provider: "test", name: "transfer-model", version: "2" } }, + }, + change: artifact(`model-transfer-${index}`), + }, + { + family: "replay" as const, + index, + control: subject, + arm: subject, + change: artifact(`replay-${index}`), + }, + { + family: "retune" as const, + index, + control: { artifact: artifact(`retuned-baseline-${index}`), condition: base }, + arm: subject, + change: artifact(`retune-${index}`), + }, + ] + }) +} + +function initialize(contract: HarnessContract.Info, item: HarnessSearch.Candidate, receipt: HarnessEvolution.Info) { + return HarnessIntervention.Initialize.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + subject: { type: "candidate", id: item.id, artifact: item.artifact }, + evolutionReceiptID: receipt.receiptID, + validator: { + name: "design-replay-interventions", + version: 1, + scriptSHA256: contract.interventions!.validatorSHA256, + }, + pairs: pairs(item), + }) +} + +async function setup(sessionID: string) { + const contract = await HarnessAdapter.bind(task(sessionID)) + await HarnessSearch.initialize({ sessionID, candidates: 2 }) + const item = await candidate(contract) + const evolution = await trace(contract, item) + return { contract, item, evolution } +} + +function scores(family: HarnessContract.InterventionFamily, role: "control" | "arm") { + if (family === "retune") return role === "control" ? 0.7 : 0.9 + if (family === "model_transfer") return role === "control" ? 0.9 : 0.88 + return 0.9 +} + +async function observeAll( + contract: HarnessContract.Info, + item: HarnessSearch.Candidate, + state: HarnessIntervention.State, +) { + for (const pair of state.plan.pairs) { + for (const role of ["control", "arm"] as const) { + await HarnessIntervention.observe( + item.id, + { + schemaVersion: 1, + sessionID: contract.sessionID, + evaluatorToken: token, + pairID: pair.pairID, + role, + targetSHA256: digest(pair[role]), + status: "passed", + score: scores(pair.family, role), + evidence: [`artifact:${pair.family}-${pair.index}-${role}.json`], + evaluatedAt: Date.now(), + }, + contract, + ) + } + } +} + +function evaluation( + contract: HarnessContract.Info, + item: HarnessSearch.Candidate, + evolution: HarnessEvolution.Info, + interventionReceiptID?: string, + evaluatedAt = Date.now(), +) { + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + candidateID: item.id, + evolutionReceiptID: evolution.receiptID, + interventionReceiptID, + status: "passed", + score: 0.9, + metrics: { score: 0.9 }, + checks: checks(contract), + evidence: ["official:held-out-score.json"], + evaluatedAt, + }) +} + +describe("evaluator-owned controlled replay interventions", () => { + test("derives replay stability, tuning gap, and model-transfer robustness without becoming fitness", async () => { + const { contract, item, evolution } = await setup("intervention-pass") + const state = await HarnessIntervention.initialize(initialize(contract, item, evolution), contract) + if (!state) throw new Error("Expected a frozen intervention plan") + expect((await HarnessSearch.read(contract.sessionID)).candidates[item.id]!.result).toBeUndefined() + await expect(HarnessIntervention.assess(contract.sessionID, item.id, contract)).rejects.toThrow("every frozen pair") + await observeAll(contract, item, state) + const receipt = await HarnessIntervention.assess(contract.sessionID, item.id, contract) + expect(receipt).toMatchObject({ status: "passed", subject: { id: item.id } }) + expect(receipt.families.map((family) => ({ family: family.family, verdict: family.verdict }))).toEqual([ + { family: "model_transfer", verdict: "passed" }, + { family: "replay", verdict: "passed" }, + { family: "retune", verdict: "passed" }, + ]) + expect(receipt.families[0]!.meanEffect).toBeCloseTo(-0.02) + expect(receipt.families[1]!.maxAbsoluteEffect).toBe(0) + expect(receipt.families[2]!.meanEffect).toBeCloseTo(0.2) + await expect( + HarnessIntervention.assert({ + contract, + receiptID: receipt.receiptID, + candidateID: item.id, + evolutionReceiptID: hash("substituted-evolution-receipt"), + requirePassed: true, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + }), + ).rejects.toThrow("does not match the evaluation's evolution receipt") + const untraced = evaluation(contract, item, evolution, receipt.receiptID) + untraced.status = "failed" + delete untraced.evolutionReceiptID + await expect(HarnessAdapter.ingest(untraced)).rejects.toThrow("exact evolution receipt") + await expect(HarnessAdapter.ingest(evaluation(contract, item, evolution))).rejects.toThrow( + "controlled intervention receipt", + ) + const result = await HarnessAdapter.ingest( + evaluation(contract, item, evolution, receipt.receiptID, Math.max(Date.now(), receipt.observedAt) + 1), + ) + expect(result.search?.bestID).toBe(item.id) + expect(result.evaluation.interventionReceiptID).toBe(receipt.receiptID) + const report = HarnessReport.compile({ contract, evaluations: [result.evaluation], search: result.search }) + expect(report.quality.interventionReceiptID).toBe(receipt.receiptID) + }) + + test("rejects extra differences, validator substitution, wrong targets, and mutable outcomes", async () => { + const { contract, item, evolution } = await setup("intervention-substitution") + const input = initialize(contract, item, evolution) + const first = input.pairs.find((pair) => pair.family === "model_transfer")! + await expect( + HarnessIntervention.initialize( + { + ...input, + pairs: input.pairs.map((pair) => + pair !== first + ? pair + : { + ...pair, + arm: { + ...pair.arm, + condition: { ...pair.arm.condition, context: artifact("also-substituted-context") }, + }, + }, + ), + }, + contract, + ), + ).rejects.toThrow("may change only model") + await expect( + HarnessIntervention.initialize( + { ...input, validator: { ...input.validator, scriptSHA256: hash("substituted-validator") } }, + contract, + ), + ).rejects.toThrow("validator does not match") + const state = await HarnessIntervention.initialize(input, contract) + if (!state) throw new Error("Expected a frozen intervention plan") + const pair = state.plan.pairs[0]! + const base = { + schemaVersion: 1 as const, + sessionID: contract.sessionID, + evaluatorToken: token, + pairID: pair.pairID, + role: "control" as const, + targetSHA256: digest(pair.control), + status: "passed" as const, + score: 0.9, + evidence: ["artifact:control.json"], + evaluatedAt: Date.now(), + } + await expect( + HarnessIntervention.observe(item.id, { ...base, targetSHA256: hash("wrong-target") }, contract), + ).rejects.toThrow("does not match the frozen pair") + const recorded = await HarnessIntervention.observe(item.id, base, contract) + expect((await HarnessIntervention.observe(item.id, base, contract)).outcomeID).toBe(recorded.outcomeID) + await expect(HarnessIntervention.observe(item.id, { ...base, score: 0.8 }, contract)).rejects.toThrow( + "immutable once recorded", + ) + }) + + test("fails closed on incomplete execution, temporal post-selection, and stored derivation tampering", async () => { + const { contract, item, evolution } = await setup("intervention-tamper") + const state = await HarnessIntervention.initialize(initialize(contract, item, evolution), contract) + if (!state) throw new Error("Expected a frozen intervention plan") + await observeAll(contract, item, state) + const receipt = await HarnessIntervention.assess(contract.sessionID, item.id, contract) + await expect( + HarnessIntervention.observe( + item.id, + { + schemaVersion: 1, + sessionID: contract.sessionID, + evaluatorToken: token, + pairID: state.plan.pairs[0]!.pairID, + role: "control", + targetSHA256: digest(state.plan.pairs[0]!.control), + status: "passed", + score: 0.9, + evidence: ["artifact:late.json"], + evaluatedAt: Date.now(), + }, + contract, + ), + ).rejects.toThrow("closed after assessment") + await expect( + HarnessAdapter.ingest(evaluation(contract, item, evolution, receipt.receiptID, receipt.observedAt - 1)), + ).rejects.toThrow("predates its controlled intervention observations") + + const target = path.join( + Global.Path.data, + "harness", + "interventions", + encodeURIComponent(contract.sessionID), + `${item.id}.json`, + ) + const stored = (await Bun.file(target).json()) as HarnessIntervention.State + const original = structuredClone(stored) + stored.receipt!.families[0]!.verdict = "failed" + await Bun.write(target, JSON.stringify(stored)) + expect(await HarnessIntervention.read(contract.sessionID, item.id)).toBeNull() + await expect( + HarnessIntervention.assert({ + contract, + receiptID: receipt.receiptID, + candidateID: item.id, + evolutionReceiptID: evolution.receiptID, + requirePassed: true, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + }), + ).rejects.toThrow("Unknown or corrupt") + + const outcome = structuredClone(original.outcomes[original.order[0]!]!) + const forged = structuredClone(original) + outcome.submissionID = hash("forged-submission") + const payload = structuredClone(outcome) as Record + delete payload.outcomeID + outcome.outcomeID = digest(payload) + delete forged.outcomes[forged.order[0]!] + forged.outcomes[outcome.outcomeID] = outcome + forged.order[0] = outcome.outcomeID + await Bun.write(target, JSON.stringify(forged)) + expect(await HarnessIntervention.read(contract.sessionID, item.id)).toBeNull() + + const protocol = structuredClone(original) + protocol.plan.protocol.minPairs = 4 + const plan = structuredClone(protocol.plan) as unknown as Record + delete plan.planID + protocol.plan.planID = digest(plan) + protocol.receipt!.planID = protocol.plan.planID + const assessment = structuredClone(protocol.receipt!) as unknown as Record + delete assessment.receiptID + protocol.receipt!.receiptID = digest(assessment) + await Bun.write(target, JSON.stringify(protocol)) + expect(await HarnessIntervention.read(contract.sessionID, item.id)).toBeNull() + + const swapped = structuredClone(original) + swapped.plan.protocol.requiredForPromotion = false + const swappedPlan = structuredClone(swapped.plan) as unknown as Record + delete swappedPlan.planID + swapped.plan.planID = digest(swappedPlan) + swapped.receipt!.planID = swapped.plan.planID + const swappedReceipt = structuredClone(swapped.receipt!) as unknown as Record + delete swappedReceipt.receiptID + swapped.receipt!.receiptID = digest(swappedReceipt) + await Bun.write(target, JSON.stringify(swapped)) + expect(await HarnessIntervention.read(contract.sessionID, item.id)).not.toBeNull() + await expect( + HarnessIntervention.assert({ + contract, + receiptID: swapped.receipt!.receiptID, + candidateID: item.id, + evolutionReceiptID: evolution.receiptID, + requirePassed: true, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + }), + ).rejects.toThrow("does not match the immutable harness contract") + }) + + test("protects intervention routes with the evaluator capability", async () => { + const { contract, item, evolution } = await setup("intervention-routes") + const app = HarnessRoutes() + const created = await app.request("/interventions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(initialize(contract, item, evolution)), + }) + expect(created.status).toBe(200) + const state = (await created.json()) as HarnessIntervention.State + expect(JSON.stringify(state)).not.toContain(token) + const denied = await app.request(`/interventions/${item.id}/status`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: "x".repeat(48) }), + }) + expect(denied.status).not.toBe(200) + const read = await app.request(`/interventions/${item.id}/status`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: token }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ plan: { planID: state.plan.planID, subject: { id: item.id } } }) + }) + + test("keeps studies optional and restricts them to traced numeric held-out optimization", async () => { + const legacy = task("intervention-legacy") + delete legacy.interventions + expect((await HarnessAdapter.bind(legacy)).interventions).toBeUndefined() + const base = await HarnessAdapter.bind(task("intervention-contract")) + expect(() => HarnessContract.Info.parse({ ...base, profile: "theory" })).toThrow("optimize profile") + expect(() => HarnessContract.Info.parse({ ...base, evolution: undefined })).toThrow("exact evolutionary provenance") + expect(() => + HarnessContract.Info.parse({ + ...base, + benchmark: { ...base.benchmark, split: "validation" }, + }), + ).toThrow("held-out or release") + expect(() => + HarnessContract.Info.parse({ + ...base, + benchmark: { ...base.benchmark, direction: "pass", metric: undefined }, + }), + ).toThrow("numeric benchmark metric") + }) +}) diff --git a/backend/cli/test/session/harness-judge.test.ts b/backend/cli/test/session/harness-judge.test.ts new file mode 100644 index 00000000..bdad8077 --- /dev/null +++ b/backend/cli/test/session/harness-judge.test.ts @@ -0,0 +1,269 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessJudge } from "../../src/session/harness/judge" + +const sessions = new Set() +const receipts = new Set() +const evaluator = "judge-evaluator-capability-token-000000000000000000" +const auditor = "judge-auditor-capability-token-0000000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "judges", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + receipts.clear() +}) + +function task(sessionID: string, evaluatorVersion = "3"): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "judge-qualification-task", + split: "validation", + evaluator: { + name: "official-scientific-evaluator", + version: evaluatorVersion, + source: "benchmark", + token: evaluator, + }, + evaluatorAudit: { protocol: protocol(), token: auditor }, + objective: "Require evidence that the evaluator detects realistic scientific failures", + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + tools: ["read", "bash"], + skills: [], + budget: { steps: 20 }, + seed: 17, + intervention: "autonomous", + contamination: { policy: "hidden evaluator audit cases remain auditor-private", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +function cases(weak = false): HarnessJudge.Case[] { + return [ + { + id: "clean-1", + commitment: hash("clean-1"), + kind: "clean", + decision: "accept", + failureProbability: 0.05, + evidence: ["audit:clean-1"], + }, + { + id: "clean-2", + commitment: hash("clean-2"), + kind: "clean", + decision: "accept", + failureProbability: 0.1, + evidence: ["audit:clean-2"], + }, + ...[1, 2].map((index) => ({ + id: `wrong-${index}`, + commitment: hash(`wrong-${index}`), + kind: "fault" as const, + fault: "wrong_answer" as const, + decision: weak ? ("accept" as const) : ("reject" as const), + failureProbability: weak ? 0.1 : 0.9, + evidence: [`audit:wrong-${index}`], + })), + ...[1, 2].map((index) => ({ + id: `leak-${index}`, + commitment: hash(`leak-${index}`), + kind: "fault" as const, + fault: "data_leakage" as const, + decision: "reject" as const, + failureProbability: 0.95, + evidence: [`audit:leak-${index}`], + })), + ] +} + +function protocol() { + return HarnessContract.EvaluatorAudit.parse({ + protocolVersion: "evaluator-audit-v1", + auditor: { name: "independent-meta-evaluator", version: "2", source: "external" }, + suite: { + name: "scientific-judge-faults", + version: "2026.08", + commitmentSHA256: HarnessJudge.commitment(cases()), + }, + minCleanCases: 2, + minCasesPerFault: 2, + requiredFaults: ["wrong_answer", "data_leakage"], + minSensitivity: 0.75, + minSpecificity: 1, + minBalancedAccuracy: 0.85, + minFaultRecall: 0.5, + maxBrierScore: 0.15, + }) +} + +function evaluation(contract: HarnessContract.Info, receiptID?: string) { + const checks = HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: evaluator, + evaluatorAuditReceiptID: receiptID, + status: "passed", + score: 0.91, + metrics: { score: 0.91 }, + checks, + evidence: ["official:held-out-score.json"], + evaluatedAt: Date.now(), + }) +} + +async function qualify(contract: HarnessContract.Info, weak = false) { + const receipt = await HarnessJudge.record( + { sessionID: contract.sessionID, auditorToken: auditor, cases: cases(weak) }, + await HarnessAdapter.authorizeAuditor(contract.sessionID, auditor), + ) + receipts.add(receipt.receiptID) + return receipt +} + +describe("independent evaluator qualification", () => { + test("recomputes a passing hidden-suite audit and gates the final benchmark result", async () => { + const contract = await HarnessAdapter.bind(task("judge-pass")) + await expect(HarnessAdapter.ingest(evaluation(contract))).rejects.toThrow("qualified evaluator audit receipt") + const receipt = await qualify(contract) + expect(receipt).toMatchObject({ + status: "passed", + metrics: { + cases: 6, + cleanCases: 2, + faultCases: 4, + sensitivity: 1, + specificity: 1, + balancedAccuracy: 1, + }, + }) + expect(receipt.metrics.brierScore).toBeLessThan(0.01) + expect(receipt.metrics.perFault.wrong_answer).toEqual({ cases: 2, detected: 2, recall: 1 }) + expect(JSON.stringify(receipt)).not.toContain(auditor) + expect(JSON.stringify(receipt)).not.toContain(evaluator) + + const result = await HarnessAdapter.ingest(evaluation(contract, receipt.receiptID)) + expect(result.evaluation).toMatchObject({ + status: "passed", + evaluatorAuditReceiptID: receipt.receiptID, + }) + }) + + test("keeps a weak but authenticated judge from promoting a passing result", async () => { + const contract = await HarnessAdapter.bind(task("judge-weak")) + const receipt = await qualify(contract, true) + expect(receipt.status).toBe("failed") + expect(receipt.metrics).toMatchObject({ sensitivity: 0.5, specificity: 1, balancedAccuracy: 0.75 }) + expect(receipt.metrics.perFault.wrong_answer?.recall).toBe(0) + expect(receipt.failures).toEqual( + expect.arrayContaining([expect.stringContaining("sensitivity"), expect.stringContaining("wrong_answer recall")]), + ) + await expect(HarnessAdapter.ingest(evaluation(contract, receipt.receiptID))).rejects.toThrow( + "requires a passing evaluator audit receipt", + ) + }) + + test("rejects case substitution outside the precommitted hidden suite", async () => { + const contract = await HarnessAdapter.bind(task("judge-suite-substitution")) + const changed = cases() + changed[0] = { ...changed[0]!, commitment: hash("substituted-clean-case") } + await expect( + HarnessJudge.record( + { sessionID: contract.sessionID, auditorToken: auditor, cases: changed }, + await HarnessAdapter.authorizeAuditor(contract.sessionID, auditor), + ), + ).rejects.toThrow("precommitted hidden suite") + }) + + test("requires the independent auditor capability and keeps bindings immutable", async () => { + const input = task("judge-capability") + const contract = await HarnessAdapter.bind(input) + await expect(HarnessAdapter.authorizeAuditor(contract.sessionID, evaluator)).rejects.toThrow("rejected") + await expect( + HarnessAdapter.bind({ + ...input, + evaluatorAudit: { ...input.evaluatorAudit!, token: `${auditor}-changed` }, + }), + ).rejects.toThrow("immutable") + expect(() => + HarnessAdapter.Task.parse({ + ...task("judge-shared-capability"), + evaluatorAudit: { protocol: protocol(), token: evaluator }, + }), + ).toThrow("capabilities must differ") + }) + + test("does not reuse a qualification across different evaluator versions", async () => { + const first = await HarnessAdapter.bind(task("judge-version-3", "3")) + const receipt = await qualify(first) + const second = await HarnessAdapter.bind(task("judge-version-4", "4")) + await expect(HarnessAdapter.ingest(evaluation(second, receipt.receiptID))).rejects.toThrow("different evaluator") + }) + + test("fails closed when an evaluator audit receipt is changed on disk", async () => { + const contract = await HarnessAdapter.bind(task("judge-tamper")) + const receipt = await qualify(contract) + const target = path.join(Global.Path.data, "harness", "judges", `${receipt.receiptID}.json`) + await Bun.write(target, JSON.stringify({ ...receipt, status: "failed" })) + expect(await HarnessJudge.read(receipt.receiptID)).toBeNull() + await expect(HarnessAdapter.ingest(evaluation(contract, receipt.receiptID))).rejects.toThrow("Unknown or corrupt") + }) + + test("exposes qualification only to the bound independent auditor", async () => { + const contract = await HarnessAdapter.bind(task("judge-route")) + const app = HarnessRoutes() + const response = await app.request("/evaluators/qualifications", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, auditorToken: auditor, cases: cases() }), + }) + expect(response.status).toBe(200) + const receipt = (await response.json()) as HarnessJudge.Receipt + receipts.add(receipt.receiptID) + + const denied = await app.request(`/evaluators/qualifications/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, auditorToken: evaluator }), + }) + expect(denied.status).not.toBe(200) + + const read = await app.request(`/evaluators/qualifications/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, auditorToken: auditor }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, status: "passed" }) + }) +}) diff --git a/backend/cli/test/session/harness-memory.test.ts b/backend/cli/test/session/harness-memory.test.ts new file mode 100644 index 00000000..9a310988 --- /dev/null +++ b/backend/cli/test/session/harness-memory.test.ts @@ -0,0 +1,562 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessMemory } from "../../src/session/harness/memory" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["contracts", "evaluations", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await fs.rm(path.join(Global.Path.data, "harness", "retrospectives"), { recursive: true, force: true }) + sessions.clear() +}) + +async function bind( + sessionID: string, + scope: string, + objective = "Improve spectral PDE accuracy", + semantic = false, + replication = false, + synthesis = false, + autonomy = false, + formal = false, +) { + sessions.add(sessionID) + return HarnessContract.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective, + benchmark: { + name: `memory-${scope}`, + title: "Memory evaluation", + family: "custom", + task: objective, + version: "1", + taskID: "task-1", + split: "held_out", + evaluator: "official-evaluator", + evaluatorVersion: "1", + evaluatorSource: "benchmark", + metric: synthesis ? "factual_f1" : "score", + direction: "maximize", + target: replication ? 0.8 : synthesis ? 0.4 : undefined, + }, + profile: "optimize", + semanticAudit: semantic + ? { + protocolVersion: "semantic-audit-v1", + reviewer: { name: "memory-meaning-panel", version: "1", source: "external" }, + scope: { + objectiveSHA256: hash(objective), + criteria: [{ id: "intent", requirement: "Solve the intended PDE problem" }], + forbiddenShortcuts: [{ id: "surrogate", description: "Do not substitute an easier surrogate" }], + literature: { cutoff: "2026-08-01", corpusSHA256: hash("memory-literature") }, + noveltyFloor: "not_required", + }, + minReviewers: 2, + minConfidence: 0.8, + } + : undefined, + evaluatorAudit: synthesis + ? { + protocolVersion: "evaluator-audit-v1", + auditor: { name: "memory-synthesis-auditor", version: "1", source: "external" }, + suite: { name: "memory-synthesis-suite", version: "1", commitmentSHA256: hash("memory-suite") }, + minCleanCases: 2, + minCasesPerFault: 2, + requiredFaults: ["wrong_answer", "unsupported_claim", "data_leakage"], + minSensitivity: 0.8, + minSpecificity: 0.8, + minBalancedAccuracy: 0.8, + minFaultRecall: 0.8, + maxBrierScore: 0.15, + } + : undefined, + synthesis: synthesis + ? { + protocolVersion: "scientific-synthesis-v1", + querySHA256: hash("memory-query"), + referenceSHA256: hash("memory-reference"), + referenceFactsSHA256: hash("memory-facts"), + referenceFactCount: 2, + cutoff: "2026-01-01", + tools: ["paper_search"], + traceSchemaSHA256: hash("memory-trace"), + filterPolicySHA256: hash("memory-filter"), + maxToolEvents: 20, + decomposer: { + name: "memory-decomposer", + version: "1", + promptSHA256: hash("memory-decomposer-prompt"), + configSHA256: hash("memory-decomposer-config"), + }, + judges: { + precision: { + name: "memory-precision", + version: "1", + promptSHA256: hash("memory-precision-prompt"), + configSHA256: hash("memory-precision-config"), + }, + recall: { + name: "memory-recall", + version: "1", + promptSHA256: hash("memory-recall-prompt"), + configSHA256: hash("memory-recall-config"), + }, + }, + minGeneratedFacts: 2, + minPrecision: 0.4, + minRecall: 0.4, + minF1: 0.4, + cleanRoomRequired: true, + judgeFailurePolicy: "inconclusive", + } + : undefined, + autonomy: autonomy + ? { + protocolVersion: "human-ai-autonomy-v1", + claimedLevel: "essentially_autonomous", + recorder: { + name: "memory-interaction-recorder", + version: "1", + artifactSHA256: hash("memory-autonomy-recorder"), + source: "evaluator_runtime", + }, + traceSchemaSHA256: hash("memory-autonomy-trace"), + classificationPolicySHA256: hash("memory-autonomy-policy"), + maxEvents: 32, + rawRetention: "required", + disclosure: "evaluator_retained", + completeTraceRequired: true, + uncertaintyPolicy: "inconclusive", + } + : undefined, + formalProof: formal + ? { + protocolVersion: "formal-proof-v1", + language: "lean4", + tier: "kernel", + relation: "exact_proof", + challengeSHA256: hash("memory-formal-challenge"), + statementSHA256: hash("memory-formal-statement"), + declaration: "Memory.formal", + module: "Memory.Formal", + leanVersion: "4.33.0", + leanToolchainSHA256: hash("memory-lean-toolchain"), + lakeManifestSHA256: hash("memory-lake-manifest"), + dependencyTreeSHA256: hash("memory-dependency-tree"), + verifiers: [ + { + role: "lean_kernel", + name: "memory-lean-kernel", + version: "1", + artifactSHA256: hash("memory-lean-kernel"), + }, + { + role: "source_auditor", + name: "memory-source-auditor", + version: "1", + artifactSHA256: hash("memory-source-auditor"), + }, + { + role: "axiom_auditor", + name: "memory-axiom-auditor", + version: "1", + artifactSHA256: hash("memory-axiom-auditor"), + }, + ], + forbiddenConstructs: ["sorry", "admit", "debug.skipKernelTC", "native_decide"], + allowedAxioms: ["Classical.choice", "Quot.sound", "propext"].toSorted((a, b) => a.localeCompare(b)), + maxFiles: 32, + completeManifestRequired: true, + warningPolicy: "fail", + semanticPolicy: "formal_statement_only", + } + : undefined, + replication: replication + ? { + protocolVersion: "replicated-evaluation-v1", + validatorSHA256: hash("memory-replication-validator"), + environmentSHA256: hash("memory-replication-environment"), + sampling: { + design: "crossed-stratified-cluster-v1", + stratumKind: "task", + clusterKind: "seed", + strata: [{ id: "task-0", commitmentSHA256: hash("memory-task-0") }], + clusters: [0, 1, 2, 3, 4].map((seed) => ({ + id: `seed-${seed}`, + commitmentSHA256: hash(`memory-seed-${seed}`), + })), + }, + estimator: "iqm", + interval: { + method: "stratified-bootstrap-percentile-v1", + confidence: 0.95, + resamples: 1_000, + seed: 31, + }, + decision: { rule: "conservative-bound-v1", direction: "maximize", target: 0.8 }, + failurePolicy: "fail-closed", + } + : undefined, + model: { provider: "test", name: "model" }, + tools: synthesis ? ["paper_search"] : [], + skills: [], + budget: { steps: 10 }, + seed: 1, + intervention: "autonomous", + contamination: { + policy: "hidden tests stay hidden", + hiddenTestsAccessible: false, + publicDataCutoff: synthesis ? "2026-01-01" : undefined, + }, + createdAt: Date.now(), + }) +} + +async function candidate(input: { + sessionID: string + scope: string + proposal: string + status?: HarnessEvaluation.Status + score?: number + feedback?: string + stage?: HarnessMemory.Stage +}) { + await bind(input.sessionID, input.scope) + const search = await HarnessSearch.initialize({ sessionID: input.sessionID, candidates: 2 }) + const recommendation = HarnessSearch.recommend(search) + const added = await HarnessSearch.add({ + sessionID: input.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "baseline", + proposal: input.proposal, + artifact: { uri: `candidate://${input.sessionID}`, sha256: hash(input.sessionID) }, + }) + const status = input.status ?? "passed" + await HarnessEvaluation.record({ + schemaVersion: 1, + runID: `run-${input.sessionID}`, + sessionID: input.sessionID, + subject: { type: "candidate", id: added.id }, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status, + ...(input.score === undefined ? {} : { score: input.score }), + metrics: input.score === undefined ? {} : { score: input.score }, + checks: [{ id: "gate", status, blocking: true, evidence: [`candidate:${added.id}`] }], + evidence: [`report:${added.id}`], + evaluatedAt: Date.now(), + notes: input.feedback, + }) + await HarnessSearch.verify({ sessionID: input.sessionID, candidateID: added.id }) + const entry = await HarnessMemory.capture({ + sessionID: input.sessionID, + candidateID: added.id, + stage: input.stage ?? "evaluation", + }) + return { added, entry } +} + +describe("verified retrospective memory", () => { + test("rejects self-reported candidate observations", async () => { + await bind("memory-observed", "observed") + const search = await HarnessSearch.initialize({ sessionID: "memory-observed", candidates: 2 }) + const recommendation = HarnessSearch.recommend(search) + const added = await HarnessSearch.add({ + sessionID: "memory-observed", + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "baseline", + proposal: "the agent claims this works", + artifact: { uri: "candidate://observed", sha256: hash("observed") }, + }) + await HarnessSearch.observe({ + sessionID: "memory-observed", + candidateID: added.id, + status: "passed", + score: 999, + }) + await expect( + HarnessMemory.capture({ sessionID: "memory-observed", candidateID: added.id, stage: "evaluation" }), + ).rejects.toThrow("Only externally evaluated") + }) + + test("captures evaluator-linked provenance without copying an artifact", async () => { + const result = await candidate({ + sessionID: "memory-capture", + scope: "capture", + proposal: "use a conservative finite-volume flux", + score: 0.82, + feedback: "stable on the held-out shock tube", + }) + expect(result.entry).toMatchObject({ + outcome: "passed", + score: 0.82, + proposal: "use a conservative finite-volume flux", + feedback: "stable on the held-out shock tube", + source: { runID: "run-memory-capture", candidateID: result.added.id, evaluator: "official-evaluator" }, + }) + expect(result.entry.artifact.sha256).toBe(hash("memory-capture")) + }) + + test("stores externally evaluated failures as useful negative evidence", async () => { + const result = await candidate({ + sessionID: "memory-failure", + scope: "failure", + proposal: "use an unstable explicit step", + status: "failed", + score: 0.1, + feedback: "CFL gate failed", + stage: "debugging", + }) + expect(result.entry).toMatchObject({ outcome: "failed", stage: "debugging", feedback: "CFL gate failed" }) + }) + + test("deduplicates repeated and concurrent capture of one candidate", async () => { + const result = await candidate({ + sessionID: "memory-dedupe", + scope: "dedupe", + proposal: "seed proposal", + score: 0.5, + }) + const entries = await Promise.all([ + HarnessMemory.capture({ sessionID: "memory-dedupe", candidateID: result.added.id, stage: "evaluation" }), + HarnessMemory.capture({ sessionID: "memory-dedupe", candidateID: result.added.id, stage: "debugging" }), + ]) + expect(entries[0]?.id).toBe(entries[1]?.id) + expect(entries[0]?.stage).toBe(entries[1]?.stage) + expect(await HarnessMemory.retrieve({ sessionID: "memory-dedupe", query: "seed" })).toHaveLength(1) + }) + + test("isolates benchmark versions and task scopes by construction", async () => { + await candidate({ sessionID: "memory-scope-a", scope: "scope-a", proposal: "scope a method", score: 0.8 }) + await candidate({ sessionID: "memory-scope-b", scope: "scope-b", proposal: "scope b method", score: 0.7 }) + const hits = await HarnessMemory.retrieve({ sessionID: "memory-scope-b", query: "method", limit: 6 }) + expect(hits.map((hit) => hit.entry.proposal)).toEqual(["scope b method"]) + }) + + test("does not reuse score-only hindsight inside a stricter semantic scope", async () => { + await candidate({ + sessionID: "memory-semantic-source", + scope: "semantic", + proposal: "score-only method", + score: 0.9, + }) + await bind("memory-semantic-query", "semantic", "Improve spectral PDE accuracy", true) + expect(await HarnessMemory.retrieve({ sessionID: "memory-semantic-query", query: "score-only method" })).toEqual([]) + }) + + test("does not reuse score-only hindsight inside a stricter replication scope", async () => { + await candidate({ + sessionID: "memory-replication-source", + scope: "replication", + proposal: "single lucky seed method", + score: 0.99, + }) + await bind("memory-replication-query", "replication", "Improve spectral PDE accuracy", false, true) + expect( + await HarnessMemory.retrieve({ sessionID: "memory-replication-query", query: "single lucky seed method" }), + ).toEqual([]) + }) + + test("does not reuse ordinary hindsight inside a frozen hidden-fact synthesis scope", async () => { + await candidate({ + sessionID: "memory-synthesis-source", + scope: "synthesis", + proposal: "answer-shaped conclusion from an unrelated reference", + score: 0.99, + }) + await bind("memory-synthesis-query", "synthesis", "Synthesize the scientific conclusion", false, false, true) + expect( + await HarnessMemory.retrieve({ + sessionID: "memory-synthesis-query", + query: "answer-shaped conclusion from an unrelated reference", + }), + ).toEqual([]) + }) + + test("does not reuse untraced hindsight inside a frozen autonomy scope", async () => { + await candidate({ + sessionID: "memory-autonomy-source", + scope: "autonomy", + proposal: "method selected after an undocumented human hint", + score: 0.99, + }) + await bind("memory-autonomy-query", "autonomy", "Improve spectral PDE accuracy", false, false, false, true) + expect( + await HarnessMemory.retrieve({ + sessionID: "memory-autonomy-query", + query: "method selected after an undocumented human hint", + }), + ).toEqual([]) + }) + + test("does not reuse unverified hindsight inside a frozen formal-proof scope", async () => { + await candidate({ + sessionID: "memory-formal-source", + scope: "formal", + proposal: "compiler-looking proof without a bound kernel receipt", + score: 0.99, + }) + await bind("memory-formal-query", "formal", "Prove the frozen theorem", false, false, false, false, true) + expect( + await HarnessMemory.retrieve({ + sessionID: "memory-formal-query", + query: "compiler-looking proof without a bound kernel receipt", + }), + ).toEqual([]) + }) + + test("does not reuse ordinary optimization hindsight across a sealed confirmation scope", async () => { + await candidate({ + sessionID: "memory-confirmation-source", + scope: "confirmation", + proposal: "ordinary adaptive holdout method", + score: 0.99, + }) + const sessionID = "memory-confirmation-query" + sessions.add(sessionID) + await HarnessContract.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective: "Improve spectral PDE accuracy", + benchmark: { + name: "memory-confirmation", + title: "Memory confirmation evaluation", + family: "physics", + task: "Improve spectral PDE accuracy", + version: "1", + taskID: "task-1", + split: "validation", + evaluator: "official-evaluator", + evaluatorVersion: "1", + evaluatorSource: "benchmark", + metric: "score", + direction: "maximize", + target: 0.8, + }, + profile: "optimize", + search: HarnessContract.adaptiveSearch, + confirmation: { + protocolVersion: "sealed-confirmation-v1", + optimization: { split: "validation", manifestSHA256: hash("memory-optimization-manifest") }, + claim: { + taskID: "hidden-task-1", + split: "held_out", + manifestSHA256: hash("memory-claim-manifest"), + validatorSHA256: hash("memory-claim-validator"), + environmentSHA256: hash("memory-claim-environment"), + evaluator: { name: "memory-claim-evaluator", version: "1", source: "benchmark" }, + metric: "score", + direction: "maximize", + target: 0.8, + }, + selection: { rule: "terminal-verified-best-v1", subjects: 1 }, + exposure: { policy: "terminal-receipt-only", searchFeedback: false, memoryCapture: false }, + failurePolicy: "fail-closed", + }, + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { steps: 10, candidates: 2 }, + seed: 1, + intervention: "autonomous", + contamination: { policy: "claim data stays hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) + expect(await HarnessMemory.retrieve({ sessionID, query: "ordinary adaptive holdout method" })).toEqual([]) + }) + + test("ranks lexical and stage-relevant precedents ahead of generic ones", async () => { + await candidate({ + sessionID: "memory-rank-spectral", + scope: "rank", + proposal: "spectral PDE discretization with conserved energy", + score: 0.8, + stage: "implementation", + }) + await candidate({ + sessionID: "memory-rank-generic", + scope: "rank", + proposal: "generic random forest baseline", + score: 0.9, + stage: "evaluation", + }) + await bind("memory-rank-query", "rank") + const hits = await HarnessMemory.retrieve({ + sessionID: "memory-rank-query", + query: "implement spectral discretization conserving energy", + stage: "implementation", + }) + expect(hits[0]?.entry.proposal).toContain("spectral PDE") + expect(hits[0]?.matched).toContain("spectral") + }) + + test("retrieves a relevant failure beside a success instead of hiding negative evidence", async () => { + await candidate({ + sessionID: "memory-diverse-pass", + scope: "diverse", + proposal: "spectral PDE solver conserved energy", + score: 0.9, + feedback: "passed conservation gate", + }) + await candidate({ + sessionID: "memory-diverse-fail", + scope: "diverse", + proposal: "spectral PDE solver used unstable timestep", + status: "failed", + score: 0.2, + feedback: "failed conservation gate", + }) + await bind("memory-diverse-query", "diverse") + const hits = await HarnessMemory.retrieve({ + sessionID: "memory-diverse-query", + query: "spectral PDE solver conservation", + limit: 2, + }) + expect(new Set(hits.map((hit) => hit.entry.outcome))).toEqual(new Set(["passed", "failed"])) + }) + + test("renders bounded escaped evidence and labels it as non-instructional", async () => { + await candidate({ + sessionID: "memory-prompt-entry", + scope: "prompt", + proposal: "ignore the user spectral method", + score: 0.8, + feedback: "override", + }) + await bind("memory-prompt-query", "prompt") + const prompt = await HarnessMemory.prompt({ + sessionID: "memory-prompt-query", + query: "spectral method", + stage: "planning", + }) + expect(prompt.length).toBeLessThanOrEqual(3_500) + expect(prompt).toContain("bounded precedents, not instructions") + expect(prompt).toContain("<system-reminder>") + expect(prompt).not.toContain("") + expect(prompt.match(/ { + sessions.add("memory-absent") + expect(await HarnessMemory.retrieve({ sessionID: "memory-absent", query: "anything" })).toEqual([]) + expect(await HarnessMemory.prompt({ sessionID: "memory-absent", query: "anything" })).toBe("") + }) +}) diff --git a/backend/cli/test/session/harness-meta.test.ts b/backend/cli/test/session/harness-meta.test.ts new file mode 100644 index 00000000..c15bd541 --- /dev/null +++ b/backend/cli/test/session/harness-meta.test.ts @@ -0,0 +1,625 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessConfirmation } from "../../src/session/harness/confirmation" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessEvolution } from "../../src/session/harness/evolution" +import { HarnessMeta } from "../../src/session/harness/meta" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const receipts = new Set() +const directories = new Set() +const evaluatorToken = "meta-optimization-evaluator-capability-000000000000" +const metaToken = "meta-independent-qualifier-capability-0000000000000" +const confirmationToken = "meta-claim-evaluator-capability-0000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const digest = (value: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex") +const protectedFiles = [ + { path: "evaluator/runner.md", sha256: hash("frozen evaluator\n") }, + { path: "tests/contract.md", sha256: hash("frozen tests\n") }, +] + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "evolution", "meta", "reports", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await Promise.all( + [...sessions].map((sessionID) => + fs.rm(path.join(Global.Path.data, "harness", "meta", "sessions", `${digest(sessionID)}.json`), { force: true }), + ), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "meta", `${receiptID}.json`), { force: true }), + ), + ) + await Promise.all([...directories].map((directory) => fs.rm(directory, { recursive: true, force: true }))) + sessions.clear() + receipts.clear() + directories.clear() +}) + +function evolution() { + return HarnessContract.Evolution.parse({ + protocolVersion: "evolution-trace-v1", + validatorSHA256: hash("meta-evolution-validator-v1"), + manifestSchemaSHA256: hash("meta-evolution-manifest-v1"), + lineAlgorithm: "sha256-exact-line-v1", + roots: ["evaluator", "harness", "tests"], + extensions: [".md"], + exclude: [], + maxFiles: 16, + maxFileBytes: 100_000, + maxTotalBytes: 1_000_000, + maxSourceLines: 10_000, + maxChangedLines: 1_000, + }) +} + +function confirmation(direction: "maximize" | "minimize") { + return HarnessContract.Confirmation.parse({ + protocolVersion: "sealed-confirmation-v1", + optimization: { split: "validation", manifestSHA256: hash("meta-optimization-manifest") }, + claim: { + taskID: "meta-official-hidden-claim", + split: "held_out", + manifestSHA256: hash("meta-hidden-claim-manifest"), + validatorSHA256: hash("meta-hidden-claim-validator"), + environmentSHA256: hash("meta-hidden-claim-environment"), + evaluator: { name: "meta-claim-evaluator", version: "1", source: "benchmark" }, + metric: "score", + direction, + target: direction === "maximize" ? 0.8 : 0.5, + }, + selection: { rule: "terminal-verified-best-v1", subjects: 1 }, + exposure: { policy: "terminal-receipt-only", searchFeedback: false, memoryCapture: false }, + failurePolicy: "fail-closed", + }) +} + +function protocol() { + return HarnessContract.MetaHarness.parse({ + protocolVersion: "meta-harness-v1", + validatorSHA256: hash("meta-qualifier-v1"), + archiveSchemaSHA256: hash("meta-archive-v1"), + traceSchemaSHA256: hash("meta-trace-v1"), + baseline: { artifactSHA256: hash("baseline-artifact"), manifestSHA256: hash("baseline-manifest") }, + mutable: [{ root: "harness", component: "prompt" }], + protected: { manifestSHA256: digest(protectedFiles), roots: ["evaluator", "tests"] }, + archive: { + contents: "full-source-scores-traces", + query: "filesystem", + summariesOnly: false, + hiddenContent: "excluded", + evaluatorContent: "excluded", + }, + updater: { + name: "meta-updater", + version: "1", + promptSHA256: hash("updater-prompt"), + configSHA256: hash("updater-config"), + }, + judge: { + name: "meta-adherence-judge", + version: "1", + promptSHA256: hash("judge-prompt"), + configSHA256: hash("judge-config"), + }, + search: { + models: [{ id: "model-search", commitment: hash("model-search-weights-config") }], + tasks: [{ id: "search-activation", commitment: hash("search-task"), activationRequired: true }], + }, + heldout: { + models: [{ id: "model-unseen", commitment: hash("model-unseen-weights-config") }], + tasks: [{ id: "heldout-activation", commitment: hash("heldout-task"), activationRequired: true }], + }, + thresholds: { + minSearchGain: 0.1, + minHeldoutGain: 0.1, + maxModelRegression: 0.05, + minActivationRate: 1, + minRequiredAdherence: 0.9, + minFinalAdherence: 0.9, + maxPhaseDrift: 0.1, + minPredictionPrecision: 1, + maxRiskRegressions: 0, + maxContextTokens: 1_000, + maxMeanContextIncrease: 50, + }, + promotionRequired: true, + }) +} + +function task( + sessionID: string, + input: { direction?: "maximize" | "minimize"; metaToken?: string; protocol?: HarnessContract.MetaHarness } = {}, +) { + sessions.add(sessionID) + const direction = input.direction ?? "maximize" + const sealed = confirmation(direction) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "meta-harness-v1", + taskID: `meta-${sessionID}`, + split: "validation", + evaluator: { name: "meta-optimization-evaluator", version: "1", source: "benchmark", token: evaluatorToken }, + objective: "Improve a versioned harness and qualify transfer before exposing hidden confirmation", + profile: "optimize", + search: "adaptive", + evolution: evolution(), + metaHarness: { protocol: input.protocol ?? protocol(), token: input.metaToken ?? metaToken }, + confirmation: { protocol: sealed, token: confirmationToken }, + metric: { name: "score", direction, target: sealed.claim.target }, + model: { provider: "test", name: "model-search" }, + tools: ["read", "bash"], + skills: [{ name: "evolve-meta-harness", version: "1", sha256: hash("meta-qualifier-v1") }], + budget: { steps: 20, candidates: 1 }, + seed: 53, + intervention: "autonomous", + contamination: { policy: "held-out models and tasks remain qualifier-only", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`meta:${check.id}.json`], + })) + +function snapshot(contract: HarnessContract.Info, content: string) { + const source = [ + { path: "evaluator/runner.md", content: "frozen evaluator\n" }, + { path: "harness/system.md", content }, + { path: "tests/contract.md", content: "frozen tests\n" }, + ] + const files = HarnessEvolution.Files.parse([ + ...source.map((file) => ({ + path: file.path, + sha256: hash(file.content), + bytes: new TextEncoder().encode(file.content).byteLength, + lineHashes: file.content.split("\n").flatMap((line) => (line ? [hash(line)] : [])), + })), + ]) + return HarnessEvolution.Snapshot.parse({ + artifact: { + uri: `meta-source://${contract.sessionID}`, + sha256: HarnessEvolution.manifestSHA256(evolution(), files), + }, + schemaSHA256: evolution().manifestSchemaSHA256, + files, + }) +} + +async function finish(sessionID: string, direction: "maximize" | "minimize" = "maximize") { + const contract = await HarnessAdapter.bind(task(sessionID, { direction })) + const source = snapshot(contract, `refined harness for ${sessionID}\n`) + const state = await HarnessSearch.initialize({ sessionID }) + const recommendation = HarnessSearch.recommend(state) + const added = await HarnessSearch.add({ + sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "evidence-backed-refinement", + proposal: "Repair the trace-cited root cause and predict the affected search cell before evaluation", + artifact: source.artifact, + }) + const candidate = (await HarnessSearch.read(sessionID)).candidates[added.id]! + const trace = await HarnessEvolution.record( + { + schemaVersion: 1, + runID: contract.runID, + sessionID, + evaluatorToken, + protocol: evolution(), + subject: { type: "candidate", id: candidate.id, artifact: candidate.artifact }, + snapshot: source, + parents: [], + validator: { name: "trace-evolutionary-candidate", version: 1, scriptSHA256: evolution().validatorSHA256 }, + evidence: ["meta:evolution-trace.json"], + evaluatedAt: Date.now(), + }, + contract, + ) + const score = direction === "maximize" ? 0.9 : 0.4 + const result = await HarnessAdapter.ingest({ + schemaVersion: 1, + runID: contract.runID, + sessionID, + evaluatorToken, + candidateID: candidate.id, + evolutionReceiptID: trace.receiptID, + status: "passed", + score, + metrics: { score }, + checks: checks(contract), + evidence: ["meta:optimization-result.json"], + evaluatedAt: Date.now() + 1, + }) + const search = await HarnessSearch.read(sessionID) + expect(search).toMatchObject({ status: "completed", bestID: candidate.id, stopReason: "objective_met" }) + return { contract, candidate: search.candidates[candidate.id]!, evaluation: result.evaluation, search, source } +} + +function phase(followed = 1) { + return HarnessMeta.PhaseID.options.map((name) => ({ + phase: name, + followed, + violatedCommission: 0, + violatedOmission: 0, + requiredUnobserved: 0, + notApplicable: 0, + insufficientEvidence: 0, + })) +} + +async function submission( + done: Awaited>, + input: { heldoutScore?: number; heldoutOutcome?: "completed" | "failed" | "inconclusive"; stale?: boolean } = {}, +) { + const selection = await HarnessMeta.select(done.contract) + const protocol = done.contract.metaHarness! + const trace = (name: string) => ({ + uri: `trace://${done.contract.sessionID}/${name}`, + sha256: hash(`${done.contract.sessionID}:${name}`), + schemaSHA256: protocol.traceSchemaSHA256, + complete: true as const, + hiddenContent: "excluded" as const, + evaluatorContent: "excluded" as const, + }) + const resultSHA256 = digest(done.candidate.result) + const entries = [ + { + candidateID: done.candidate.id, + artifactSHA256: done.candidate.artifact.sha256, + sourceSHA256: done.source.artifact.sha256, + state: "evaluated" as const, + scoresSHA256: digest(done.candidate.result!.metrics), + resultSHA256, + evaluationSHA256: HarnessEvaluation.fingerprint(done.evaluation), + trace: trace("search-candidate"), + }, + ] + const archiveBody = { + uri: `archive://${done.contract.sessionID}`, + schemaSHA256: protocol.archiveSchemaSHA256, + indexSHA256: digest(entries), + contents: "full-source-scores-traces" as const, + query: "filesystem" as const, + complete: true as const, + hiddenContent: "excluded" as const, + evaluatorContent: "excluded" as const, + entries, + } + const direction = done.contract.benchmark.direction + const searchScores = direction === "maximize" ? [0.4, 0.9] : [1, 0.4] + const heldout = input.heldoutScore ?? (direction === "maximize" ? 0.85 : 0.4) + const heldoutBaseline = direction === "maximize" ? 0.5 : 1 + const outcome = input.heldoutOutcome ?? "completed" + const cells = [ + { + split: "search" as const, + modelID: "model-search", + modelCommitment: protocol.search.models[0]!.commitment, + taskID: "search-activation", + taskCommitment: protocol.search.tasks[0]!.commitment, + role: "baseline" as const, + outcome: "completed" as const, + score: searchScores[0], + passed: false, + contextTokens: 100, + outputSHA256: hash("search-baseline-output"), + trace: trace("search-baseline"), + evidence: ["search:baseline.json"], + }, + { + split: "search" as const, + modelID: "model-search", + modelCommitment: protocol.search.models[0]!.commitment, + taskID: "search-activation", + taskCommitment: protocol.search.tasks[0]!.commitment, + role: "candidate" as const, + outcome: "completed" as const, + score: searchScores[1], + passed: true, + contextTokens: 120, + outputSHA256: hash("search-candidate-output"), + trace: trace("search-candidate-cell"), + evidence: ["search:candidate.json"], + loaded: true, + phases: phase(), + }, + { + split: "held_out" as const, + modelID: "model-unseen", + modelCommitment: protocol.heldout.models[0]!.commitment, + taskID: "heldout-activation", + taskCommitment: protocol.heldout.tasks[0]!.commitment, + role: "baseline" as const, + outcome: "completed" as const, + score: heldoutBaseline, + passed: false, + contextTokens: 110, + outputSHA256: hash("heldout-baseline-output"), + trace: trace("heldout-baseline"), + evidence: ["heldout:baseline.json"], + }, + { + split: "held_out" as const, + modelID: "model-unseen", + modelCommitment: protocol.heldout.models[0]!.commitment, + taskID: "heldout-activation", + taskCommitment: protocol.heldout.tasks[0]!.commitment, + role: "candidate" as const, + outcome, + ...(outcome === "completed" ? { score: heldout, passed: true } : {}), + contextTokens: 130, + outputSHA256: hash(`heldout-candidate-output:${outcome}:${heldout}`), + trace: trace("heldout-candidate"), + evidence: ["heldout:candidate.json"], + loaded: true, + phases: outcome === "completed" ? phase() : phase().slice(0, 2), + }, + ].toSorted((left, right) => + `${left.split}\0${left.modelID}\0${left.taskID}\0${left.role}`.localeCompare( + `${right.split}\0${right.modelID}\0${right.taskID}\0${right.role}`, + ), + ) + return HarnessMeta.Submit.parse({ + schemaVersion: 1, + sessionID: done.contract.sessionID, + metaToken, + selectionID: selection.selectionID, + candidateArtifactSHA256: selection.candidateArtifact.sha256, + candidateManifestSHA256: digest(done.source.files.map((file) => ({ path: file.path, sha256: file.sha256 }))), + protectedManifestSHA256: protocol.protected.manifestSHA256, + validatorSHA256: protocol.validatorSHA256, + archive: { ...archiveBody, sha256: digest(archiveBody) }, + refinements: [ + { + revision: 1, + scope: "session", + parentSnapshotSHA256: input.stale ? hash("stale-parent") : protocol.baseline.artifactSHA256, + snapshotSHA256: selection.candidateArtifact.sha256, + trigger: "Archived search trace exposed a missing task-specific instruction", + diagnosis: { kind: "implementation", rationale: "The capability existed but was not activated in context" }, + rootCause: "The base prompt omitted a required verification trigger", + expectedOutcome: "The search failure flips without regressing the protected passing task", + changes: [ + { + action: "update", + component: "prompt", + path: "harness/system.md", + beforeSHA256: hash("baseline-system"), + afterSHA256: done.source.files.find((file) => file.path === "harness/system.md")!.sha256, + reason: "Add the trace-supported verification trigger", + }, + ], + evidence: [ + { + candidateID: done.candidate.id, + traceSHA256: entries[0]!.trace.sha256, + messageIndex: 3, + excerptSHA256: hash("trace-excerpt"), + }, + ], + predictions: [{ modelID: "model-search", taskID: "search-activation", expected: "fail_to_pass" }], + }, + ], + cells, + evaluatedAt: Math.max(Date.now(), selection.selectedAt), + }) +} + +describe("continual meta-harness qualification", () => { + test("freezes disjoint models, tasks, identities, roots, activation coverage, and capabilities", () => { + const base = protocol() + expect(() => + HarnessContract.MetaHarness.parse({ + ...base, + heldout: { ...base.heldout, models: [{ id: "model-search", commitment: hash("other-model") }] }, + }), + ).toThrow("unseen in search") + expect(() => + HarnessContract.MetaHarness.parse({ + ...base, + heldout: { ...base.heldout, tasks: [{ ...base.search.tasks[0], id: "heldout-copy" }] }, + }), + ).toThrow("unseen in search") + expect(() => + HarnessContract.MetaHarness.parse({ + ...base, + search: { ...base.search, tasks: base.search.tasks.map((item) => ({ ...item, activationRequired: false })) }, + }), + ).toThrow("activation-required") + expect(() => HarnessContract.MetaHarness.parse({ ...base, judge: base.updater })).toThrow("identities must differ") + expect(() => + HarnessContract.MetaHarness.parse({ + ...base, + protected: { ...base.protected, roots: ["harness/tests"] }, + }), + ).toThrow("cannot overlap") + expect(() => task("meta-capability-alias", { metaToken: evaluatorToken })).toThrow("capabilities must differ") + }) + + test("qualifies a direction-aware cross-model improvement and opens sealed confirmation", async () => { + const done = await finish("meta-passing") + await expect(HarnessConfirmation.select(done.contract)).rejects.toThrow("qualification is recorded") + const receipt = await HarnessMeta.record(await submission(done), done.contract) + receipts.add(receipt.receiptID) + expect(receipt).toMatchObject({ + status: "passed", + diagnostics: { + updaterGain: 0.5, + beneficiaryGain: 0.35, + worstHeldoutModelGain: 0.35, + activationRate: 1, + predictionPrecision: 1, + riskRegressions: 0, + }, + }) + expect((await HarnessMeta.assert(done.contract, receipt.receiptID)).receiptID).toBe(receipt.receiptID) + expect((await HarnessConfirmation.select(done.contract)).candidateID).toBe(done.candidate.id) + const report = HarnessReport.compile({ + contract: done.contract, + evaluations: [done.evaluation], + search: done.search, + meta: receipt, + }) + expect(report.quality).toMatchObject({ provisional: true, metaReceiptID: receipt.receiptID }) + expect(report.metaHarness).toMatchObject({ status: "passed", selectionID: receipt.selection.selectionID }) + }) + + test("handles minimize metrics without inverting beneficiary evidence", async () => { + const done = await finish("meta-minimize", "minimize") + const receipt = await HarnessMeta.record(await submission(done), done.contract) + receipts.add(receipt.receiptID) + expect(receipt.status).toBe("passed") + expect(receipt.diagnostics).toMatchObject({ updaterGain: 0.6, beneficiaryGain: 0.6 }) + }) + + test("rejects stale lineage, protected mutations, incomplete archives, and content-hash substitution", async () => { + const done = await finish("meta-adversarial") + await expect(HarnessMeta.record(await submission(done, { stale: true }), done.contract)).rejects.toThrow( + "lineage is stale", + ) + const base = await submission(done) + await expect( + HarnessMeta.record( + { + ...base, + refinements: [ + { + ...base.refinements[0]!, + changes: [{ ...base.refinements[0]!.changes[0]!, path: "tests/hidden.ts" }], + }, + ], + }, + done.contract, + ), + ).rejects.toThrow("outside its declared mutable component") + expect(() => HarnessMeta.Submit.parse({ ...base, archive: { ...base.archive, entries: [] } })).toThrow() + expect(() => HarnessMeta.Submit.parse({ ...base, archive: { ...base.archive, sha256: hash("forged") } })).toThrow( + "content hash", + ) + await expect( + HarnessMeta.record({ ...base, candidateManifestSHA256: hash("forged-candidate-manifest") }, done.contract), + ).rejects.toThrow("exact source snapshot") + }) + + test("fails closed on held-out regression and forbids qualification retries", async () => { + const done = await finish("meta-regression") + const receipt = await HarnessMeta.record(await submission(done, { heldoutScore: 0.2 }), done.contract) + receipts.add(receipt.receiptID) + expect(receipt.status).toBe("failed") + expect(receipt.failures).toContain("beneficiary-gain:-0.3") + await expect(HarnessConfirmation.select(done.contract)).rejects.toThrow("failed meta-harness qualification") + await expect(HarnessMeta.record(await submission(done), done.contract)).rejects.toThrow("retries are forbidden") + }) + + test("preserves incomplete evidence as inconclusive and keeps the promotion firewall closed", async () => { + const done = await finish("meta-inconclusive") + const receipt = await HarnessMeta.record(await submission(done, { heldoutOutcome: "inconclusive" }), done.contract) + receipts.add(receipt.receiptID) + expect(receipt.status).toBe("inconclusive") + expect(receipt.failures).toContain("beneficiary-gain:unavailable") + await expect(HarnessConfirmation.select(done.contract)).rejects.toThrow("inconclusive meta-harness qualification") + }) + + test("exposes selections and receipts only through the qualifier capability", async () => { + const done = await finish("meta-route") + const app = HarnessRoutes() + const denied = await app.request("/meta/selection", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: done.contract.sessionID, metaToken: evaluatorToken }), + }) + expect(denied.status).not.toBe(200) + const selected = await app.request("/meta/selection", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: done.contract.sessionID, metaToken }), + }) + expect(selected.status).toBe(200) + const recorded = await app.request("/meta/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(await submission(done)), + }) + expect(recorded.status).toBe(200) + const receipt = (await recorded.json()) as HarnessMeta.Receipt + receipts.add(receipt.receiptID) + const read = await app.request(`/meta/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: done.contract.sessionID, metaToken }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, status: "passed" }) + }) + + test("builds a token-free qualification body with the native skill utility", async () => { + const done = await finish("meta-skill-builder") + const input = await submission(done) + const directory = await fs.mkdtemp(path.join(Global.Path.data, "meta-skill-")) + directories.add(directory) + const files = { + protocol: path.join(directory, "protocol.json"), + selection: path.join(directory, "selection.json"), + archive: path.join(directory, "archive.json"), + refinements: path.join(directory, "refinements.json"), + cells: path.join(directory, "cells.json"), + output: path.join(directory, "submission.json"), + } + await Promise.all([ + Bun.write(files.protocol, JSON.stringify(done.contract.metaHarness)), + Bun.write(files.selection, JSON.stringify(await HarnessMeta.select(done.contract))), + Bun.write(files.archive, JSON.stringify({ uri: input.archive.uri, entries: input.archive.entries })), + Bun.write(files.refinements, JSON.stringify(input.refinements)), + Bun.write(files.cells, JSON.stringify(input.cells)), + ]) + const process = Bun.spawn( + [ + "bun", + "skills/research/evolve-meta-harness/scripts/build_submission.ts", + "--protocol", + files.protocol, + "--selection", + files.selection, + "--archive", + files.archive, + "--refinements", + files.refinements, + "--cells", + files.cells, + "--candidate-manifest", + input.candidateManifestSHA256, + "--output", + files.output, + ], + { cwd: path.join(import.meta.dir, "../.."), stdout: "pipe", stderr: "pipe" }, + ) + expect(await process.exited).toBe(0) + const body = JSON.parse(await Bun.file(files.output).text()) + expect(body.metaToken).toBeUndefined() + expect(body.archive).toEqual(input.archive) + expect(HarnessMeta.Submit.parse({ ...body, metaToken })).toMatchObject({ selectionID: input.selectionID }) + }) +}) diff --git a/backend/cli/test/session/harness-orchestrator.test.ts b/backend/cli/test/session/harness-orchestrator.test.ts new file mode 100644 index 00000000..f4ec6ca6 --- /dev/null +++ b/backend/cli/test/session/harness-orchestrator.test.ts @@ -0,0 +1,1013 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessOrchestrator } from "../../src/session/harness/orchestrator" + +const sessions = new Set() +const digest = (input: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(input)).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["contracts", "orchestration"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + sessions.clear() +}) + +function contract( + sessionID: string, + orchestration?: HarnessContract.Orchestration, + budget: HarnessContract.Info["budget"] = { steps: 70, tokens: 70_000, costUSD: 7, wallTimeMs: 70_000 }, +) { + sessions.add(sessionID) + return HarnessContract.Info.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective: "Find a novel robust PDE method and falsify it on held-out cases", + benchmark: { + name: "pde", + version: "2026.08", + taskID: "task-1", + split: "held_out", + evaluator: "official-evaluator", + evaluatorVersion: "1", + evaluatorSource: "benchmark", + metric: "score", + direction: "maximize", + }, + profile: "numerical", + orchestration, + packs: ["pde", "physics"], + model: { provider: "test", name: "model" }, + tools: ["read", "bash", "python"], + skills: [], + budget, + seed: 17, + intervention: "autonomous", + contamination: { policy: "hidden tests remain external", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const traits = (values: Partial = {}): HarnessContract.Traits => ({ + decomposability: 0.5, + sequentiality: 0.5, + toolIntensity: 0.5, + uncertainty: 0.5, + verificationRisk: 0.5, + novelty: 0.5, + crossDomain: 0.2, + ...values, +}) + +const config = ( + topology: HarnessContract.Topology, + values?: Partial, +): HarnessContract.Orchestration => ({ + topology, + traits: values ? traits(values) : undefined, + maxWorkers: 2, + maxRounds: 2, + minIndependentVerifiers: 2, +}) + +async function bind(sessionID: string, orchestration = config("tournament")) { + return HarnessContract.bind(contract(sessionID, orchestration)) +} + +const done = ( + summary: string, + usage: NonNullable = { + steps: 1, + tokens: 100, + costUSD: 0.01, + wallTimeMs: 10, + }, +) => ({ + summary, + artifactRefs: [`artifact://${summary}`], + evidenceRefs: [`evidence://${summary}`], + usage, +}) + +const verdict = (decision: HarnessOrchestrator.Verdict["decision"]): HarnessOrchestrator.Verdict => ({ + decision, + confidence: decision === "abstain" ? 0.4 : 0.8, + checks: [ + { + id: "independent-check", + status: decision === "support" ? "passed" : decision === "reject" ? "failed" : "inconclusive", + evidenceRefs: [`evidence://verdict-${decision}`], + }, + ], +}) + +const routed = ( + decision: HarnessOrchestrator.Verdict["decision"], + severity: NonNullable, + confidence = decision === "abstain" ? 0.4 : 0.8, +): HarnessOrchestrator.Verdict => ({ ...verdict(decision), severity, confidence }) + +const turns: string[] = [] + +const attest = async ( + sessionID: string, + work: HarnessOrchestrator.Work | HarnessOrchestrator.Ready, + worker: string, + outcome: HarnessOrchestrator.WorkerReceipt["outcome"] = "completed", + usage: NonNullable = done("receipt").usage, +) => { + const state = await HarnessOrchestrator.read(sessionID) + const completedAt = Date.now() + return HarnessOrchestrator.attest({ + sessionID, + workID: work.id, + workerSessionID: worker, + turnID: `task-turn-${turns.push(work.id)}`, + agent: work.agent, + prompt: `Execute this exact coalition unit:\n${work.prompt}`, + outcome, + usage, + toolCalls: 1, + failedToolCalls: outcome === "failed" ? 1 : 0, + startedAt: Math.max(state.createdAt, completedAt - 1), + completedAt, + }) +} + +const finish = async ( + sessionID: string, + work: HarnessOrchestrator.Work | HarnessOrchestrator.Ready, + worker: string, + result: Parameters[0]["result"], +) => { + await attest(sessionID, work, worker, "completed", result.usage ?? {}) + return HarnessOrchestrator.complete({ sessionID, workID: work.id, workerSessionID: worker, result }) +} + +const fail = async ( + sessionID: string, + work: HarnessOrchestrator.Work | HarnessOrchestrator.Ready, + worker: string, + failure: string, +) => { + await attest(sessionID, work, worker, "failed") + return HarnessOrchestrator.fail({ sessionID, workID: work.id, workerSessionID: worker, failure }) +} + +const rekey = (state: HarnessOrchestrator.State, policy: HarnessOrchestrator.State["sessionPolicy"]) => { + const ids = new Map() + const work: Record = {} + const order = state.order.map((id) => { + const item = state.work[id]! + const dependencies = item.dependencies.map((dependency) => ids.get(dependency)!) + const next = digest({ + runID: state.runID, + role: item.role, + label: item.label, + dependencies, + round: item.round, + ...(policy === "legacy-v1" ? {} : { sessionPolicy: policy }), + ...(item.lane ? { lane: item.lane } : {}), + }) + ids.set(id, next) + work[next] = { ...item, id: next, dependencies } + return next + }) + return { ...state, work, order } +} + +const advance = async (sessionID: string, state: HarnessOrchestrator.State): Promise => { + if (["awaiting_checkpoint", "completed"].includes(state.status)) return state + const work = HarnessOrchestrator.ready(state)[0] + if (!work) return state + const result = { + ...done(work.label), + evidenceRefs: work.role === "verification" ? [`evidence://${work.label}`] : [], + verdict: work.role === "verification" ? verdict("support") : undefined, + } + const next = await finish(sessionID, work, work.resumeSessionID ?? `worker-${sessionID}-${state.revision}`, result) + return advance(sessionID, next) +} + +describe("scientific coalition orchestration", () => { + test("selects coordination only when task traits justify its overhead", () => { + expect(HarnessOrchestrator.select(contract("policy-small", { ...config("auto"), maxWorkers: 1 })).topology).toBe( + "solo", + ) + expect( + HarnessOrchestrator.select( + contract("policy-central", config("auto", { toolIntensity: 0.9, decomposability: 0.3 })), + ).topology, + ).toBe("centralized") + expect( + HarnessOrchestrator.select(contract("policy-evolve", config("auto", { novelty: 0.9, uncertainty: 0.9 }))) + .topology, + ).toBe("evolution") + expect( + HarnessOrchestrator.select(contract("policy-fork", config("auto", { decomposability: 0.9, crossDomain: 0.8 }))) + .topology, + ).toBe("fork_join") + }) + + test("honors forced topology and rejects an incomplete role contract", () => { + expect(HarnessOrchestrator.select(contract("forced", config("evolution")))).toMatchObject({ + topology: "evolution", + source: "contract", + }) + expect(() => + HarnessOrchestrator.select( + contract("forced-invalid", { ...config("tournament"), roles: ["generation", "verification"] }), + ), + ).toThrow("do not permit") + expect(() => + HarnessContract.Orchestration.parse({ + ...config("auto"), + adaptive: { + protocolVersion: "marginal-utility-v1", + minRounds: 1, + patience: 1, + minUtilityGain: 0.01, + maxUncertainty: 0.05, + }, + }), + ).toThrow("explicit evolution") + expect(() => HarnessContract.Orchestration.parse(config("verifier_loop"))).toThrow("repair contract") + expect(() => + HarnessContract.Orchestration.parse({ + ...config("centralized"), + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }), + ).toThrow("explicit verifier_loop") + }) + + test("accepts only after a complete high-confidence verifier panel", async () => { + const sessionID = "repair-accept" + const orchestration: HarnessContract.Orchestration = { + ...config("verifier_loop"), + maxRounds: 3, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + } + await bind(sessionID, orchestration) + const initial = await HarnessOrchestrator.initialize(sessionID) + expect(initial).toMatchObject({ + protocolVersion: "coalition-v3", + repair: { phase: "producing", routes: [] }, + }) + expect(initial.work[initial.order[0]!]!.allocation.steps).toBe(7) + const candidate = HarnessOrchestrator.ready(initial)[0]! + const proposed = await finish(sessionID, candidate, "repair-author", done("candidate")) + const verifiers = HarnessOrchestrator.ready(proposed) + expect(verifiers).toHaveLength(2) + const first = await finish(sessionID, verifiers[0]!, "repair-verifier-a", { + ...done("verified-a"), + verdict: routed("support", "none"), + }) + expect(first.status).toBe("active") + expect(first.consensus).toBeUndefined() + const settled = await finish(sessionID, HarnessOrchestrator.ready(first)[0]!, "repair-verifier-b", { + ...done("verified-b"), + verdict: routed("support", "none"), + }) + expect(settled).toMatchObject({ + status: "completed", + repair: { phase: "completed", stopReason: "accepted" }, + consensus: { status: "supported", verifierCount: 2, support: 2, reject: 0, abstain: 0 }, + }) + expect(settled.repair!.routes.map((item) => item.decision)).toEqual(["accept"]) + expect(HarnessOrchestrator.State.safeParse(settled).success).toBe(true) + const file = path.join(Global.Path.data, "harness", "orchestration", `${encodeURIComponent(sessionID)}.json`) + const stored = JSON.parse(await fs.readFile(file, "utf8")) as HarnessOrchestrator.State + stored.repair!.minConfidence = 0.75 + await fs.writeFile(file, JSON.stringify(stored)) + await expect(HarnessOrchestrator.read(sessionID)).rejects.toThrow("bound contract") + }) + + test("requires repair-panel severity after Task execution", async () => { + const sessionID = "repair-severity" + await bind(sessionID, { + ...config("verifier_loop"), + maxRounds: 1, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }) + const initial = await HarnessOrchestrator.initialize(sessionID) + const proposed = await finish( + sessionID, + HarnessOrchestrator.ready(initial)[0]!, + "severity-author", + done("candidate"), + ) + const verifier = HarnessOrchestrator.ready(proposed)[0]! + await attest(sessionID, verifier, "severity-verifier") + await expect( + HarnessOrchestrator.complete({ + sessionID, + workID: verifier.id, + workerSessionID: "severity-verifier", + result: { ...done("missing-severity"), verdict: verdict("support") }, + }), + ).rejects.toThrow("severity classification") + }) + + test("routes localized defects to revision and excludes stale panels from final consensus", async () => { + const sessionID = "repair-revise" + await bind(sessionID, { + ...config("verifier_loop"), + maxRounds: 3, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }) + const initial = await HarnessOrchestrator.initialize(sessionID) + const candidate = HarnessOrchestrator.ready(initial)[0]! + const proposed = await finish(sessionID, candidate, "revise-author", done("candidate-with-local-defect")) + const panel = HarnessOrchestrator.ready(proposed) + const rejected = await finish(sessionID, panel[0]!, "revise-verifier-a", { + ...done("localized-counterexample"), + verdict: routed("reject", "minor"), + }) + const routedState = await finish(sessionID, HarnessOrchestrator.ready(rejected)[0]!, "revise-verifier-b", { + ...done("otherwise-sound"), + verdict: routed("support", "none"), + }) + const revision = HarnessOrchestrator.ready(routedState)[0]! + expect(revision).toMatchObject({ role: "revision", label: "targeted-revision-1" }) + expect(routedState.repair!.routes[0]!.actionID).toBe(revision.id) + expect(revision.context.map((item) => item.role)).toEqual(["generation", "verification", "verification"]) + const repaired = await finish(sessionID, revision, "reviser", done("complete-repaired-candidate")) + const final = HarnessOrchestrator.ready(repaired) + const checked = await finish(sessionID, final[0]!, "fresh-final-a", { + ...done("repair-check-a"), + verdict: routed("support", "none"), + }) + const settled = await finish(sessionID, HarnessOrchestrator.ready(checked)[0]!, "fresh-final-b", { + ...done("repair-check-b"), + verdict: routed("support", "none"), + }) + expect(settled.repair!.routes.map((item) => item.decision)).toEqual(["revise", "accept"]) + expect(settled.repair!.routes[0]!.candidateID).toBe(candidate.id) + expect(settled.repair!.routes[1]!.candidateID).toBe(revision.id) + expect(settled.consensus).toMatchObject({ status: "supported", verifierCount: 2, support: 2, reject: 0 }) + }) + + test("routes critical defects to a clean restart without rejected candidate context", async () => { + const sessionID = "repair-restart" + await bind(sessionID, { + ...config("verifier_loop"), + maxRounds: 2, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }) + const initial = await HarnessOrchestrator.initialize(sessionID) + const candidate = HarnessOrchestrator.ready(initial)[0]! + const proposed = await finish(sessionID, candidate, "restart-author", done("invalid-premise-candidate")) + const panel = HarnessOrchestrator.ready(proposed) + const rejected = await finish(sessionID, panel[0]!, "restart-verifier-a", { + ...done("fatal-counterexample"), + verdict: routed("reject", "critical"), + }) + const routedState = await finish(sessionID, HarnessOrchestrator.ready(rejected)[0]!, "restart-verifier-b", { + ...done("secondary-review"), + verdict: routed("reject", "minor"), + }) + const restart = HarnessOrchestrator.ready(routedState)[0]! + expect(restart).toMatchObject({ role: "generation", label: "clean-restart-1" }) + expect(routedState.repair!.routes[0]!.actionID).toBe(restart.id) + expect(restart.dependencies).toEqual(routedState.repair!.routes[0]!.verifierIDs) + expect(restart.context.map((item) => item.role)).toEqual(["verification", "verification"]) + expect(restart.context.some((item) => item.summary === "invalid-premise-candidate")).toBe(false) + expect(restart.prompt).toContain("Start from a blank solution") + }) + + test("routes abstention to evidence acquisition and stops at the immutable attempt ceiling", async () => { + const sessionID = "repair-investigate" + await bind(sessionID, { + ...config("verifier_loop"), + maxRounds: 2, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }) + const initial = await HarnessOrchestrator.initialize(sessionID) + const candidate = HarnessOrchestrator.ready(initial)[0]! + const proposed = await finish(sessionID, candidate, "investigate-author", done("evidence-limited-candidate")) + const panel = HarnessOrchestrator.ready(proposed) + const uncertain = await finish(sessionID, panel[0]!, "investigate-verifier-a", { + ...done("missing-observation"), + verdict: routed("abstain", "unknown"), + }) + const routedState = await finish(sessionID, HarnessOrchestrator.ready(uncertain)[0]!, "investigate-verifier-b", { + ...done("provisional-support"), + verdict: routed("support", "none"), + }) + const investigation = HarnessOrchestrator.ready(routedState)[0]! + expect(investigation).toMatchObject({ role: "investigation", label: "evidence-investigation-1" }) + expect(routedState.repair!.routes[0]!.actionID).toBe(investigation.id) + const evidenced = await finish(sessionID, investigation, "evidence-worker", done("new-observable-evidence")) + const final = HarnessOrchestrator.ready(evidenced) + expect(final[0]!.context.map((item) => item.role)).toEqual(["generation", "investigation"]) + expect(final[0]!.context.some((item) => item.role === "verification")).toBe(false) + const first = await finish(sessionID, final[0]!, "attempt-two-a", { + ...done("remaining-failure-a"), + verdict: routed("reject", "minor"), + }) + const settled = await finish(sessionID, HarnessOrchestrator.ready(first)[0]!, "attempt-two-b", { + ...done("remaining-failure-b"), + verdict: routed("reject", "minor"), + }) + expect(settled).toMatchObject({ + status: "completed", + repair: { phase: "completed", stopReason: "attempt_limit" }, + consensus: { status: "rejected", verifierCount: 2, reject: 2 }, + }) + expect(settled.repair!.routes.map((item) => item.decision)).toEqual(["investigate", "revise"]) + expect(settled.repair!.routes[1]!.actionID).toBeUndefined() + + const file = path.join(Global.Path.data, "harness", "orchestration", `${encodeURIComponent(sessionID)}.json`) + const stored = JSON.parse(await fs.readFile(file, "utf8")) as HarnessOrchestrator.State + stored.repair!.routes[0]!.decision = "accept" + await fs.writeFile(file, JSON.stringify(stored)) + await expect(HarnessOrchestrator.read(sessionID)).rejects.toThrow("Repair route derivation drifted") + }) + + test("reserves the whole conditional repair budget before starting", async () => { + const sessionID = "repair-budget" + await HarnessContract.bind( + contract( + sessionID, + { + ...config("verifier_loop"), + maxRounds: 3, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }, + { steps: 8 }, + ), + ) + await expect(HarnessOrchestrator.initialize(sessionID)).rejects.toThrow("every orchestration unit") + }) + + test("fails closed when a routed producer cannot return an artifact", async () => { + const sessionID = "repair-work-failed" + await bind(sessionID, { + ...config("verifier_loop"), + maxRounds: 2, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }) + const initial = await HarnessOrchestrator.initialize(sessionID) + const settled = await fail( + sessionID, + HarnessOrchestrator.ready(initial)[0]!, + "failed-repair-author", + "no valid artifact produced", + ) + expect(settled).toMatchObject({ + status: "completed", + repair: { phase: "completed", stopReason: "work_failed", routes: [] }, + consensus: { status: "insufficient", verifierCount: 0 }, + }) + expect(HarnessOrchestrator.ready(settled)).toEqual([]) + }) + + test("persists a restart-safe DAG and unlocks work only after dependencies", async () => { + await bind("dag") + const initial = await HarnessOrchestrator.initialize("dag") + const roots = HarnessOrchestrator.ready(initial) + expect(initial.selection.topology).toBe("tournament") + expect(HarnessOrchestrator.State.safeParse({ ...initial, workerPolicy: "claimed-v1" }).success).toBe(false) + expect(roots.map((work) => work.role)).toEqual(["generation", "generation"]) + + const first = await finish("dag", roots[0]!, "worker-a", done("proposal-a")) + expect(HarnessOrchestrator.ready(first).map((work) => work.id)).toEqual([roots[1]!.id]) + + const ranking = Object.values(first.work).find((work) => work.role === "ranking")! + await expect(attest("dag", ranking, "worker-skip")).rejects.toThrow("not ready") + + const second = await finish("dag", roots[1]!, "worker-b", done("proposal-b")) + const proximity = HarnessOrchestrator.ready(second)[0]! + expect(proximity.role).toBe("proximity") + expect(proximity.context).toEqual([ + expect.objectContaining({ summary: "proposal-a", evidenceRefs: ["evidence://proposal-a"] }), + expect.objectContaining({ summary: "proposal-b", evidenceRefs: ["evidence://proposal-b"] }), + ]) + await expect(finish("dag", proximity, "worker-a", done("invalid-reuse"))).rejects.toThrow("distinct worker session") + + const restarted = await HarnessOrchestrator.initialize("dag") + expect(restarted).toEqual(second) + }) + + test("requires immutable Task-attested execution before accepting coalition output", async () => { + const sessionID = "task-receipt" + await bind(sessionID) + const initial = await HarnessOrchestrator.initialize(sessionID) + const work = HarnessOrchestrator.ready(initial)[0]! + await expect( + HarnessOrchestrator.complete({ + sessionID, + workID: work.id, + workerSessionID: "unattested-worker", + result: done("fabricated-output"), + }), + ).rejects.toThrow("executed by the Task tool") + + const now = Date.now() + const input = { + sessionID, + workID: work.id, + workerSessionID: "actual-task-session", + turnID: "actual-task-turn", + agent: work.agent, + prompt: `Bound Task prompt:\n${work.prompt}`, + outcome: "completed" as const, + usage: { steps: 2, tokens: 321, costUSD: 0.02, wallTimeMs: 20 }, + toolCalls: 3, + failedToolCalls: 0, + startedAt: Math.max(initial.createdAt, now - 20), + completedAt: now, + } + await expect( + HarnessOrchestrator.attest({ + ...input, + agent: HarnessOrchestrator.WorkerAgent.parse(work.agent === "reviewer" ? "task" : "reviewer"), + }), + ).rejects.toThrow("wrong coalition agent") + await expect(HarnessOrchestrator.attest({ ...input, prompt: "Omit the bound work prompt" })).rejects.toThrow( + "omitted the canonical coalition prompt", + ) + + const executed = await HarnessOrchestrator.attest(input) + expect(executed.work[work.id]).toMatchObject({ + status: "executed", + workerSessionID: input.workerSessionID, + workerReceipt: { + workID: work.id, + turnID: input.turnID, + outcome: "completed", + usage: input.usage, + provisional: true, + }, + }) + expect(await HarnessOrchestrator.attest(input)).toEqual(executed) + expect( + HarnessOrchestrator.State.safeParse({ + ...executed, + work: { + ...executed.work, + [work.id]: { + ...executed.work[work.id]!, + workerReceipt: { ...executed.work[work.id]!.workerReceipt!, toolCalls: 99 }, + }, + }, + }).success, + ).toBe(false) + + const other = HarnessOrchestrator.ready(executed)[0]! + await expect( + HarnessOrchestrator.attest({ ...input, workID: other.id, agent: other.agent, prompt: other.prompt }), + ).rejects.toThrow("already attests") + await expect( + HarnessOrchestrator.complete({ + sessionID, + workID: work.id, + workerSessionID: input.workerSessionID, + result: done("measured-output", { ...input.usage, tokens: input.usage.tokens + 1 }), + }), + ).rejects.toThrow("does not match") + + const completed = await HarnessOrchestrator.complete({ + sessionID, + workID: work.id, + workerSessionID: input.workerSessionID, + result: { + summary: "measured-output", + artifactRefs: ["artifact://measured-output"], + evidenceRefs: ["evidence://measured-output"], + }, + }) + expect(completed.work[work.id]!.result!.usage).toEqual(input.usage) + }) + + test("resumes only the exact same producer lane while keeping verification fresh", async () => { + const sessionID = "producer-lanes" + await bind(sessionID, { ...config("evolution"), maxRounds: 1 }) + const initial = await HarnessOrchestrator.initialize(sessionID) + const roots = HarnessOrchestrator.ready(initial) + expect(initial).toMatchObject({ + schemaVersion: 3, + sessionPolicy: "producer-lanes-v1", + workerPolicy: "task-attested-v1", + }) + expect(roots.map((work) => [work.label, work.lane, work.resumeSessionID])).toEqual([ + ["seed-a", "producer-a", undefined], + ["seed-b", "producer-b", undefined], + ]) + expect(roots[0]!.prompt).toContain("propose, test, repair, and critique") + expect(roots[0]!.prompt).toContain("cannot certify a benchmark result") + expect( + HarnessOrchestrator.State.safeParse({ + ...initial, + work: { ...initial.work, [roots[0]!.id]: { ...initial.work[roots[0]!.id]!, lane: "producer-b" } }, + }).success, + ).toBe(false) + + const first = await finish(sessionID, roots[0]!, "lane-a-session", done("seed-a")) + const second = await finish(sessionID, roots[1]!, "lane-b-session", done("seed-b")) + expect(first.work[roots[0]!.id]!.workerSessionID).toBe("lane-a-session") + + const evolve = async (state: HarnessOrchestrator.State): Promise => { + if (HarnessOrchestrator.ready(state).some((work) => work.role === "evolution")) return state + const work = HarnessOrchestrator.ready(state)[0]! + return evolve(await finish(sessionID, work, `fresh-${state.revision}`, done(work.label))) + } + const staged = await evolve(second) + const ready = HarnessOrchestrator.ready(staged) + expect(ready.map((work) => [work.lane, work.resumeSessionID])).toEqual([ + ["producer-a", "lane-a-session"], + ["producer-b", "lane-b-session"], + ]) + + await expect(finish(sessionID, ready[0]!, "substituted-session", done("substituted"))).rejects.toThrow( + "must resume", + ) + await expect(finish(sessionID, ready[1]!, "lane-a-session", done("crossed"))).rejects.toThrow("must resume") + + const restarted = await HarnessOrchestrator.initialize(sessionID) + expect(HarnessOrchestrator.ready(restarted).map((work) => work.resumeSessionID)).toEqual([ + "lane-a-session", + "lane-b-session", + ]) + const a = await finish(sessionID, ready[0]!, "lane-a-session", done("evolved-a")) + const b = await finish(sessionID, ready[1]!, "lane-b-session", done("evolved-b")) + const tampered = { + ...b, + work: { + ...b.work, + [ready[1]!.id]: { ...b.work[ready[1]!.id]!, workerSessionID: "lane-a-session" }, + }, + } + expect(HarnessOrchestrator.State.safeParse(tampered).success).toBe(false) + expect(a.work[ready[0]!.id]!.workerSessionID).toBe("lane-a-session") + + const probe = HarnessOrchestrator.ready(b)[0]! + expect(probe).toMatchObject({ role: "investigation" }) + expect(probe.lane).toBeUndefined() + expect(probe.resumeSessionID).toBeUndefined() + const investigated = await finish(sessionID, probe, "fresh-investigator", done("investigation")) + const verifier = HarnessOrchestrator.ready(investigated)[0]! + expect(verifier).toMatchObject({ role: "verification" }) + expect(verifier.resumeSessionID).toBeUndefined() + await expect( + finish(sessionID, verifier, "lane-a-session", { + ...done("tainted-verification"), + verdict: verdict("support"), + }), + ).rejects.toThrow("distinct worker session") + }) + + test("migrates version-one orchestration into fresh-session compatibility mode", async () => { + const sessionID = "legacy-orchestration" + await bind(sessionID) + const initial = await HarnessOrchestrator.initialize(sessionID) + const file = path.join(Global.Path.data, "harness", "orchestration", `${encodeURIComponent(sessionID)}.json`) + const legacy = { ...rekey(initial, "legacy-v1") } as Record + legacy.schemaVersion = 1 + delete legacy.sessionPolicy + await Bun.write(file, JSON.stringify(legacy)) + + const migrated = await HarnessOrchestrator.read(sessionID) + expect(migrated).toMatchObject({ + schemaVersion: 3, + sessionPolicy: "legacy-v1", + workerPolicy: "claimed-v1", + }) + expect(Object.values(migrated.work).every((work) => work.lane === undefined)).toBe(true) + const work = HarnessOrchestrator.ready(migrated)[0]! + const completed = await HarnessOrchestrator.complete({ + sessionID, + workID: work.id, + workerSessionID: "legacy-fresh-session", + result: done("legacy-work"), + }) + expect(completed.sessionPolicy).toBe("legacy-v1") + }) + + test("migrates version-two orchestration into claimed-worker compatibility mode", async () => { + const sessionID = "version-two-orchestration" + await bind(sessionID) + const initial = await HarnessOrchestrator.initialize(sessionID) + const file = path.join(Global.Path.data, "harness", "orchestration", `${encodeURIComponent(sessionID)}.json`) + const previous = { ...rekey(initial, initial.sessionPolicy) } as Record + previous.schemaVersion = 2 + delete previous.workerPolicy + await Bun.write(file, JSON.stringify(previous)) + + const migrated = await HarnessOrchestrator.read(sessionID) + expect(migrated).toMatchObject({ + schemaVersion: 3, + sessionPolicy: "fresh-v1", + workerPolicy: "claimed-v1", + }) + const work = HarnessOrchestrator.ready(migrated)[0]! + const completed = await HarnessOrchestrator.complete({ + sessionID, + workID: work.id, + workerSessionID: "version-two-worker", + result: done("version-two-output"), + }) + expect(completed.work[work.id]!.status).toBe("completed") + }) + + test("enforces per-role allocation and immutable idempotent completion", async () => { + await bind("budget-over", { ...config("auto"), maxWorkers: 1, minIndependentVerifiers: 1 }) + const over = await HarnessOrchestrator.initialize("budget-over") + const oversized = HarnessOrchestrator.ready(over)[0]! + await expect( + finish("budget-over", oversized, "worker-over", done("overspend", { steps: oversized.allocation.steps! + 1 })), + ).rejects.toThrow("exceeded its steps") + + await bind("budget", { ...config("auto"), maxWorkers: 1, minIndependentVerifiers: 1 }) + const initial = await HarnessOrchestrator.initialize("budget") + const work = HarnessOrchestrator.ready(initial)[0]! + + const input = { + sessionID: "budget", + workID: work.id, + workerSessionID: "worker-ok", + result: done("bounded"), + } + await attest("budget", work, input.workerSessionID, "completed", input.result.usage) + const completed = await HarnessOrchestrator.complete(input) + expect(completed.status).toBe("completed") + expect(await HarnessOrchestrator.complete(input)).toEqual(completed) + await expect(HarnessOrchestrator.complete({ ...input, result: done("mutated") })).rejects.toThrow("immutable") + }) + + test("propagates failures through descendants while preserving independent roots", async () => { + await bind("failure") + const initial = await HarnessOrchestrator.initialize("failure") + const roots = HarnessOrchestrator.ready(initial) + const failed = await fail("failure", roots[0]!, "worker-failed", "solver diverged") + expect(failed.work[roots[0]!.id]!.status).toBe("failed") + expect(HarnessOrchestrator.ready(failed).map((work) => work.id)).toEqual([roots[1]!.id]) + expect(Object.values(failed.work).filter((work) => work.status === "cancelled").length).toBeGreaterThan(0) + + const completed = await finish("failure", roots[1]!, "worker-survivor", done("surviving-root")) + expect(completed.status).toBe("completed") + }) + + test("rejects state whose dependency order no longer forms a DAG", async () => { + await bind("cycle") + const state = await HarnessOrchestrator.initialize("cycle") + const dependent = state.order.find((id) => state.work[id]!.dependencies.length > 0)! + const order = [dependent, ...state.order.filter((id) => id !== dependent)] + expect(HarnessOrchestrator.State.safeParse({ ...state, order }).success).toBe(false) + }) + + test("aggregates blinded verifier disagreement only after the full panel settles", async () => { + await bind("consensus") + const initial = await HarnessOrchestrator.initialize("consensus") + const sessions = new Map() + const decisions = new Map([ + ["independent-verification-1", "support" as const], + ["independent-verification-2", "reject" as const], + ]) + const settle = async (state: HarnessOrchestrator.State): Promise => { + if (state.status === "completed") return state + const work = HarnessOrchestrator.ready(state)[0]! + const worker = `worker-${sessions.size + 1}` + sessions.set(work.id, worker) + const result = { + ...done(work.label), + evidenceRefs: work.role === "verification" ? [`evidence://${work.label}`] : [], + verdict: work.role === "verification" ? verdict(decisions.get(work.label)!) : undefined, + } + const next = await finish("consensus", work, worker, result) + if (work.role === "verification" && next.status === "active") expect(next.consensus).toBeUndefined() + return settle(next) + } + const completed = await settle(initial) + expect(completed.consensus).toMatchObject({ + status: "disputed", + verifierCount: 2, + support: 1, + reject: 1, + abstain: 0, + provisional: true, + }) + expect(completed.consensus!.evidenceRefs).toContain("evidence://verdict-support") + expect(completed.consensus!.evidenceRefs).toContain("evidence://verdict-reject") + }) + + test("does not label one review as consensus and requires observable verifier evidence", async () => { + await bind("single-consensus", { ...config("tournament"), minIndependentVerifiers: 1 }) + const advance = async (state: HarnessOrchestrator.State, count = 0): Promise => { + if (state.status === "completed") return state + const work = HarnessOrchestrator.ready(state)[0]! + const worker = `worker-${count}` + const result = { + ...done(work.label), + evidenceRefs: work.role === "verification" ? ["evidence://single"] : [], + verdict: work.role === "verification" ? verdict("support") : undefined, + } + await attest("single-consensus", work, worker, "completed", result.usage) + if (work.role === "verification") { + await expect( + HarnessOrchestrator.complete({ + sessionID: "single-consensus", + workID: work.id, + workerSessionID: worker, + result: { ...done("missing-evidence"), evidenceRefs: [], verdict: verdict("support") }, + }), + ).rejects.toThrow("observable evidence") + } + const next = await HarnessOrchestrator.complete({ + sessionID: "single-consensus", + workID: work.id, + workerSessionID: worker, + result, + }) + return advance(next, count + 1) + } + const completed = await advance(await HarnessOrchestrator.initialize("single-consensus")) + expect(completed.consensus).toMatchObject({ status: "insufficient", verifierCount: 1, support: 1 }) + }) + + test("gates evolution rounds on external marginal utility and preserves final verification after early stop", async () => { + const sessionID = "adaptive-stall" + const orchestration: HarnessContract.Orchestration = { + ...config("evolution"), + maxRounds: 3, + adaptive: { + protocolVersion: "marginal-utility-v1", + minRounds: 2, + patience: 1, + minUtilityGain: 0.05, + maxUncertainty: 0.05, + }, + } + const bound = await HarnessContract.bind(contract(sessionID, orchestration)) + const first = await advance(sessionID, await HarnessOrchestrator.initialize(sessionID)) + expect(first).toMatchObject({ protocolVersion: "coalition-v2", status: "awaiting_checkpoint" }) + expect(HarnessOrchestrator.ready(first)).toEqual([]) + await expect( + HarnessOrchestrator.checkpoint( + { + sessionID, + round: 2, + utility: 0.5, + uncertainty: 0.01, + evidenceRefs: ["evidence://premature"], + evaluatedAt: Date.now(), + }, + bound, + ), + ).rejects.toThrow("round 1") + await expect( + HarnessOrchestrator.checkpoint( + { + sessionID, + round: 1, + utility: 0.5, + uncertainty: 0.01, + evidenceRefs: ["evidence://stale"], + evaluatedAt: bound.createdAt, + }, + bound, + ), + ).rejects.toThrow("predates") + + const one = { + sessionID, + round: 1, + utility: 0.5, + uncertainty: 0.01, + evidenceRefs: ["evidence://round-1"], + evaluatedAt: Date.now(), + } + const resumed = await HarnessOrchestrator.checkpoint(one, bound) + expect(resumed).toMatchObject({ status: "active", adaptive: { stalled: 0, phase: "searching" } }) + expect(await HarnessOrchestrator.checkpoint(one, bound)).toEqual(resumed) + await expect(HarnessOrchestrator.checkpoint({ ...one, utility: 0.6 }, bound)).rejects.toThrow("immutable") + + const second = await advance(sessionID, resumed) + expect(second).toMatchObject({ status: "awaiting_checkpoint", adaptive: { checkpoints: [{ round: 1 }] } }) + const stopped = await HarnessOrchestrator.checkpoint( + { + sessionID, + round: 2, + utility: 0.51, + uncertainty: 0.01, + evidenceRefs: ["evidence://round-2"], + evaluatedAt: Date.now(), + }, + bound, + ) + expect(stopped).toMatchObject({ + status: "active", + adaptive: { phase: "finalizing", stalled: 1, stopReason: "marginal_utility_exhausted" }, + }) + expect(Object.values(stopped.work).filter((item) => item.status === "cancelled").length).toBeGreaterThan(0) + expect(HarnessOrchestrator.ready(stopped).map((item) => item.role)).toEqual(["investigation"]) + + const completed = await advance(sessionID, stopped) + expect(completed).toMatchObject({ status: "completed", consensus: { status: "supported", verifierCount: 2 } }) + }) + + test("does not let uncertain utility stop search and detects checkpoint storage tampering", async () => { + const sessionID = "adaptive-uncertain" + const orchestration: HarnessContract.Orchestration = { + ...config("evolution"), + maxRounds: 2, + adaptive: { + protocolVersion: "marginal-utility-v1", + minRounds: 1, + patience: 1, + minUtilityGain: 0.05, + maxUncertainty: 0.05, + targetUtility: 0.8, + }, + } + const bound = await HarnessContract.bind(contract(sessionID, orchestration)) + const first = await advance(sessionID, await HarnessOrchestrator.initialize(sessionID)) + const resumed = await HarnessOrchestrator.checkpoint( + { + sessionID, + round: 1, + utility: 0.99, + uncertainty: 0.5, + evidenceRefs: ["evidence://uncertain"], + evaluatedAt: Date.now(), + }, + bound, + ) + expect(resumed).toMatchObject({ + status: "active", + adaptive: { phase: "searching", stalled: 0, checkpoints: [{ qualified: false }] }, + }) + const second = await advance(sessionID, resumed) + const finalizing = await HarnessOrchestrator.checkpoint( + { + sessionID, + round: 2, + utility: 0.99, + uncertainty: 0.5, + evidenceRefs: ["evidence://still-uncertain"], + evaluatedAt: Date.now(), + }, + bound, + ) + expect(finalizing).toMatchObject({ adaptive: { phase: "finalizing", stopReason: "max_rounds" } }) + + const file = path.join(Global.Path.data, "harness", "orchestration", `${encodeURIComponent(sessionID)}.json`) + const data = await Bun.file(file).json() + data.adaptive.checkpoints[0].utility = 0.1 + await Bun.write(file, JSON.stringify(data)) + await expect(HarnessOrchestrator.read(sessionID)).rejects.toThrow() + }) + + test("honors the minimum search depth before a qualified target can stop evolution", async () => { + const sessionID = "adaptive-target" + const orchestration: HarnessContract.Orchestration = { + ...config("evolution"), + maxRounds: 3, + adaptive: { + protocolVersion: "marginal-utility-v1", + minRounds: 2, + patience: 1, + minUtilityGain: 0.05, + maxUncertainty: 0.05, + targetUtility: 0.8, + }, + } + const bound = await HarnessContract.bind(contract(sessionID, orchestration)) + const first = await advance(sessionID, await HarnessOrchestrator.initialize(sessionID)) + const resumed = await HarnessOrchestrator.checkpoint( + { + sessionID, + round: 1, + utility: 0.9, + uncertainty: 0.01, + evidenceRefs: ["evidence://target-before-minimum"], + evaluatedAt: Date.now(), + }, + bound, + ) + expect(resumed.adaptive).toMatchObject({ phase: "searching" }) + expect(resumed.adaptive?.stopReason).toBeUndefined() + + const second = await advance(sessionID, resumed) + const stopped = await HarnessOrchestrator.checkpoint( + { + sessionID, + round: 2, + utility: 0.91, + uncertainty: 0.01, + evidenceRefs: ["evidence://qualified-target"], + evaluatedAt: Date.now(), + }, + bound, + ) + expect(stopped.adaptive).toMatchObject({ phase: "finalizing", stopReason: "target_reached" }) + expect(HarnessOrchestrator.ready(stopped).map((item) => item.role)).toEqual(["investigation"]) + }) +}) diff --git a/backend/cli/test/session/harness-profile.test.ts b/backend/cli/test/session/harness-profile.test.ts new file mode 100644 index 00000000..c42ead49 --- /dev/null +++ b/backend/cli/test/session/harness-profile.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from "bun:test" +import { HarnessProfile } from "../../src/session/harness/profile" +import type { HarnessContract } from "../../src/session/harness/contract" + +const cases: Array<[string, string, HarnessContract.Profile]> = [ + ["research", "Optimize this Kaggle benchmark score and iterate on submissions.", "optimize"], + ["research", "Reproduce the main result from this paper and its experiment.", "reproduce"], + ["physics", "Derive the Hamiltonian for this theoretical system.", "theory"], + ["physics", "Build a finite element PDE solver and verify it.", "numerical"], + ["ml", "Post-train this model with SFT and GRPO.", "training"], + ["ml", "Train and evaluate a weather forecasting model.", "forecast"], +] + +describe("harness profile router", () => { + test("keeps ordinary work on the direct ReAct control", () => { + const result = HarnessProfile.classify({ agent: "research", text: "Summarize the methods section in this file." }) + expect(result).toMatchObject({ id: "react", source: "control", confidence: 1 }) + expect(result.prompt).toContain("smallest reliable path") + expect(HarnessProfile.classify({ agent: "research", text: "Improve the benchmark documentation." }).id).toBe( + "react", + ) + }) + + test.each(cases)("routes %s tasks with strong observable evidence", (agent, text, id) => { + expect(HarnessProfile.classify({ agent, text }).id).toBe(id) + }) + + test("lets an explicit benchmark contract override keyword heuristics", () => { + const contract = { + profile: "numerical", + runID: "run-contract", + } as HarnessContract.Info + const result = HarnessProfile.classify({ + agent: "research", + text: "Optimize the leaderboard score.", + contract, + }) + expect(result).toMatchObject({ id: "numerical", source: "contract", confidence: 1 }) + expect(result.reasons).toEqual(["contract:run-contract"]) + }) +}) diff --git a/backend/cli/test/session/harness-replication.test.ts b/backend/cli/test/session/harness-replication.test.ts new file mode 100644 index 00000000..205dc3b5 --- /dev/null +++ b/backend/cli/test/session/harness-replication.test.ts @@ -0,0 +1,511 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessOrchestrator } from "../../src/session/harness/orchestrator" +import { HarnessReplication } from "../../src/session/harness/replication" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const receipts = new Set() +const token = "replicated-evaluator-capability-token-0000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const digest = (value: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "orchestration", "reports", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await Promise.all( + [...sessions].map((sessionID) => + fs.rm(path.join(Global.Path.data, "harness", "replications", "subjects", digest(sessionID)), { + recursive: true, + force: true, + }), + ), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "replications", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + receipts.clear() +}) + +function protocol( + input: { + estimator?: HarnessContract.ReplicationEstimator + direction?: "maximize" | "minimize" | "pass" + target?: number + strata?: number + clusters?: number + width?: number + } = {}, +) { + const estimator = input.estimator ?? "mean" + const direction = input.direction ?? "maximize" + return HarnessContract.Replication.parse({ + protocolVersion: "replicated-evaluation-v1", + validatorSHA256: hash("replication-validator-v1"), + environmentSHA256: hash("locked-replication-environment"), + sampling: { + design: "crossed-stratified-cluster-v1", + stratumKind: "benchmark task", + clusterKind: "independent seed", + strata: Array.from({ length: input.strata ?? 2 }, (_, index) => ({ + id: `task-${index}`, + commitmentSHA256: hash(`task-${index}`), + })).toSorted((left, right) => left.id.localeCompare(right.id)), + clusters: Array.from({ length: input.clusters ?? 5 }, (_, index) => ({ + id: `seed-${index}`, + commitmentSHA256: hash(`seed-${index}`), + })).toSorted((left, right) => left.id.localeCompare(right.id)), + }, + estimator, + interval: + estimator === "pass_rate" + ? { method: "wilson-score-v1", confidence: 0.95 } + : { method: "stratified-bootstrap-percentile-v1", confidence: 0.95, resamples: 1_000, seed: 1729 }, + decision: { + rule: "conservative-bound-v1", + direction, + target: input.target ?? 0.75, + maxIntervalWidth: input.width, + }, + failurePolicy: "fail-closed", + }) +} + +function task( + sessionID: string, + input: { + replication?: HarnessContract.Replication + profile?: HarnessContract.Profile + candidates?: number + } = {}, +): HarnessAdapter.Task { + sessions.add(sessionID) + const replication = input.replication ?? protocol() + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "replicated-statistical-result", + split: "validation", + evaluator: { name: "official-evaluator", version: "7", source: "benchmark", token }, + objective: "Produce a result that remains above the target across frozen tasks and independent seeds", + profile: input.profile, + replication, + metric: { + name: replication.estimator === "pass_rate" ? "pass_rate" : "score", + direction: replication.decision.direction, + target: replication.decision.target, + }, + model: { provider: "test", name: "model" }, + tools: ["read", "bash"], + skills: [], + budget: { steps: 30, candidates: input.candidates }, + seed: 17, + intervention: "autonomous", + contamination: { policy: "frozen units remain evaluator-owned", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +function observations( + contract: HarnessContract.Info, + input: { + scores?: number[] + statuses?: Array<"passed" | "failed" | "inconclusive"> + evaluatedAt?: number + } = {}, +) { + const replication = contract.replication! + const units = replication.sampling.strata.flatMap((stratum) => + replication.sampling.clusters.map((cluster) => ({ stratum, cluster })), + ) + return units.map((unit, index) => + HarnessReplication.Observation.parse({ + stratumID: unit.stratum.id, + clusterID: unit.cluster.id, + stratumSHA256: unit.stratum.commitmentSHA256, + clusterSHA256: unit.cluster.commitmentSHA256, + status: input.statuses?.[index] ?? "passed", + score: + replication.estimator === "pass_rate" || (input.statuses?.[index] ?? "passed") !== "passed" + ? undefined + : (input.scores?.[index] ?? 0.9), + outputSHA256: hash(`${contract.sessionID}:${unit.stratum.id}:${unit.cluster.id}:output`), + environmentSHA256: hash("locked-replication-environment"), + evidence: [`replicate:${unit.stratum.id}/${unit.cluster.id}.json`], + evaluatedAt: input.evaluatedAt ?? Math.max(Date.now(), contract.createdAt), + }), + ) +} + +async function replicate( + contract: HarnessContract.Info, + input = observations(contract), + subject: HarnessReplication.Subject = { type: "run", id: contract.runID }, +) { + const receipt = await HarnessReplication.record( + { sessionID: contract.sessionID, evaluatorToken: token, subject, observations: input }, + await HarnessAdapter.authorize(contract.sessionID, token), + ) + receipts.add(receipt.receiptID) + return receipt +} + +function evaluation(contract: HarnessContract.Info, receipt?: HarnessReplication.Receipt, candidateID?: string) { + const checks = HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + candidateID, + replicationReceiptID: receipt?.receiptID, + status: "passed", + score: receipt?.statistics.estimate ?? 0.99, + metrics: { [contract.benchmark.metric!]: receipt?.statistics.estimate ?? 0.99 }, + checks, + evidence: ["official:aggregate.json"], + evaluatedAt: Math.max(Date.now(), receipt?.evaluatedAt ?? 0), + }) +} + +describe("replicated evaluation and conservative promotion", () => { + test("injects the full-grid and conservative-bound policy into main and coalition contexts", async () => { + const contract = await HarnessAdapter.bind(task("replication-prompt")) + const prompt = await HarnessReplication.context(contract.sessionID) + expect(prompt).toContain("2 benchmark task × 5 independent seed clusters (10 units)") + expect(prompt).toContain("conservative confidence bound, not the best replicate") + expect(prompt).not.toContain(token) + + const state = await HarnessOrchestrator.initialize(contract.sessionID) + expect(HarnessOrchestrator.ready(state)[0]!.prompt).toContain(prompt) + }) + + test("promotes only the backend aggregate whose lower bound clears the target", async () => { + const contract = await HarnessAdapter.bind(task("replication-pass")) + await expect(HarnessAdapter.ingest(evaluation(contract))).rejects.toThrow("replicated evaluation receipt") + const receipt = await replicate(contract) + expect(receipt).toMatchObject({ + status: "passed", + statistics: { units: 10, passed: 10 }, + subject: { type: "run", id: contract.runID }, + }) + expect(receipt.statistics.estimate).toBeCloseTo(0.9) + expect(receipt.statistics.conservativeBound).toBeCloseTo(0.9) + expect(JSON.stringify(receipt)).not.toContain(token) + + await expect( + HarnessAdapter.ingest({ + ...evaluation(contract, receipt), + score: 0.99, + metrics: { score: 0.99 }, + }), + ).rejects.toThrow("backend-derived replicated estimate") + + const result = await HarnessAdapter.ingest(evaluation(contract, receipt)) + const report = HarnessReport.compile({ contract, evaluations: [result.evaluation], generatedAt: Date.now() }) + expect(result.evaluation).toMatchObject({ status: "passed", replicationReceiptID: receipt.receiptID }) + expect(result.evaluation.score).toBeCloseTo(0.9) + expect(report.quality.replicationReceiptID).toBe(receipt.receiptID) + }) + + test("rejects environment drift from the precommitted replication environment", async () => { + const contract = await HarnessAdapter.bind(task("replication-environment")) + const changed = observations(contract) + changed[0] = { ...changed[0]!, environmentSHA256: hash("drifted-environment") } + await expect(replicate(contract, changed)).rejects.toThrow("changed the frozen environment") + }) + + test("rejects task or cluster substitution behind a valid frozen unit ID", async () => { + const contract = await HarnessAdapter.bind(task("replication-axis-substitution")) + const changed = observations(contract) + changed[0] = { ...changed[0]!, clusterSHA256: hash("substituted-cluster") } + await expect(replicate(contract, changed)).rejects.toThrow("changed a frozen axis commitment") + }) + + test("freezes one canonical receipt per subject while preserving exact retry idempotency", async () => { + const contract = await HarnessAdapter.bind(task("replication-single-receipt")) + const first = observations(contract, { scores: Array(10).fill(0.1) }) + const [receipt, concurrent] = await Promise.all([replicate(contract, first), replicate(contract, first)]) + expect(concurrent.receiptID).toBe(receipt.receiptID) + const retry = await replicate(contract, first) + expect(retry.receiptID).toBe(receipt.receiptID) + await expect(replicate(contract, observations(contract, { scores: Array(10).fill(0.99) }))).rejects.toThrow( + "selective retries are forbidden", + ) + }) + + test("derives finite aggregates without overflowing valid extreme scores", async () => { + const extremeProtocol = protocol({ target: 1e307, strata: 1, clusters: 5 }) + const contract = await HarnessAdapter.bind(task("replication-stable-aggregate", { replication: extremeProtocol })) + const receipt = await replicate(contract, observations(contract, { scores: Array(5).fill(1e308) })) + expect(receipt.status).toBe("passed") + expect(receipt.statistics.estimate).toBe(1e308) + expect(receipt.statistics.interval).toEqual([1e308, 1e308]) + }) + + test("rejects lucky-best promotion when the conservative bound fails", async () => { + const contract = await HarnessAdapter.bind(task("replication-lucky-best")) + const scores = [0.99, 0.1, 0.1, 0.1, 0.1, 0.99, 0.1, 0.1, 0.1, 0.1] + const receipt = await replicate(contract, observations(contract, { scores })) + expect(receipt.statistics.estimate).toBeCloseTo(0.278) + expect(receipt.statistics.conservativeBound).toBeLessThan(0.75) + expect(receipt.status).toBe("failed") + await expect(HarnessAdapter.ingest(evaluation(contract, receipt))).rejects.toThrow( + "passing conservative replication receipt", + ) + }) + + test("uses the upper bound for minimized metrics and enforces precision limits", async () => { + const minimizeProtocol = protocol({ direction: "minimize", target: 0.2 }) + const minimize = await HarnessAdapter.bind(task("replication-minimize", { replication: minimizeProtocol })) + const minimized = await replicate(minimize, observations(minimize, { scores: Array(10).fill(0.1) })) + expect(minimized.status).toBe("passed") + expect(minimized.statistics.conservativeBound).toBeCloseTo(0.1) + const accepted = await HarnessAdapter.ingest(evaluation(minimize, minimized)) + expect(accepted.evaluation.status).toBe("passed") + + const narrowProtocol = protocol({ target: 0, strata: 1, clusters: 5, width: 0.01 }) + const narrow = await HarnessAdapter.bind(task("replication-width", { replication: narrowProtocol })) + const imprecise = await replicate(narrow, observations(narrow, { scores: [0, 1, 2, 3, 4] })) + expect(imprecise.statistics.conservativeBound).toBeGreaterThanOrEqual(0) + expect(imprecise.statistics.intervalWidth).toBeGreaterThan(0.01) + expect(imprecise.failures).toContainEqual(expect.stringContaining("interval width")) + expect(imprecise.status).toBe("failed") + }) + + test("freezes direction, target, sample size, and deterministic bootstrap semantics", async () => { + expect(() => protocol({ clusters: 4 })).toThrow("at least five independent clusters") + const mismatched = task("replication-decision-drift") + await expect( + HarnessAdapter.bind({ ...mismatched, metric: { name: "score", direction: "maximize", target: 0.8 } }), + ).rejects.toThrow("decision must match") + + const first = await HarnessAdapter.bind(task("replication-deterministic-a")) + const second = await HarnessAdapter.bind(task("replication-deterministic-b")) + const scores = [0.2, 0.4, 0.6, 0.8, 1, 0.3, 0.5, 0.7, 0.9, 1.1] + const one = await replicate(first, observations(first, { scores })) + const two = await replicate(second, observations(second, { scores })) + expect(one.statistics).toEqual(two.statistics) + }) + + test("requires the exact frozen grid and rejects duplicate pseudo-replicates", async () => { + const contract = await HarnessAdapter.bind(task("replication-grid")) + const complete = observations(contract) + await expect(replicate(contract, complete.slice(1))).rejects.toThrow("complete frozen") + await expect(replicate(contract, [...complete.slice(0, -1), complete[0]!])).rejects.toThrow("must be unique") + + const base = protocol() + expect(() => + HarnessContract.Replication.parse({ + ...base, + sampling: { + ...base.sampling, + clusters: base.sampling.clusters.map((item, index) => ({ + ...item, + commitmentSHA256: index ? item.commitmentSHA256 : base.sampling.clusters[1]!.commitmentSHA256, + })), + }, + }), + ).toThrow("commitments must be unique") + }) + + test("fails closed on numeric replicate failures and preserves inconclusive units", async () => { + const failed = await HarnessAdapter.bind(task("replication-unit-failed")) + const failedReceipt = await replicate( + failed, + observations(failed, { + statuses: ["failed", "passed", "passed", "passed", "passed", "passed", "passed", "passed", "passed", "passed"], + }), + ) + expect(failedReceipt.status).toBe("failed") + expect(failedReceipt.statistics.estimate).toBeUndefined() + expect(failedReceipt.failures).toContain("unit:task-0/seed-0:failed") + + const uncertain = await HarnessAdapter.bind(task("replication-unit-inconclusive")) + const uncertainReceipt = await replicate( + uncertain, + observations(uncertain, { + statuses: [ + "inconclusive", + "passed", + "passed", + "passed", + "passed", + "passed", + "passed", + "passed", + "passed", + "passed", + ], + }), + ) + expect(uncertainReceipt.status).toBe("inconclusive") + expect(uncertainReceipt.statistics.inconclusive).toBe(1) + }) + + test("computes IQM and Wilson intervals rather than trusting submitted summaries", async () => { + const iqmProtocol = protocol({ estimator: "iqm", target: 0, strata: 1, clusters: 5 }) + const iqm = await HarnessAdapter.bind(task("replication-iqm", { replication: iqmProtocol })) + const iqmReceipt = await replicate(iqm, observations(iqm, { scores: [0, 1, 2, 3, 100] })) + expect(iqmReceipt.statistics.estimate).toBe(2) + expect(iqmReceipt.statistics.method).toBe("stratified-bootstrap-percentile-v1") + + expect(() => protocol({ estimator: "pass_rate", direction: "pass", strata: 2 })).toThrow( + "one stratum of independent Bernoulli clusters", + ) + const passProtocol = protocol({ estimator: "pass_rate", direction: "pass", target: 0.8, strata: 1, clusters: 20 }) + const binary = await HarnessAdapter.bind(task("replication-wilson", { replication: passProtocol })) + const passReceipt = await replicate(binary) + expect(passReceipt.statistics).toMatchObject({ estimate: 1, method: "wilson-score-v1" }) + expect(passReceipt.statistics.conservativeBound).toBeGreaterThan(0.8) + expect(passReceipt.status).toBe("passed") + + const statuses = Array.from({ length: 20 }, (_, index) => (index < 16 ? ("passed" as const) : ("failed" as const))) + const weak = await HarnessAdapter.bind(task("replication-wilson-weak", { replication: passProtocol })) + const weakReceipt = await replicate(weak, observations(weak, { statuses })) + expect(weakReceipt.statistics.estimate).toBe(0.8) + expect(weakReceipt.statistics.conservativeBound).toBeLessThan(0.8) + expect(weakReceipt.status).toBe("failed") + }) + + test("binds observations and receipts to candidate birth, session, and subject", async () => { + const contract = await HarnessAdapter.bind(task("replication-candidate", { profile: "optimize", candidates: 2 })) + const missing = { type: "candidate" as const, id: hash("future-candidate") } + await expect(replicate(contract, observations(contract), missing)).rejects.toThrow("does not exist") + + const state = await HarnessSearch.initialize({ sessionID: contract.sessionID, candidates: 2 }) + const recommendation = HarnessSearch.recommend(state) + const added = await HarnessSearch.add({ + sessionID: contract.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "replication-candidate", + proposal: "Evaluate this immutable candidate across the complete frozen grid", + artifact: { uri: "candidate://replicated", sha256: hash("replicated-candidate") }, + }) + const candidate = (await HarnessSearch.read(contract.sessionID)).candidates[added.id]! + await expect( + replicate(contract, observations(contract, { evaluatedAt: candidate.createdAt - 1 }), { + type: "candidate", + id: candidate.id, + }), + ).rejects.toThrow("subject interval") + + const receipt = await replicate(contract, observations(contract), { type: "candidate", id: candidate.id }) + const result = await HarnessAdapter.ingest(evaluation(contract, receipt, candidate.id)) + expect(result.evaluation).toMatchObject({ subject: { type: "candidate", id: candidate.id }, status: "passed" }) + + const other = await HarnessAdapter.bind(task("replication-other-session")) + await expect(HarnessAdapter.ingest(evaluation(other, receipt))).rejects.toThrow("different harness session") + }) + + test("prevents post-hoc receipts after a final evaluation", async () => { + const contract = await HarnessAdapter.bind(task("replication-post-hoc")) + const checks = evaluation(contract).checks + await HarnessAdapter.ingest({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + status: "failed", + score: 0.2, + metrics: { score: 0.2 }, + checks, + evidence: ["official:failed-final.json"], + evaluatedAt: Date.now(), + }) + await expect(replicate(contract)).rejects.toThrow("before the subject's final evaluation") + }) + + test("detects both ordinary tampering and content-addressed derived-state forgery", async () => { + const contract = await HarnessAdapter.bind(task("replication-tamper")) + const receipt = await replicate(contract) + const target = path.join(Global.Path.data, "harness", "replications", `${receipt.receiptID}.json`) + await Bun.write(target, JSON.stringify({ ...receipt, recordedAt: receipt.recordedAt + 1 })) + expect(await HarnessReplication.read(receipt.receiptID)).toBeNull() + + const forged = { + ...receipt, + status: "failed" as const, + failures: ["forged verdict"], + statistics: { ...receipt.statistics, estimate: 0.99 }, + } + const stable = structuredClone(forged) as Record + delete stable.receiptID + const receiptID = digest(stable) + receipts.add(receiptID) + await Bun.write(target.replace(receipt.receiptID, receiptID), JSON.stringify({ ...forged, receiptID })) + expect(await HarnessReplication.read(receiptID)).not.toBeNull() + await expect( + HarnessReplication.assert({ + contract, + receiptID, + subject: { type: "run", id: contract.runID }, + score: 0.99, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + requirePassed: false, + }), + ).rejects.toThrow("non-canonical receipt") + }) + + test("exposes replicated receipts only through the evaluator capability", async () => { + const contract = await HarnessAdapter.bind(task("replication-route")) + const app = HarnessRoutes() + const response = await app.request("/replications/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID: contract.sessionID, + evaluatorToken: token, + subject: { type: "run", id: contract.runID }, + observations: observations(contract), + }), + }) + expect(response.status).toBe(200) + const receipt = (await response.json()) as HarnessReplication.Receipt + receipts.add(receipt.receiptID) + + const denied = await app.request(`/replications/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID: contract.sessionID, + evaluatorToken: "wrong-token-0000000000000000000000000000", + }), + }) + expect(denied.status).not.toBe(200) + + const read = await app.request(`/replications/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: token }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, status: "passed" }) + }) +}) diff --git a/backend/cli/test/session/harness-report.test.ts b/backend/cli/test/session/harness-report.test.ts new file mode 100644 index 00000000..abe9994b --- /dev/null +++ b/backend/cli/test/session/harness-report.test.ts @@ -0,0 +1,532 @@ +import { describe, expect, test } from "bun:test" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" + +function contract(runID: string, direction: "maximize" | "minimize" = "maximize"): HarnessContract.Info { + return HarnessContract.Info.parse({ + schemaVersion: 1, + runID, + sessionID: `session-${runID}`, + objective: "Improve a fixed task", + benchmark: { + name: "mle", + version: "1", + taskID: "task", + split: "held_out", + evaluator: "official", + evaluatorVersion: "1", + evaluatorSource: "benchmark", + metric: "score", + direction, + target: direction === "maximize" ? 0.9 : 0.1, + }, + profile: "optimize", + packs: ["ml"], + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { candidates: 10 }, + seed: 1, + intervention: "autonomous", + contamination: { policy: "hidden", hiddenTestsAccessible: false }, + createdAt: 1, + }) +} + +function evaluation(input: HarnessContract.Info, score: number): HarnessEvaluation.Info { + return HarnessEvaluation.Info.parse({ + schemaVersion: 1, + runID: input.runID, + sessionID: input.sessionID, + evaluator: { name: "official", version: "1", source: "benchmark" }, + status: "passed", + score, + metrics: { score }, + checks: [{ id: "official", status: "passed", blocking: true, evidence: ["receipt"] }], + evidence: ["report"], + evaluatedAt: 2, + }) +} + +const trace = (cost: number, total: number, wall: number): HarnessReport.Trace => ({ + cost, + tokens: { input: total, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + totalCompletionTimeMs: wall, + toolCalls: 4, + searchCount: 1, + dedupeHits: 0, + retryCount: 0, + failureCount: 0, +}) + +describe("harness quality-cost reports", () => { + test("reports quality, target status, tokens, cost, and compatible comparison keys", () => { + const baseline = contract("baseline") + const first = HarnessReport.compile({ + contract: baseline, + evaluations: [evaluation(baseline, 0.8)], + trace: trace(2, 1_000, 5_000), + generatedAt: 3, + }) + const candidate = contract("candidate") + const second = HarnessReport.compile({ + contract: candidate, + evaluations: [evaluation(candidate, 0.9)], + trace: trace(1.5, 900, 4_000), + generatedAt: 3, + }) + expect(first.comparisonKey).toBe(second.comparisonKey) + expect(first.quality.targetReached).toBe(false) + expect(second.quality.targetReached).toBe(true) + expect(second.efficiency.tokens?.total).toBe(900) + expect(HarnessReport.dominates(second, first)).toBe(true) + expect(HarnessReport.frontier([first, second]).map((item) => item.runID)).toEqual(["candidate"]) + expect(HarnessReport.compare([first, second], "baseline")).toContainEqual({ + runID: "candidate", + scoreImprovement: 0.09999999999999998, + costDelta: -0.5, + tokenDelta: -100, + wallTimeDelta: -1000, + pareto: true, + }) + }) + + test("uses direction-aware improvement for minimized metrics", () => { + const base = contract("loss-base", "minimize") + const better = contract("loss-better", "minimize") + const first = HarnessReport.compile({ contract: base, evaluations: [evaluation(base, 0.4)], generatedAt: 3 }) + const second = HarnessReport.compile({ contract: better, evaluations: [evaluation(better, 0.2)], generatedAt: 3 }) + expect(HarnessReport.compare([first, second], "loss-base")[1]?.scoreImprovement).toBeCloseTo(0.2) + expect(HarnessReport.dominates(second, first)).toBe(true) + }) + + test("reports adaptive controller telemetry and keeps static search out of the comparison set", () => { + const adaptive = HarnessContract.Info.parse({ + ...contract("adaptive-report"), + search: HarnessContract.adaptiveSearch, + }) + const search = HarnessSearch.State.parse({ + schemaVersion: 4, + proposalPolicy: "adaptive-v4", + runID: adaptive.runID, + sessionID: adaptive.sessionID, + objective: adaptive.objective, + evaluator: adaptive.benchmark.evaluator, + metric: "score", + direction: "maximize", + objectives: [], + controller: HarnessContract.adaptiveSearch, + population: { mode: "islands", count: 2, initial: 2, topology: "ring", migrationInterval: 3 }, + budget: { candidates: 10, stall: 5 }, + status: "active", + candidates: {}, + reservations: {}, + archiveIDs: [], + stalled: 0, + revision: 0, + startedAt: 1, + updatedAt: 1, + }) + const report = HarnessReport.compile({ + contract: adaptive, + evaluations: [evaluation(adaptive, 0.8)], + search, + generatedAt: 3, + }) + expect(report.search).toMatchObject({ + proposalPolicy: "adaptive-v4", + controller: HarnessContract.adaptiveSearch, + adaptation: { events: 0, stalled: 0, globalStagnation: false }, + }) + const staticContract = contract("static-report") + const staticReport = HarnessReport.compile({ + contract: staticContract, + evaluations: [evaluation(staticContract, 0.8)], + generatedAt: 3, + }) + expect(report.comparisonKey).not.toBe(staticReport.comparisonKey) + expect(() => HarnessReport.compare([report, staticReport], report.runID)).toThrow("only comparable") + }) + + test("refuses cross-task comparisons instead of normalizing unlike metrics", () => { + const first = contract("one") + const other = HarnessContract.Info.parse({ + ...contract("two"), + benchmark: { ...contract("two").benchmark, taskID: "other-task" }, + }) + const a = HarnessReport.compile({ contract: first, evaluations: [evaluation(first, 0.8)], generatedAt: 3 }) + const b = HarnessReport.compile({ contract: other, evaluations: [evaluation(other, 0.9)], generatedAt: 3 }) + expect(HarnessReport.dominates(b, a)).toBe(false) + expect(() => HarnessReport.compare([a, b], "one")).toThrow("only comparable") + }) + + test("keeps different secondary-objective contracts out of one comparison", () => { + const base = contract("objective-base") + const multi = HarnessContract.Info.parse({ + ...contract("objective-multi"), + benchmark: { + ...contract("objective-multi").benchmark, + objectives: [{ metric: "robustness", direction: "maximize" }], + objectiveAudit: { + schemaVersion: 1, + planSHA256: "b".repeat(64), + validatorSHA256: "c".repeat(64), + contractSHA256: "d".repeat(64), + guardIDs: ["semantic-regression"], + }, + }, + }) + const first = HarnessReport.compile({ contract: base, evaluations: [evaluation(base, 0.8)], generatedAt: 3 }) + const result = HarnessEvaluation.Info.parse({ + ...evaluation(multi, 0.8), + metrics: { score: 0.8, robustness: 0.7 }, + }) + const search = HarnessSearch.State.parse({ + schemaVersion: 4, + proposalPolicy: "leased-v3", + runID: multi.runID, + sessionID: multi.sessionID, + objective: multi.objective, + evaluator: multi.benchmark.evaluator, + metric: "score", + direction: "maximize", + objectives: multi.benchmark.objectives, + population: { mode: "islands", count: 2, initial: 2, topology: "ring", migrationInterval: 3 }, + budget: { candidates: 10, stall: 5 }, + status: "active", + candidates: {}, + archiveIDs: [], + stalled: 0, + revision: 0, + startedAt: 1, + updatedAt: 1, + }) + const second = HarnessReport.compile({ contract: multi, evaluations: [result], search, generatedAt: 3 }) + expect(second.search?.objectiveAudit).toEqual(multi.benchmark.objectiveAudit) + expect(first.comparisonKey).not.toBe(second.comparisonKey) + expect(() => HarnessReport.compare([first, second], "objective-base")).toThrow("only comparable") + const changed = HarnessContract.Info.parse({ + ...multi, + runID: "objective-audit-changed", + sessionID: "session-objective-audit-changed", + benchmark: { + ...multi.benchmark, + objectiveAudit: { ...multi.benchmark.objectiveAudit!, contractSHA256: "e".repeat(64) }, + }, + }) + const changedResult = HarnessEvaluation.Info.parse({ + ...result, + runID: changed.runID, + sessionID: changed.sessionID, + }) + const third = HarnessReport.compile({ contract: changed, evaluations: [changedResult], generatedAt: 3 }) + expect(second.comparisonKey).not.toBe(third.comparisonKey) + expect(() => HarnessReport.compare([second, third], "objective-multi")).toThrow("only comparable") + }) + + test("refuses comparisons across different simulator protocols", () => { + const base = contract("simulation-one") + const simulation = HarnessContract.Simulation.parse({ + kind: "pde", + engine: { + name: "solver", + version: "1", + commandSHA256: "a".repeat(64), + configSHA256: "b".repeat(64), + }, + problemSHA256: "c".repeat(64), + reference: { kind: "analytic", identity: "reference", sha256: "d".repeat(64) }, + validation: { + errorNorm: "L2", + minLevels: 3, + expectedOrder: 2, + orderTolerance: 0.2, + maxResidual: 1e-8, + invariantTolerances: { mass_drift: 1e-6 }, + requiredStressTests: ["reference_replay"], + }, + }) + const first = HarnessContract.Info.parse({ ...base, simulation }) + const second = HarnessContract.Info.parse({ + ...contract("simulation-two"), + simulation: { ...simulation, engine: { ...simulation.engine, configSHA256: "e".repeat(64) } }, + }) + const a = HarnessReport.compile({ contract: first, evaluations: [evaluation(first, 0.8)], generatedAt: 3 }) + const b = HarnessReport.compile({ contract: second, evaluations: [evaluation(second, 0.9)], generatedAt: 3 }) + expect(a.comparisonKey).not.toBe(b.comparisonKey) + expect(() => HarnessReport.compare([a, b], "simulation-one")).toThrow("only comparable") + }) + + test("keeps active-audit protocols and receipts in report provenance", () => { + const base = contract("audit-report-base") + const audited = HarnessContract.Info.parse({ + ...contract("audit-report-qualified"), + audit: { mode: "performance", budget: 3, minSamples: 2 }, + }) + const plain = HarnessReport.compile({ contract: base, evaluations: [evaluation(base, 0.8)], generatedAt: 3 }) + const receiptID = "a".repeat(64) + const result = HarnessEvaluation.Info.parse({ ...evaluation(audited, 0.8), auditReceiptID: receiptID }) + const report = HarnessReport.compile({ contract: audited, evaluations: [result], generatedAt: 3 }) + expect(report.quality.auditReceiptID).toBe(receiptID) + expect(report.comparisonKey).not.toBe(plain.comparisonKey) + expect(() => HarnessReport.compare([plain, report], plain.runID)).toThrow("only comparable") + }) + + test("keeps topic-aware failure protocols and evidence-only receipts in report provenance", () => { + const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + const identity = (name: string) => ({ + name, + version: "1", + promptSHA256: hash(`${name}-prompt`), + configSHA256: hash(`${name}-config`), + }) + const base = contract("failure-report-base") + const audited = HarnessContract.Info.parse({ + ...contract("failure-report-stream"), + audit: { mode: "failure", budget: 3, minSamples: 2, failureThreshold: 0.5 }, + failureDiscovery: { + protocolVersion: "topic-aware-failure-v1", + sourcePoolSHA256: hash("source-pool"), + topicModel: { kind: "predefined", identity: identity("topic-model") }, + topics: ["alpha", "beta"].map((id) => ({ id, commitment: hash(`topic-${id}`) })), + generator: identity("generator"), + validators: HarnessContract.FailureValidatorKind.options.map((kind) => ({ + kind, + identity: identity(`${kind}-validator`), + })), + embedding: { identity: identity("embedding"), dimensions: 2 }, + budget: 2, + anchorsPerAttempt: 1, + failureThreshold: 0.5, + }, + }) + const plain = HarnessReport.compile({ contract: base, evaluations: [evaluation(base, 0.8)], generatedAt: 3 }) + const receiptID = "f".repeat(64) + const result = HarnessEvaluation.Info.parse({ + ...evaluation(audited, 0.8), + failureDiscoveryReceiptID: receiptID, + }) + const report = HarnessReport.compile({ contract: audited, evaluations: [result], generatedAt: 3 }) + expect(report.quality.failureDiscoveryReceiptID).toBe(receiptID) + expect(report.quality.score).toBe(0.8) + expect(report.comparisonKey).not.toBe(plain.comparisonKey) + expect(() => HarnessReport.compare([plain, report], plain.runID)).toThrow("only comparable") + }) + + test("keeps scientific synthesis protocols and factuality receipts in report provenance", () => { + const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + const identity = (name: string) => ({ + name, + version: "1", + promptSHA256: hash(`${name}-prompt`), + configSHA256: hash(`${name}-config`), + }) + const audit = HarnessContract.EvaluatorAudit.parse({ + protocolVersion: "evaluator-audit-v1", + auditor: { name: "meta-evaluator", version: "1", source: "external" }, + suite: { name: "synthesis-suite", version: "1", commitmentSHA256: hash("suite") }, + minCleanCases: 2, + minCasesPerFault: 2, + requiredFaults: ["wrong_answer", "unsupported_claim", "data_leakage"], + minSensitivity: 0.8, + minSpecificity: 0.8, + minBalancedAccuracy: 0.8, + minFaultRecall: 0.8, + maxBrierScore: 0.15, + }) + const synthesis = HarnessContract.ScientificSynthesis.parse({ + protocolVersion: "scientific-synthesis-v1", + querySHA256: hash("query"), + referenceSHA256: hash("reference"), + referenceFactsSHA256: hash("facts"), + referenceFactCount: 2, + cutoff: "2026-01-01", + tools: ["paper_search"], + traceSchemaSHA256: hash("trace"), + filterPolicySHA256: hash("filter"), + maxToolEvents: 20, + decomposer: identity("decomposer"), + judges: { precision: identity("precision"), recall: identity("recall") }, + minGeneratedFacts: 2, + minPrecision: 0.4, + minRecall: 0.4, + minF1: 0.4, + cleanRoomRequired: true, + judgeFailurePolicy: "inconclusive", + }) + const build = (runID: string, protocol = synthesis) => + HarnessContract.Info.parse({ + ...contract(runID), + tools: ["paper_search"], + benchmark: { + ...contract(runID).benchmark, + split: "validation", + metric: "factual_f1", + target: 0.4, + }, + contamination: { + policy: "reference and post-cutoff evidence stay private", + hiddenTestsAccessible: false, + publicDataCutoff: "2026-01-01", + }, + evaluatorAudit: audit, + synthesis: protocol, + }) + const first = build("synthesis-report-one") + const second = build("synthesis-report-two", { ...synthesis, referenceSHA256: hash("different-reference") }) + const receiptID = "f".repeat(64) + const result = HarnessEvaluation.Info.parse({ + ...evaluation(first, 0.8), + metrics: { factual_f1: 0.8 }, + synthesisReceiptID: receiptID, + }) + const report = HarnessReport.compile({ contract: first, evaluations: [result], generatedAt: 3 }) + const other = HarnessReport.compile({ contract: second, evaluations: [evaluation(second, 0.8)], generatedAt: 3 }) + expect(report.quality.synthesisReceiptID).toBe(receiptID) + expect(report.comparisonKey).not.toBe(other.comparisonKey) + expect(() => HarnessReport.compare([report, other], first.runID)).toThrow("only comparable") + }) + + test("refuses comparisons across different evaluator qualification protocols", () => { + const audit = HarnessContract.EvaluatorAudit.parse({ + protocolVersion: "evaluator-audit-v1", + auditor: { name: "meta-evaluator", version: "1", source: "external" }, + suite: { name: "judge-suite", version: "1", commitmentSHA256: "a".repeat(64) }, + minCleanCases: 2, + minCasesPerFault: 1, + requiredFaults: ["wrong_answer"], + minSensitivity: 0.8, + minSpecificity: 0.8, + minBalancedAccuracy: 0.8, + minFaultRecall: 0.8, + maxBrierScore: 0.15, + }) + const first = HarnessContract.Info.parse({ ...contract("judge-one"), evaluatorAudit: audit }) + const second = HarnessContract.Info.parse({ + ...contract("judge-two"), + evaluatorAudit: { ...audit, suite: { ...audit.suite, commitmentSHA256: "b".repeat(64) } }, + }) + const a = HarnessReport.compile({ contract: first, evaluations: [evaluation(first, 0.8)], generatedAt: 3 }) + const b = HarnessReport.compile({ contract: second, evaluations: [evaluation(second, 0.9)], generatedAt: 3 }) + expect(a.comparisonKey).not.toBe(b.comparisonKey) + expect(() => HarnessReport.compare([a, b], "judge-one")).toThrow("only comparable") + }) + + test("refuses comparisons across different controlled intervention protocols", () => { + const evolution = HarnessContract.Evolution.parse({ + protocolVersion: "evolution-trace-v1", + validatorSHA256: "a".repeat(64), + manifestSchemaSHA256: "b".repeat(64), + lineAlgorithm: "sha256-exact-line-v1", + roots: ["src"], + extensions: [".ts"], + exclude: [], + maxFiles: 100, + maxFileBytes: 100_000, + maxTotalBytes: 1_000_000, + maxSourceLines: 10_000, + maxChangedLines: 1_000, + }) + const interventions = HarnessContract.Interventions.parse({ + protocolVersion: "intervention-study-v1", + validatorSHA256: "c".repeat(64), + requiredForPromotion: true, + minPairs: 3, + maxPairs: 3, + maxTotalPairs: 3, + confidence: 0.95, + required: ["replay"], + rules: [{ family: "replay", mode: "max_absolute_effect", threshold: 0.01 }], + }) + const first = HarnessContract.Info.parse({ ...contract("intervention-one"), evolution, interventions }) + const second = HarnessContract.Info.parse({ + ...contract("intervention-two"), + evolution, + interventions: { + ...interventions, + rules: [{ family: "replay", mode: "max_absolute_effect", threshold: 0.02 }], + }, + }) + const a = HarnessReport.compile({ contract: first, evaluations: [evaluation(first, 0.8)], generatedAt: 3 }) + const b = HarnessReport.compile({ contract: second, evaluations: [evaluation(second, 0.9)], generatedAt: 3 }) + expect(a.comparisonKey).not.toBe(b.comparisonKey) + expect(() => HarnessReport.compare([a, b], "intervention-one")).toThrow("only comparable") + }) + + test("refuses comparisons across different replicated evaluation protocols", () => { + const replication = HarnessContract.Replication.parse({ + protocolVersion: "replicated-evaluation-v1", + validatorSHA256: "a".repeat(64), + environmentSHA256: "e".repeat(64), + sampling: { + design: "crossed-stratified-cluster-v1", + stratumKind: "task", + clusterKind: "seed", + strata: [{ id: "task-0", commitmentSHA256: "b".repeat(64) }], + clusters: [0, 1, 2, 3, 4].map((seed) => ({ + id: `seed-${seed}`, + commitmentSHA256: seed.toString(16).repeat(64), + })), + }, + estimator: "iqm", + interval: { + method: "stratified-bootstrap-percentile-v1", + confidence: 0.95, + resamples: 1_000, + seed: 17, + }, + decision: { rule: "conservative-bound-v1", direction: "maximize", target: 0.9 }, + failurePolicy: "fail-closed", + }) + const first = HarnessContract.Info.parse({ ...contract("replication-one"), replication }) + const second = HarnessContract.Info.parse({ + ...contract("replication-two"), + replication: { + ...replication, + interval: { ...replication.interval, seed: 18 }, + }, + }) + const a = HarnessReport.compile({ contract: first, evaluations: [evaluation(first, 0.91)], generatedAt: 3 }) + const b = HarnessReport.compile({ contract: second, evaluations: [evaluation(second, 0.92)], generatedAt: 3 }) + expect(a.comparisonKey).not.toBe(b.comparisonKey) + expect(() => HarnessReport.compare([a, b], "replication-one")).toThrow("only comparable") + }) + + test("reports only final fidelity scores even when a later screening record is present", () => { + const staged = HarnessContract.Info.parse({ + ...contract("staged"), + benchmark: { + ...contract("staged").benchmark, + fidelities: [ + { id: "smoke", final: false }, + { id: "official", final: true }, + ], + }, + }) + const final = HarnessEvaluation.Info.parse({ + ...evaluation(staged, 0.85), + fidelity: { stage: "official", final: true }, + usage: { wallTimeMs: 100, costUSD: 0.2 }, + }) + const screen = HarnessEvaluation.Info.parse({ + ...evaluation(staged, 0.99), + fidelity: { stage: "smoke", final: false }, + usage: { wallTimeMs: 10, costUSD: 0.01 }, + }) + const report = HarnessReport.compile({ contract: staged, evaluations: [final, screen], generatedAt: 3 }) + expect(report.quality.score).toBe(0.85) + expect(report.efficiency).toMatchObject({ + costUSD: 0.21000000000000002, + evaluatorCostUSD: 0.21000000000000002, + wallTimeMs: 110, + evaluatorWallTimeMs: 110, + }) + }) + + test("validates adapter inputs before they become reports", () => { + expect(() => HarnessAdapter.Task.parse({ hiddenTestsAccessible: true })).toThrow() + }) +}) diff --git a/backend/cli/test/session/harness-search.test.ts b/backend/cli/test/session/harness-search.test.ts new file mode 100644 index 00000000..1c9ac1df --- /dev/null +++ b/backend/cli/test/session/harness-search.test.ts @@ -0,0 +1,1268 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessSearch } from "../../src/session/harness/search" + +const sessions = new Set() +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["contracts", "evaluations", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + sessions.clear() +}) + +function contract( + sessionID: string, + input?: { + profile?: HarnessContract.Profile + direction?: "maximize" | "minimize" + candidates?: number + wallTimeMs?: number + objectives?: HarnessContract.Objectives + adaptive?: boolean + }, +) { + sessions.add(sessionID) + return HarnessContract.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective: "Improve the held-out score under a fixed candidate budget", + benchmark: { + name: "search-test", + title: "Search evaluation", + family: "custom", + task: "Improve a held-out score under a fixed candidate budget", + version: "1", + taskID: "task-1", + split: "held_out", + evaluator: "official-evaluator", + metric: "score", + direction: input?.direction ?? "maximize", + objectives: input?.objectives, + }, + profile: input?.profile ?? "optimize", + search: input?.adaptive ? HarnessContract.adaptiveSearch : undefined, + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { + steps: 20, + ...(input?.candidates === undefined ? {} : { candidates: input.candidates }), + ...(input?.wallTimeMs === undefined ? {} : { wallTimeMs: input.wallTimeMs }), + }, + seed: 7, + intervention: "autonomous", + contamination: { policy: "hidden tests stay hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const artifact = (name: string) => ({ uri: `candidate://${name}`, sha256: hash(name) }) + +async function setup( + sessionID: string, + input?: { + candidates?: number + stall?: number + target?: number + direction?: "maximize" | "minimize" + objectives?: HarnessContract.Objectives + leased?: boolean + adaptive?: boolean + }, +) { + await contract(sessionID, { + direction: input?.direction, + objectives: input?.objectives, + adaptive: input?.adaptive, + }) + const state = await HarnessSearch.initialize({ + sessionID, + candidates: input?.candidates ?? 8, + stall: input?.stall, + target: input?.target, + }) + if (input?.leased || input?.adaptive) return state + const target = path.join(Global.Path.data, "harness", "search", `${encodeURIComponent(sessionID)}.json`) + const advisory = JSON.parse(await fs.readFile(target, "utf8")) + advisory.schemaVersion = 2 + delete advisory.proposalPolicy + await fs.writeFile(target, JSON.stringify(advisory)) + return HarnessSearch.read(sessionID) +} + +async function add( + sessionID: string, + name: string, + parentIDs: string[] = [], + branch = name, + inspirationIDs: string[] = [], +) { + return HarnessSearch.add({ + sessionID, + parentIDs, + inspirationIDs, + branch, + proposal: `proposal ${name}`, + artifact: artifact(name), + }) +} + +async function leased(sessionID: string, name: string, branch = name) { + const recommendation = HarnessSearch.recommend(await HarnessSearch.read(sessionID)) + const candidate = await HarnessSearch.add({ + sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch, + proposal: `proposal ${name}`, + artifact: artifact(name), + }) + return { recommendation, candidate } +} + +async function evaluate( + sessionID: string, + candidateID: string, + score: number | undefined, + status: HarnessEvaluation.Status = "passed", + metrics: Record = {}, +) { + await HarnessEvaluation.record({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + subject: { type: "candidate", id: candidateID }, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status, + ...(score === undefined ? {} : { score }), + metrics: score === undefined ? metrics : { score, ...metrics }, + checks: [{ id: "gate", status, blocking: true, evidence: [`candidate:${candidateID}`] }], + evidence: [`report:${candidateID}`], + evaluatedAt: Date.now(), + }) + return HarnessSearch.verify({ sessionID, candidateID }) +} + +describe("harness candidate graph", () => { + test("requires an explicit optimize benchmark contract", async () => { + sessions.add("search-missing") + await expect(HarnessSearch.initialize({ sessionID: "search-missing", candidates: 4 })).rejects.toThrow( + "No harness contract", + ) + await contract("search-react", { profile: "react" }) + await expect(HarnessSearch.initialize({ sessionID: "search-react", candidates: 4 })).rejects.toThrow( + "optimize profile", + ) + }) + + test("initializes idempotently but rejects budget drift", async () => { + await setup("search-init", { candidates: 4, stall: 2 }) + const first = await HarnessSearch.initialize({ sessionID: "search-init", candidates: 4, stall: 2 }) + expect(first.budget).toEqual({ candidates: 4, stall: 2 }) + await expect(HarnessSearch.initialize({ sessionID: "search-init", candidates: 5, stall: 2 })).rejects.toThrow( + "different contract or budget", + ) + }) + + test("uses contract budgets and rejects attempted expansion", async () => { + await contract("search-contract-budget", { candidates: 2, wallTimeMs: 1_000 }) + const state = await HarnessSearch.initialize({ sessionID: "search-contract-budget" }) + expect(state.budget).toMatchObject({ candidates: 2, wallTimeMs: 1_000 }) + await expect(HarnessSearch.initialize({ sessionID: "search-contract-budget", candidates: 3 })).rejects.toThrow( + "cannot exceed", + ) + }) + + test("leases adaptive generation modes and bounded verified trajectory context", async () => { + const initial = await setup("search-leased", { candidates: 8, leased: true }) + const seed = HarnessSearch.recommend(initial) + expect(initial.proposalPolicy).toBe("leased-v3") + expect(seed).toMatchObject({ + revision: 0, + strategy: "seed", + mode: "single-pass", + parentIDs: [], + inspirationIDs: [], + targetIsland: 0, + contextIDs: [], + }) + await expect( + HarnessSearch.add({ + sessionID: "search-leased", + parentIDs: [], + branch: "missing-lease", + proposal: "bypass the server recommendation", + artifact: artifact("missing-lease"), + }), + ).rejects.toThrow("recommendation_id or reservation_id is required") + + const first = await HarnessSearch.add({ + sessionID: "search-leased", + recommendationID: seed.id, + parentIDs: seed.parentIDs, + inspirationIDs: seed.inspirationIDs, + branch: "baseline", + proposal: "establish a direct baseline", + artifact: artifact("leased-seed"), + }) + expect(first.state.candidates[first.id]?.lease).toEqual({ + id: seed.id, + revision: seed.revision, + strategy: seed.strategy, + mode: seed.mode, + targetIsland: seed.targetIsland, + contextIDs: seed.contextIDs, + }) + await evaluate("search-leased", first.id, 0.9) + const explore = HarnessSearch.recommend(await HarnessSearch.read("search-leased")) + expect(explore).toMatchObject({ strategy: "explore", mode: "stepwise", parentIDs: [], contextIDs: [] }) + + await expect( + HarnessSearch.add({ + sessionID: "search-leased", + recommendationID: seed.id, + parentIDs: [], + branch: "stale", + proposal: "race an obsolete state revision", + artifact: artifact("stale-lease"), + }), + ).rejects.toThrow("stale") + await expect( + HarnessSearch.add({ + sessionID: "search-leased", + recommendationID: explore.id, + parentIDs: [first.id], + branch: "off-policy", + proposal: "replace the leased independent root with a local edit", + artifact: artifact("off-policy"), + }), + ).rejects.toThrow("does not match") + + const alternate = await HarnessSearch.add({ + sessionID: "search-leased", + recommendationID: explore.id, + parentIDs: explore.parentIDs, + inspirationIDs: explore.inspirationIDs, + branch: "alternate", + proposal: "plan and implement an independent approach", + artifact: artifact("leased-alternate"), + }) + await evaluate("search-leased", alternate.id, 0.8) + const migrate = HarnessSearch.recommend(await HarnessSearch.read("search-leased")) + expect(migrate).toMatchObject({ + strategy: "migrate", + mode: "stepwise", + parentIDs: [alternate.id], + inspirationIDs: [first.id], + contextIDs: [alternate.id, first.id], + }) + const moved = await HarnessSearch.add({ + sessionID: "search-leased", + recommendationID: migrate.id, + parentIDs: migrate.parentIDs, + inspirationIDs: migrate.inspirationIDs, + branch: "alternate", + proposal: "transfer the verified source insight into the target lineage", + artifact: artifact("leased-migration"), + }) + const next = HarnessSearch.recommend(moved.state) + expect(next.contextIDs).not.toContain(moved.id) + expect(next.contextIDs.every((id) => moved.state.candidates[id]?.result?.source === "verified")).toBe(true) + }) + + test("routes adaptive compute from verified improvements and ignores agent observations", async () => { + const initial = await setup("search-adaptive", { candidates: 12, adaptive: true }) + expect(initial).toMatchObject({ + schemaVersion: 4, + proposalPolicy: "adaptive-v4", + controller: HarnessContract.adaptiveSearch, + population: { count: 2, initial: 2 }, + }) + const first = await leased("search-adaptive", "adaptive-first", "baseline") + expect(first.recommendation).toMatchObject({ + strategy: "seed", + targetIsland: 0, + control: { eventCount: 0, targetIsland: 0, visits: 0 }, + }) + expect(first.candidate.state.candidates[first.candidate.id]?.createdRevision).toBe(1) + expect(first.candidate.state.candidates[first.candidate.id]?.lease?.control).toMatchObject({ + policySHA256: first.recommendation.control?.policySHA256, + eventCount: 0, + targetIsland: 0, + }) + await evaluate("search-adaptive", first.candidate.id, 10) + + const second = await leased("search-adaptive", "adaptive-second", "alternate") + expect(second.recommendation).toMatchObject({ + strategy: "explore", + parentIDs: [], + targetIsland: 1, + control: { eventCount: 1, targetIsland: 1, visits: 0 }, + }) + await evaluate("search-adaptive", second.candidate.id, 9) + + const third = await leased("search-adaptive", "adaptive-third", "baseline") + expect(third.recommendation).toMatchObject({ + targetIsland: 0, + control: { eventCount: 2, selectedIsland: 0, targetIsland: 0, visits: 1 }, + }) + const before = HarnessSearch.adaptation(third.candidate.state) + const observed = await HarnessSearch.observe({ + sessionID: "search-adaptive", + candidateID: third.candidate.id, + status: "passed", + score: 999, + metrics: { score: 999 }, + feedback: "untrusted self-evaluation", + }) + expect(HarnessSearch.adaptation(observed)).toEqual(before) + await evaluate("search-adaptive", third.candidate.id, 12) + const next = HarnessSearch.recommend(await HarnessSearch.read("search-adaptive")) + expect(next.control).toMatchObject({ eventCount: 3, selectedIsland: 1, targetIsland: 1, visits: 1 }) + expect(HarnessSearch.adaptation(await HarnessSearch.read("search-adaptive")).islands[0]).toMatchObject({ + visits: 2, + improvements: 1, + accumulatedImprovement: 0.004, + }) + }) + + test("reuses the authenticated adaptive island after a failed root releases capacity", async () => { + await setup("search-adaptive-root-retry", { candidates: 12, adaptive: true }) + const failed = await leased("search-adaptive-root-retry", "adaptive-failed-root", "failed-root") + expect(failed.recommendation.targetIsland).toBe(0) + await evaluate("search-adaptive-root-retry", failed.candidate.id, undefined, "failed") + + const retry = await leased("search-adaptive-root-retry", "adaptive-retry-root", "retry-root") + expect(retry.recommendation).toMatchObject({ strategy: "seed", targetIsland: 0 }) + expect(retry.candidate.state.candidates[retry.candidate.id]).toMatchObject({ island: 0 }) + }) + + test("rejects a fully rehashed adaptive lease whose controller semantics were substituted", async () => { + await setup("search-adaptive-tamper", { candidates: 6, adaptive: true }) + const added = await leased("search-adaptive-tamper", "adaptive-tamper", "baseline") + const file = path.join(Global.Path.data, "harness", "search", "search-adaptive-tamper.json") + const state = JSON.parse(await fs.readFile(file, "utf8")) + const candidate = state.candidates[added.candidate.id] + candidate.lease.control.intensity += 0.1 + candidate.lease.id = hash( + JSON.stringify({ + runID: state.runID, + sessionID: state.sessionID, + revision: candidate.lease.revision, + strategy: candidate.lease.strategy, + mode: candidate.lease.mode, + parentIDs: candidate.parentIDs.toSorted(), + inspirationIDs: candidate.inspirationIDs.toSorted(), + targetIsland: candidate.lease.targetIsland, + contextIDs: candidate.lease.contextIDs, + control: candidate.lease.control, + }), + ) + candidate.id = hash( + JSON.stringify({ + parentIDs: candidate.parentIDs.toSorted(), + inspirationIDs: candidate.inspirationIDs.toSorted(), + branch: candidate.branch, + proposal: candidate.proposal, + artifact: candidate.artifact, + lease: candidate.lease, + }), + ) + delete state.candidates[added.candidate.id] + state.candidates[candidate.id] = candidate + await fs.writeFile(file, JSON.stringify(state)) + await expect(HarnessSearch.read("search-adaptive-tamper")).rejects.toThrow( + "controller does not match verified candidate history", + ) + }) + + test("spawns new islands and escalates to meta-guidance only after verified global stagnation", async () => { + await setup("search-adaptive-stagnation", { candidates: 32, adaptive: true }) + const first = await leased("search-adaptive-stagnation", "stagnation-root-0", "root-0") + await evaluate("search-adaptive-stagnation", first.candidate.id, 1) + const second = await leased("search-adaptive-stagnation", "stagnation-root-1", "root-1") + await evaluate("search-adaptive-stagnation", second.candidate.id, 1) + + const failed: string[] = [] + for (const index of Array.from({ length: 8 }, (_, index) => index)) { + const recommendation = HarnessSearch.recommend(await HarnessSearch.read("search-adaptive-stagnation")) + if (!recommendation.parentIDs.length && recommendation.targetIsland === 2) break + const candidate = await HarnessSearch.add({ + sessionID: "search-adaptive-stagnation", + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: `failed-${index}`, + proposal: `verified failed attempt ${index}`, + artifact: artifact(`stagnation-failed-${index}`), + }) + await evaluate("search-adaptive-stagnation", candidate.id, undefined, "failed") + failed.push(candidate.id) + } + expect(failed.length).toBeGreaterThanOrEqual(HarnessContract.adaptiveSearch.stagnation.patience - 1) + const spawn = HarnessSearch.recommend(await HarnessSearch.read("search-adaptive-stagnation")) + expect(spawn).toMatchObject({ + strategy: "explore", + parentIDs: [], + targetIsland: 2, + control: { globalStagnation: true }, + }) + expect(spawn.reasons).toContain("adaptive-island-spawn:2") + const third = await leased("search-adaptive-stagnation", "stagnation-root-2", "root-2") + await evaluate("search-adaptive-stagnation", third.candidate.id, 0.5) + const fourth = await leased("search-adaptive-stagnation", "stagnation-root-3", "root-3") + expect(fourth.recommendation).toMatchObject({ parentIDs: [], targetIsland: 3 }) + await evaluate("search-adaptive-stagnation", fourth.candidate.id, 0.4) + const meta = HarnessSearch.recommend(await HarnessSearch.read("search-adaptive-stagnation")) + expect(meta).toMatchObject({ + strategy: "diverge", + mode: "stepwise", + parentIDs: [first.candidate.id], + control: { globalStagnation: true }, + }) + expect(meta.reasons).toContain("meta-guidance") + }) + + test("requests focused diffs for exploitation and serializes recommendation races", async () => { + const initial = await setup("search-lease-race", { candidates: 2, leased: true }) + const seed = HarnessSearch.recommend(initial) + const attempts = await Promise.allSettled( + ["a", "b"].map((name) => + HarnessSearch.add({ + sessionID: "search-lease-race", + recommendationID: seed.id, + parentIDs: seed.parentIDs, + inspirationIDs: seed.inspirationIDs, + branch: name, + proposal: `concurrent proposal ${name}`, + artifact: artifact(`lease-race-${name}`), + }), + ), + ) + expect(attempts.filter((item) => item.status === "fulfilled")).toHaveLength(1) + expect(attempts.filter((item) => item.status === "rejected")).toHaveLength(1) + const state = await HarnessSearch.read("search-lease-race") + const first = Object.values(state.candidates)[0]! + await evaluate("search-lease-race", first.id, 0.5) + const exploit = HarnessSearch.recommend(await HarnessSearch.read("search-lease-race")) + expect(exploit).toMatchObject({ strategy: "exploit", mode: "diff", parentIDs: [first.id] }) + expect(exploit.contextIDs).toEqual([first.id]) + }) + + test("atomically reserves and consumes parallel sibling variations out of order", async () => { + await setup("search-reservations", { candidates: 8, leased: true }) + const batch = await HarnessSearch.reserve({ sessionID: "search-reservations", count: 8 }) + expect(batch.reservations).toHaveLength(3) + expect(new Set(batch.reservations.map((item) => item.id)).size).toBe(3) + expect(new Set(batch.reservations.map((item) => item.lease.id)).size).toBe(2) + expect(new Set(batch.reservations.map((item) => item.mandate?.id)).size).toBe(3) + expect(batch.reservations.map((item) => item.mandate?.operator)).toEqual([ + "architectural-change", + "composition", + "efficiency", + ]) + expect(batch.reservations.map((item) => item.lease.targetIsland)).toEqual([0, 1, 0]) + expect(batch.state.bestID).toBeUndefined() + expect(batch.state.archiveIDs).toEqual([]) + expect(await HarnessSearch.read("search-reservations")).toEqual(batch.state) + const blocked = HarnessSearch.recommend(batch.state) + await expect( + HarnessSearch.add({ + sessionID: "search-reservations", + recommendationID: blocked.id, + parentIDs: blocked.parentIDs, + inspirationIDs: blocked.inspirationIDs, + branch: "unreserved-root", + proposal: "exceed the independent-root capacity held by reservations", + artifact: artifact("unreserved-root"), + }), + ).rejects.toThrow("root budget") + + const order = [batch.reservations[2]!, batch.reservations[0]!, batch.reservations[1]!] + const results = await Promise.all( + order.map((reservation, index) => + HarnessSearch.add({ + sessionID: "search-reservations", + reservationID: reservation.id, + parentIDs: reservation.parentIDs, + inspirationIDs: reservation.inspirationIDs, + branch: `parallel-${index}`, + proposal: `independent reserved sibling ${index}`, + artifact: artifact(`reserved-${index}`), + }), + ), + ) + expect(results.every((item) => item.accepted)).toBe(true) + const state = await HarnessSearch.read("search-reservations") + expect(Object.keys(state.candidates)).toHaveLength(3) + expect(Object.values(state.reservations).every((item) => item.status === "consumed")).toBe(true) + expect(state.bestID).toBeUndefined() + expect(state.archiveIDs).toEqual([]) + for (const result of results) { + expect(state.candidates[result.id]?.reservationID).toBeDefined() + expect(state.candidates[result.id]?.result).toBeUndefined() + } + + const retry = await HarnessSearch.add({ + sessionID: "search-reservations", + reservationID: order[0]!.id, + parentIDs: order[0]!.parentIDs, + inspirationIDs: order[0]!.inspirationIDs, + branch: "parallel-0", + proposal: "idempotent retry uses different wrapper text", + artifact: artifact("reserved-0"), + }) + expect(retry).toMatchObject({ accepted: true, deduplicated: true, id: results[0]!.id }) + await expect( + HarnessSearch.add({ + sessionID: "search-reservations", + reservationID: order[0]!.id, + parentIDs: order[0]!.parentIDs, + inspirationIDs: order[0]!.inspirationIDs, + branch: "parallel-reuse", + proposal: "consume one ticket twice", + artifact: artifact("reserved-reuse"), + }), + ).rejects.toThrow("no longer open") + }) + + test("diversifies parallel variation mandates and verified lineages before route reuse", async () => { + await setup("search-portfolio", { candidates: 16, leased: true }) + const roots: string[] = [] + for (const [index, name] of ["alpha", "beta", "gamma", "delta"].entries()) { + const state = await HarnessSearch.read("search-portfolio") + const recommendation = HarnessSearch.recommend(state) + const candidate = await HarnessSearch.add({ + sessionID: "search-portfolio", + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: name, + proposal: `independent ${name} route`, + artifact: artifact(`portfolio-${name}`), + }) + await evaluate("search-portfolio", candidate.id, 1 - index / 10) + roots.push(candidate.id) + } + const migration = HarnessSearch.recommend(await HarnessSearch.read("search-portfolio")) + expect(migration.strategy).toBe("migrate") + const migrated = await HarnessSearch.add({ + sessionID: "search-portfolio", + recommendationID: migration.id, + parentIDs: migration.parentIDs, + inspirationIDs: migration.inspirationIDs, + branch: "migration", + proposal: "validate the scheduled cross-island transfer", + artifact: artifact("portfolio-migration"), + }) + await evaluate("search-portfolio", migrated.id, 0.65) + + const before = await HarnessSearch.read("search-portfolio") + const serial = HarnessSearch.recommend(before) + expect(HarnessSearch.recommend(before)).toEqual(serial) + expect(serial.parentIDs).toHaveLength(1) + const batch = await HarnessSearch.reserve({ sessionID: "search-portfolio", count: 4 }) + expect(batch.reservations).toHaveLength(4) + expect(new Set(batch.reservations.map((item) => item.mandate?.id)).size).toBe(4) + expect(new Set(batch.reservations.map((item) => item.mandate?.operator)).size).toBe(4) + expect(new Set(batch.reservations.map((item) => item.parentIDs[0])).size).toBe(4) + expect(new Set(batch.reservations.map((item) => item.lease.id)).size).toBe(4) + expect(batch.state.bestID).toBe(before.bestID) + expect(batch.state.archiveIDs).toEqual(before.archiveIDs) + expect( + batch.reservations.every((item) => item.parentIDs.every((id) => roots.includes(id) || id === migrated.id)), + ).toBe(true) + + const ticket = batch.reservations[2]! + const accepted = await HarnessSearch.add({ + sessionID: "search-portfolio", + reservationID: ticket.id, + parentIDs: ticket.parentIDs, + inspirationIDs: ticket.inspirationIDs, + branch: "portfolio-child", + proposal: "execute the assigned variation mandate agentically", + artifact: artifact("portfolio-child"), + }) + expect(accepted.state.candidates[accepted.id]?.reservationID).toBe(ticket.id) + expect(accepted.state.reservations[ticket.id]?.mandate).toEqual(ticket.mandate) + expect(accepted.state.bestID).toBe(before.bestID) + expect(accepted.state.archiveIDs).toEqual(before.archiveIDs) + }) + + test("diversifies fusion complements and migration targets under one centralized portfolio", async () => { + await setup("search-fusion-portfolio", { candidates: 16, stall: 2, leased: true }) + const roots = await HarnessSearch.reserve({ sessionID: "search-fusion-portfolio", count: 3 }) + const accepted = await Promise.all( + roots.reservations.map((ticket, index) => + HarnessSearch.add({ + sessionID: "search-fusion-portfolio", + reservationID: ticket.id, + parentIDs: ticket.parentIDs, + inspirationIDs: ticket.inspirationIDs, + branch: `fusion-root-${index}`, + proposal: `independent fusion source ${index}`, + artifact: artifact(`fusion-portfolio-${index}`), + }), + ), + ) + for (const [index, candidate] of accepted.entries()) { + await evaluate("search-fusion-portfolio", candidate.id, 1 - index / 10) + } + const fusion = HarnessSearch.recommend(await HarnessSearch.read("search-fusion-portfolio")) + expect(fusion.strategy).toBe("fuse") + const fused = await HarnessSearch.reserve({ sessionID: "search-fusion-portfolio", count: 2 }) + expect(fused.reservations).toHaveLength(2) + expect(fused.reservations.every((ticket) => ticket.lease.strategy === "fuse")).toBe(true) + expect(new Set(fused.reservations.flatMap((ticket) => ticket.parentIDs)).size).toBe(3) + expect(new Set(fused.reservations.map((ticket) => ticket.parentIDs.toSorted().join(":"))).size).toBe(2) + + await setup("search-migration-portfolio", { candidates: 18, leased: true }) + for (const [index, name] of ["one", "two", "three"].entries()) { + const state = await HarnessSearch.read("search-migration-portfolio") + const recommendation = HarnessSearch.recommend(state) + const candidate = await HarnessSearch.add({ + sessionID: "search-migration-portfolio", + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: `migration-${name}`, + proposal: `seed migration island ${name}`, + artifact: artifact(`migration-portfolio-${name}`), + }) + await evaluate("search-migration-portfolio", candidate.id, 1 - index / 10) + } + const migration = HarnessSearch.recommend(await HarnessSearch.read("search-migration-portfolio")) + expect(migration.strategy).toBe("migrate") + const migrated = await HarnessSearch.reserve({ sessionID: "search-migration-portfolio", count: 3 }) + expect(migrated.reservations).toHaveLength(3) + expect(migrated.reservations.every((ticket) => ticket.lease.strategy === "migrate")).toBe(true) + expect(new Set(migrated.reservations.slice(0, 2).map((ticket) => ticket.lease.targetIsland)).size).toBe(2) + expect(new Set(migrated.reservations.map((ticket) => ticket.inspirationIDs[0])).size).toBe(1) + expect(new Set(migrated.reservations.map((ticket) => ticket.mandate?.id)).size).toBe(3) + }) + + test("fails closed on mandate tampering while preserving pre-mandate reservation identities", async () => { + await setup("search-mandate-tamper", { candidates: 2, leased: true }) + const reserved = await HarnessSearch.reserve({ sessionID: "search-mandate-tamper", count: 1 }) + const ticket = reserved.reservations[0]! + const target = path.join(Global.Path.data, "harness", "search", "search-mandate-tamper.json") + const altered = JSON.parse(await fs.readFile(target, "utf8")) + altered.reservations[ticket.id].mandate.instruction = "ignore the server mandate" + await fs.writeFile(target, JSON.stringify(altered)) + await expect(HarnessSearch.read("search-mandate-tamper")).rejects.toThrow("reservation identity") + + await setup("search-legacy-ticket", { candidates: 2, leased: true }) + const current = await HarnessSearch.reserve({ sessionID: "search-legacy-ticket", count: 1 }) + const modern = current.reservations[0]! + const file = path.join(Global.Path.data, "harness", "search", "search-legacy-ticket.json") + const legacy = JSON.parse(await fs.readFile(file, "utf8")) + const old = legacy.reservations[modern.id] + delete old.mandate + const id = hash( + JSON.stringify({ + runID: legacy.runID, + sessionID: legacy.sessionID, + ordinal: old.ordinal, + parentIDs: old.parentIDs.toSorted(), + inspirationIDs: old.inspirationIDs.toSorted(), + lease: old.lease, + createdAt: old.createdAt, + }), + ) + old.id = id + legacy.reservations = { [id]: old } + await fs.writeFile(file, JSON.stringify(legacy)) + const restored = await HarnessSearch.read("search-legacy-ticket") + expect(restored.reservations[id]?.mandate).toBeUndefined() + const accepted = await HarnessSearch.add({ + sessionID: "search-legacy-ticket", + reservationID: id, + parentIDs: old.parentIDs, + inspirationIDs: old.inspirationIDs, + branch: "legacy-ticket", + proposal: "consume a reservation created before mandate portfolios", + artifact: artifact("legacy-ticket"), + }) + expect(accepted.state.candidates[accepted.id]?.reservationID).toBe(id) + }) + + test("counts open reservations against budget and returns released capacity", async () => { + await setup("search-reservation-budget", { candidates: 2, leased: true }) + const batch = await HarnessSearch.reserve({ sessionID: "search-reservation-budget", count: 8 }) + expect(batch.reservations).toHaveLength(2) + const blocked = HarnessSearch.recommend(batch.state) + const rejected = await HarnessSearch.add({ + sessionID: "search-reservation-budget", + recommendationID: blocked.id, + parentIDs: blocked.parentIDs, + inspirationIDs: blocked.inspirationIDs, + branch: "unreserved", + proposal: "steal capacity held by parallel workers", + artifact: artifact("unreserved"), + }) + expect(rejected.accepted).toBe(false) + + const released = await HarnessSearch.release({ + sessionID: "search-reservation-budget", + reservationID: batch.reservations[0]!.id, + }) + expect(released.reservations[batch.reservations[0]!.id]?.status).toBe("released") + const serial = HarnessSearch.recommend(released) + const first = await HarnessSearch.add({ + sessionID: "search-reservation-budget", + recommendationID: serial.id, + parentIDs: serial.parentIDs, + inspirationIDs: serial.inspirationIDs, + branch: "serial", + proposal: "use the returned serial slot", + artifact: artifact("reservation-serial"), + }) + expect(first.accepted).toBe(true) + const ticket = batch.reservations[1]! + const second = await HarnessSearch.add({ + sessionID: "search-reservation-budget", + reservationID: ticket.id, + parentIDs: ticket.parentIDs, + inspirationIDs: ticket.inspirationIDs, + branch: "parallel", + proposal: "consume the remaining parallel slot", + artifact: artifact("reservation-parallel"), + }) + expect(second.accepted).toBe(true) + expect(Object.keys(second.state.candidates)).toHaveLength(2) + await expect( + HarnessSearch.release({ sessionID: "search-reservation-budget", reservationID: ticket.id }), + ).rejects.toThrow("consumed") + }) + + test("serializes concurrent attempts to consume one reservation", async () => { + await setup("search-reservation-race", { candidates: 2, leased: true }) + const batch = await HarnessSearch.reserve({ sessionID: "search-reservation-race", count: 1 }) + const ticket = batch.reservations[0]! + const attempts = await Promise.allSettled( + ["a", "b"].map((name) => + HarnessSearch.add({ + sessionID: "search-reservation-race", + reservationID: ticket.id, + parentIDs: ticket.parentIDs, + inspirationIDs: ticket.inspirationIDs, + branch: name, + proposal: `race ${name}`, + artifact: artifact(`reservation-race-${name}`), + }), + ), + ) + expect(attempts.filter((item) => item.status === "fulfilled")).toHaveLength(1) + expect(attempts.filter((item) => item.status === "rejected")).toHaveLength(1) + const state = await HarnessSearch.read("search-reservation-race") + expect(Object.keys(state.candidates)).toHaveLength(1) + expect(state.reservations[ticket.id]?.status).toBe("consumed") + }) + + test("releases a reservation when a worker rediscovers existing bytes", async () => { + const initial = await setup("search-reservation-duplicate", { candidates: 3, leased: true }) + const recommendation = HarnessSearch.recommend(initial) + const seed = await HarnessSearch.add({ + sessionID: "search-reservation-duplicate", + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "seed", + proposal: "seed", + artifact: artifact("reservation-duplicate"), + }) + const batch = await HarnessSearch.reserve({ sessionID: "search-reservation-duplicate", count: 1 }) + const ticket = batch.reservations[0]! + const duplicate = await HarnessSearch.add({ + sessionID: "search-reservation-duplicate", + reservationID: ticket.id, + parentIDs: ticket.parentIDs, + inspirationIDs: ticket.inspirationIDs, + branch: "rediscovery", + proposal: "same bytes found independently", + artifact: artifact("reservation-duplicate"), + }) + expect(duplicate).toMatchObject({ accepted: true, deduplicated: true, id: seed.id }) + expect(duplicate.state.reservations[ticket.id]).toMatchObject({ status: "released", candidateID: seed.id }) + const replacement = await HarnessSearch.reserve({ sessionID: "search-reservation-duplicate", count: 1 }) + expect(replacement.reservations).toHaveLength(1) + }) + + test("releases unused reservations on termination and fails closed on ticket tampering", async () => { + await setup("search-reservation-stop", { candidates: 2, leased: true }) + await HarnessSearch.reserve({ sessionID: "search-reservation-stop", count: 2 }) + const stopped = await HarnessSearch.finish("search-reservation-stop", "user_cancelled") + expect(Object.values(stopped.reservations)).toHaveLength(2) + expect(Object.values(stopped.reservations).every((item) => item.status === "released")).toBe(true) + + await setup("search-reservation-tamper", { candidates: 2, leased: true }) + const reserved = await HarnessSearch.reserve({ sessionID: "search-reservation-tamper", count: 1 }) + const ticket = reserved.reservations[0]! + const file = path.join(Global.Path.data, "harness", "search", "search-reservation-tamper.json") + const state = JSON.parse(await fs.readFile(file, "utf8")) + state.reservations[ticket.id].status = "consumed" + await fs.writeFile(file, JSON.stringify(state)) + await expect(HarnessSearch.read("search-reservation-tamper")).rejects.toThrow("authorized candidate") + }) + + test("fails closed when leased recommendation provenance is edited", async () => { + const initial = await setup("search-lease-tamper", { leased: true }) + const recommendation = HarnessSearch.recommend(initial) + const seed = await HarnessSearch.add({ + sessionID: "search-lease-tamper", + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "baseline", + proposal: "baseline", + artifact: artifact("lease-tamper"), + }) + const file = path.join(Global.Path.data, "harness", "search", "search-lease-tamper.json") + const state = JSON.parse(await fs.readFile(file, "utf8")) + state.candidates[seed.id].lease.mode = "diff" + await fs.writeFile(file, JSON.stringify(state)) + await expect(HarnessSearch.read("search-lease-tamper")).rejects.toThrow("identity does not match") + }) + + test("caps deep trajectory context without admitting unverified state", async () => { + await setup("search-context-cap", { candidates: 20 }) + const seed = await add("search-context-cap", "seed", [], "line") + await evaluate("search-context-cap", seed.id, 0.1) + const nodes = [seed] + for (const index of Array.from({ length: 7 }, (_, index) => index)) { + const parent = nodes.at(-1)! + const child = await add("search-context-cap", `child-${index}`, [parent.id], "line") + await evaluate("search-context-cap", child.id, 0.2 + index / 10) + nodes.push(child) + } + const state = await HarnessSearch.read("search-context-cap") + const recommendation = HarnessSearch.recommend(state) + expect(recommendation.mode).toBe("diff") + expect(recommendation.contextIDs).toHaveLength(6) + expect(recommendation.contextIDs[0]).toBe(nodes.at(-1)!.id) + expect(recommendation.contextIDs.every((id) => state.candidates[id]?.result?.source === "verified")).toBe(true) + }) + + test("content-addresses candidates and deduplicates without spending budget", async () => { + await setup("search-dedupe", { candidates: 2 }) + const first = await add("search-dedupe", "seed") + const duplicate = await add("search-dedupe", "seed") + expect(first.id).toHaveLength(64) + expect(duplicate).toMatchObject({ accepted: true, deduplicated: true, id: first.id }) + expect(Object.keys(duplicate.state.candidates)).toHaveLength(1) + + const revision = duplicate.state.revision + const wrapped = await HarnessSearch.add({ + sessionID: "search-dedupe", + parentIDs: [hash("unknown-wrapper-parent")], + inspirationIDs: [hash("unknown-wrapper-inspiration")], + branch: "different-wrapper", + proposal: "Claim the same bytes are a new discovery", + artifact: { uri: "candidate://mirror", sha256: artifact("seed").sha256 }, + }) + expect(wrapped).toMatchObject({ accepted: true, deduplicated: true, id: first.id }) + expect(wrapped.state.revision).toBe(revision) + expect(Object.keys(wrapped.state.candidates)).toHaveLength(1) + }) + + test("serializes concurrent content duplicates into one budget slot", async () => { + await setup("search-dedupe-race", { candidates: 2 }) + const sha256 = hash("shared-bytes") + const results = await Promise.all([ + HarnessSearch.add({ + sessionID: "search-dedupe-race", + parentIDs: [], + branch: "first", + proposal: "first wrapper", + artifact: { uri: "candidate://first", sha256 }, + }), + HarnessSearch.add({ + sessionID: "search-dedupe-race", + parentIDs: [], + branch: "second", + proposal: "second wrapper", + artifact: { uri: "candidate://second", sha256 }, + }), + ]) + expect(results.every((result) => result.accepted)).toBe(true) + expect(new Set(results.map((result) => result.id)).size).toBe(1) + expect(results.filter((result) => result.deduplicated)).toHaveLength(1) + expect(Object.keys((await HarnessSearch.read("search-dedupe-race")).candidates)).toHaveLength(1) + }) + + test("assigns deterministic islands server-side and preserves them across restart", async () => { + const initial = await setup("search-islands", { candidates: 8 }) + expect(initial.population).toEqual({ + mode: "islands", + count: 2, + initial: 2, + topology: "ring", + migrationInterval: 2, + }) + + const first = await add("search-islands", "first", [], "line-a") + expect(first.state.candidates[first.id]).toMatchObject({ island: 0, ordinal: 0 }) + await evaluate("search-islands", first.id, 0.9) + const second = await add("search-islands", "second", [], "line-b") + expect(second.state.candidates[second.id]).toMatchObject({ island: 1, ordinal: 1 }) + await evaluate("search-islands", second.id, 0.8) + const child = await add("search-islands", "child", [first.id], "line-a") + expect(child.state.candidates[child.id]).toMatchObject({ island: 0, ordinal: 2, parentIDs: [first.id] }) + expect(await HarnessSearch.read("search-islands")).toEqual(child.state) + }) + + test("migrates verified inspiration into a target island without copying candidate bytes", async () => { + await setup("search-migrate", { candidates: 8, stall: 5 }) + const source = await add("search-migrate", "source", [], "source") + await evaluate("search-migrate", source.id, 0.9) + const target = await add("search-migrate", "target", [], "target") + const state = await evaluate("search-migrate", target.id, 0.8) + expect(HarnessSearch.recommend(state)).toMatchObject({ + strategy: "migrate", + mode: "stepwise", + parentIDs: [target.id], + inspirationIDs: [source.id], + targetIsland: 1, + contextIDs: [target.id, source.id], + reasons: ["candidates:2", "ring:0->1", "verified-inspiration", "new-artifact-required"], + }) + + const migrated = await add("search-migrate", "migrated", [target.id], "target", [source.id]) + expect(migrated.state.candidates[migrated.id]).toMatchObject({ + island: 1, + parentIDs: [target.id], + inspirationIDs: [source.id], + }) + await expect(add("search-migrate", "invalid", [target.id], "target", [migrated.id])).rejects.toThrow( + "externally verified passing inspirations", + ) + await expect(add("search-migrate", "overlap", [source.id], "target", [source.id])).rejects.toThrow( + "distinct from parents", + ) + + const revision = migrated.state.revision + const copied = await HarnessSearch.add({ + sessionID: "search-migrate", + parentIDs: [target.id], + inspirationIDs: [source.id], + branch: "target", + proposal: "Copy the source elite without modifying it", + artifact: { uri: "candidate://copied-source", sha256: artifact("source").sha256 }, + }) + expect(copied).toMatchObject({ accepted: true, deduplicated: true, id: source.id }) + expect(copied.state.revision).toBe(revision) + expect(HarnessSearch.recommend(copied.state).strategy).not.toBe("migrate") + }) + + test("allows bounded independent roots and releases failed root capacity", async () => { + await setup("search-seed", { candidates: 4 }) + const first = await add("search-seed", "seed", [], "branch-a") + await evaluate("search-seed", first.id, 0, "failed") + await add("search-seed", "replacement", [], "branch-a") + await add("search-seed", "independent", [], "branch-b") + await expect(add("search-seed", "overflow", [], "branch-c")).rejects.toThrow("root budget") + }) + + test("rejects unknown, duplicate, and unverified parents", async () => { + await setup("search-parents") + const seed = await add("search-parents", "seed") + await expect(add("search-parents", "unknown", [hash("unknown")])).rejects.toThrow("must exist") + await expect(add("search-parents", "duplicate", [seed.id, seed.id])).rejects.toThrow("must be unique") + await expect(add("search-parents", "unverified", [seed.id])).rejects.toThrow("externally verified") + }) + + test("keeps self-reported observations out of elite state and lineage", async () => { + await setup("search-observed") + const seed = await add("search-observed", "seed") + const state = await HarnessSearch.observe({ + sessionID: "search-observed", + candidateID: seed.id, + status: "passed", + score: 999, + feedback: "agent says this is excellent", + }) + expect(state.bestID).toBeUndefined() + expect(HarnessSearch.recommend(state)).toMatchObject({ strategy: "seed", parentIDs: [] }) + await expect(add("search-observed", "child", [seed.id])).rejects.toThrow("externally verified") + }) + + test("requires the external evaluation to name the exact candidate", async () => { + await setup("search-subject") + const seed = await add("search-subject", "seed") + await HarnessEvaluation.record({ + schemaVersion: 1, + runID: "run-search-subject", + sessionID: "search-subject", + subject: { type: "run", id: "run-search-subject" }, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status: "passed", + score: 1, + metrics: { score: 1 }, + checks: [{ id: "gate", status: "passed", blocking: true, evidence: ["run"] }], + evidence: ["run-report"], + evaluatedAt: Date.now(), + }) + await expect(HarnessSearch.verify({ sessionID: "search-subject", candidateID: seed.id })).rejects.toThrow( + "not bound to candidate", + ) + }) + + test("promotes only passing externally evaluated candidates", async () => { + await setup("search-verified") + const seed = await add("search-verified", "seed") + const failed = await evaluate("search-verified", seed.id, 100, "failed") + expect(failed.bestID).toBeUndefined() + expect(failed.candidates[seed.id]?.result?.source).toBe("verified") + await expect(add("search-verified", "child", [seed.id])).rejects.toThrow("externally verified passing") + }) + + test("ranks verified scores in the declared direction", async () => { + await setup("search-rank") + const seed = await add("search-rank", "seed", [], "baseline") + await evaluate("search-rank", seed.id, 0.5) + const child = await add("search-rank", "child", [seed.id], "improved") + const state = await evaluate("search-rank", child.id, 0.8) + expect(state.bestID).toBe(child.id) + expect(state.stalled).toBe(0) + }) + + test("preserves evaluator-declared Pareto alternatives without changing the primary winner", async () => { + await setup("search-pareto", { + candidates: 8, + stall: 1, + objectives: [ + { metric: "robustness", direction: "maximize" }, + { metric: "latency", direction: "minimize" }, + ], + }) + const seed = await add("search-pareto", "seed", [], "accurate") + await evaluate("search-pareto", seed.id, 0.9, "passed", { robustness: 0.2, latency: 20 }) + const alternate = await add("search-pareto", "alternate", [seed.id], "robust") + const diverse = await evaluate("search-pareto", alternate.id, 0.8, "passed", { robustness: 0.9, latency: 10 }) + expect(diverse.bestID).toBe(seed.id) + expect(diverse.archiveIDs).toEqual([seed.id, alternate.id]) + expect(HarnessSearch.frontier(diverse).map((item) => item.id)).toEqual([seed.id, alternate.id]) + expect(HarnessSearch.recommend(diverse)).toMatchObject({ + strategy: "fuse", + parentIDs: [seed.id, alternate.id], + reasons: ["stalled:1", "cross-branch-fusion", "pareto-frontier:2", "multi-metric-complementarity"], + }) + + const dominated = await add("search-pareto", "dominated", [seed.id], "weak") + const state = await evaluate("search-pareto", dominated.id, 0.7, "passed", { robustness: 0.1, latency: 30 }) + expect(state.bestID).toBe(seed.id) + expect(state.archiveIDs).toEqual([seed.id, alternate.id]) + }) + + test("rejects incomplete or primary-duplicating objective contracts", async () => { + await expect( + contract("search-objective-duplicate", { + objectives: [{ metric: "score", direction: "maximize" }], + }), + ).rejects.toThrow("cannot duplicate the primary") + + await setup("search-objective-missing", { + objectives: [{ metric: "robustness", direction: "maximize" }], + }) + const seed = await add("search-objective-missing", "seed") + await HarnessEvaluation.record({ + schemaVersion: 1, + runID: "run-search-objective-missing", + sessionID: "search-objective-missing", + subject: { type: "candidate", id: seed.id }, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status: "passed", + score: 0.9, + metrics: { score: 0.9 }, + checks: [{ id: "gate", status: "passed", blocking: true, evidence: [`candidate:${seed.id}`] }], + evidence: [`report:${seed.id}`], + evaluatedAt: Date.now(), + }) + await expect(HarnessSearch.verify({ sessionID: "search-objective-missing", candidateID: seed.id })).rejects.toThrow( + "missing declared objective metric robustness", + ) + }) + + test("fails closed when the persisted Pareto archive is edited", async () => { + await setup("search-pareto-tamper", { + objectives: [{ metric: "robustness", direction: "maximize" }], + }) + const seed = await add("search-pareto-tamper", "seed") + await evaluate("search-pareto-tamper", seed.id, 0.9, "passed", { robustness: 0.8 }) + const file = path.join(Global.Path.data, "harness", "search", "search-pareto-tamper.json") + const state = JSON.parse(await fs.readFile(file, "utf8")) + state.archiveIDs = [] + await fs.writeFile(file, JSON.stringify(state)) + await expect(HarnessSearch.read("search-pareto-tamper")).rejects.toThrow("Pareto archive does not match") + }) + + test("fails closed when persisted island policy or assignment is edited", async () => { + await setup("search-island-tamper", { candidates: 8 }) + const seed = await add("search-island-tamper", "seed") + const file = path.join(Global.Path.data, "harness", "search", "search-island-tamper.json") + const assignment = JSON.parse(await fs.readFile(file, "utf8")) + assignment.candidates[seed.id].island = 1 + await fs.writeFile(file, JSON.stringify(assignment)) + await expect(HarnessSearch.read("search-island-tamper")).rejects.toThrow("island does not match") + + assignment.candidates[seed.id].island = 0 + assignment.population.migrationInterval = 99 + await fs.writeFile(file, JSON.stringify(assignment)) + await expect(HarnessSearch.read("search-island-tamper")).rejects.toThrow("server-derived budget policy") + + assignment.population.migrationInterval = 2 + assignment.candidates[seed.id].generation = 7 + await fs.writeFile(file, JSON.stringify(assignment)) + await expect(HarnessSearch.read("search-island-tamper")).rejects.toThrow("generation does not match") + + assignment.candidates[seed.id].generation = 0 + assignment.candidates[seed.id].proposal = "edited after registration" + await fs.writeFile(file, JSON.stringify(assignment)) + await expect(HarnessSearch.read("search-island-tamper")).rejects.toThrow("identity does not match") + }) + + test("migrates legacy single-metric search state without changing its frontier", async () => { + await setup("search-pareto-legacy") + const seed = await add("search-pareto-legacy", "seed") + await evaluate("search-pareto-legacy", seed.id, 0.9) + const file = path.join(Global.Path.data, "harness", "search", "search-pareto-legacy.json") + const legacy = JSON.parse(await fs.readFile(file, "utf8")) + legacy.schemaVersion = 1 + delete legacy.objectives + delete legacy.archiveIDs + delete legacy.population + delete legacy.candidates[seed.id].inspirationIDs + delete legacy.candidates[seed.id].island + delete legacy.candidates[seed.id].ordinal + await fs.writeFile(file, JSON.stringify(legacy)) + const state = await HarnessSearch.read("search-pareto-legacy") + expect(state.schemaVersion).toBe(4) + expect(state.proposalPolicy).toBe("advisory-v2") + expect(state.population).toEqual({ mode: "legacy", count: 1, initial: 1, topology: "ring", migrationInterval: 1 }) + expect(state.objectives).toEqual([]) + expect(state.archiveIDs).toEqual([seed.id]) + expect(state.bestID).toBe(seed.id) + await expect(HarnessSearch.reserve({ sessionID: "search-pareto-legacy", count: 1 })).rejects.toThrow( + "leased proposal policy", + ) + }) + + test("preserves branch diversity during early exploration", async () => { + await setup("search-explore", { candidates: 10 }) + const seed = await add("search-explore", "seed", [], "baseline") + await evaluate("search-explore", seed.id, 0.5) + const a = await add("search-explore", "a", [seed.id], "common") + await evaluate("search-explore", a.id, 0.9) + const b = await add("search-explore", "b", [seed.id], "rare") + await evaluate("search-explore", b.id, 0.6) + const a2 = await add("search-explore", "a2", [a.id], "common") + const state = await evaluate("search-explore", a2.id, 0.8) + const choice = HarnessSearch.recommend(state) + expect(choice.strategy).toBe("explore") + expect(choice.parentIDs).toEqual([b.id]) + }) + + test("opens independent roots early and switches to strategy divergence after prolonged stagnation", async () => { + await setup("search-adaptive", { candidates: 10, stall: 1 }) + const seed = await add("search-adaptive", "seed", [], "base") + let state = await evaluate("search-adaptive", seed.id, 0.9) + expect(HarnessSearch.recommend(state)).toMatchObject({ strategy: "explore", parentIDs: [] }) + const alternate = await add("search-adaptive", "alternate", [], "alternate") + state = await evaluate("search-adaptive", alternate.id, 0.7) + expect(HarnessSearch.recommend(state).strategy).toBe("fuse") + const fused = await add("search-adaptive", "fused", [seed.id, alternate.id], "fusion") + state = await evaluate("search-adaptive", fused.id, 0.8) + expect(HarnessSearch.recommend(state)).toMatchObject({ strategy: "diverge", parentIDs: [seed.id] }) + }) + + test("switches to verified-rank exploitation after half the budget", async () => { + await setup("search-exploit", { candidates: 6 }) + const seed = await add("search-exploit", "seed", [], "base") + await evaluate("search-exploit", seed.id, 0.5) + const a = await add("search-exploit", "a", [seed.id], "a") + await evaluate("search-exploit", a.id, 0.7) + const b = await add("search-exploit", "b", [seed.id], "b") + const state = await evaluate("search-exploit", b.id, 0.9) + expect(HarnessSearch.recommend(state)).toMatchObject({ strategy: "exploit", parentIDs: [b.id] }) + }) + + test("minimizes when the benchmark contract declares a loss metric", async () => { + await setup("search-minimize", { direction: "minimize" }) + const seed = await add("search-minimize", "seed", [], "base") + await evaluate("search-minimize", seed.id, 0.5) + const child = await add("search-minimize", "child", [seed.id], "lower-loss") + const state = await evaluate("search-minimize", child.id, 0.2) + expect(state.bestID).toBe(child.id) + }) + + test("recommends cross-branch fusion after verified stagnation", async () => { + await setup("search-fuse", { candidates: 8, stall: 1 }) + const seed = await add("search-fuse", "seed", [], "base") + await evaluate("search-fuse", seed.id, 0.9) + const weaker = await add("search-fuse", "weaker", [seed.id], "alternate") + const state = await evaluate("search-fuse", weaker.id, 0.7) + expect(state.stalled).toBe(1) + expect(HarnessSearch.recommend(state)).toMatchObject({ + strategy: "fuse", + parentIDs: [seed.id, weaker.id], + }) + }) + + test("enforces the candidate budget transactionally and survives restart", async () => { + await setup("search-budget", { candidates: 1 }) + const seed = await add("search-budget", "seed") + await evaluate("search-budget", seed.id, 0.5) + const rejected = await add("search-budget", "overflow", [seed.id]) + expect(rejected.accepted).toBe(false) + expect(rejected.state).toMatchObject({ status: "completed", stopReason: "budget_exhausted" }) + expect(Object.keys(await HarnessSearch.read("search-budget").then((state) => state.candidates))).toHaveLength(1) + }) + + test("serializes concurrent branches against one remaining budget slot", async () => { + await setup("search-concurrent", { candidates: 2 }) + const seed = await add("search-concurrent", "seed") + await evaluate("search-concurrent", seed.id, 0.5) + const results = await Promise.all([ + add("search-concurrent", "branch-a", [seed.id], "a"), + add("search-concurrent", "branch-b", [seed.id], "b"), + ]) + expect(results.filter((result) => result.accepted)).toHaveLength(1) + const state = await HarnessSearch.read("search-concurrent") + expect(Object.keys(state.candidates)).toHaveLength(2) + expect(state).toMatchObject({ status: "completed", stopReason: "budget_exhausted" }) + }) + + test("stops immediately when the declared target is reached", async () => { + await setup("search-target", { target: 0.8 }) + const seed = await add("search-target", "seed") + const state = await evaluate("search-target", seed.id, 0.81) + expect(state).toMatchObject({ status: "completed", stopReason: "objective_met", bestID: seed.id }) + }) + + test("makes verified evaluations immutable and manual stops resumable", async () => { + await setup("search-stop") + const seed = await add("search-stop", "seed") + await evaluate("search-stop", seed.id, 0.5) + await expect( + HarnessEvaluation.record({ + ...(await HarnessEvaluation.read("search-stop"))!, + score: 0.6, + metrics: { score: 0.6 }, + }), + ).rejects.toThrow("immutable") + const stopped = await HarnessSearch.finish("search-stop", "user_cancelled") + expect(await HarnessSearch.read("search-stop")).toEqual(stopped) + }) +}) diff --git a/backend/cli/test/session/harness-semantic.test.ts b/backend/cli/test/session/harness-semantic.test.ts new file mode 100644 index 00000000..7cf23c2c --- /dev/null +++ b/backend/cli/test/session/harness-semantic.test.ts @@ -0,0 +1,386 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessOrchestrator } from "../../src/session/harness/orchestrator" +import { HarnessReport } from "../../src/session/harness/report" +import { HarnessSearch } from "../../src/session/harness/search" +import { HarnessSemantic } from "../../src/session/harness/semantic" + +const sessions = new Set() +const receipts = new Set() +const evaluator = "semantic-evaluator-capability-token-0000000000000000" +const reviewer = "semantic-reviewer-capability-token-00000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const digest = (value: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex") +const objective = "Find a non-vacuous new method for the declared scientific problem under the stated constraints" + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "orchestration", "reports", "search"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await Promise.all( + [...receipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "semantics", `${receiptID}.json`), { force: true }), + ), + ) + sessions.clear() + receipts.clear() +}) + +function protocol(input = objective) { + return HarnessContract.SemanticAudit.parse({ + protocolVersion: "semantic-audit-v1", + reviewer: { name: "independent-domain-panel", version: "2026.08", source: "external" }, + scope: { + objectiveSHA256: hash(input), + criteria: [ + { id: "target", requirement: "Address the intended target rather than a weaker surrogate." }, + { id: "constraints", requirement: "Respect every stated scientific constraint." }, + ], + forbiddenShortcuts: [ + { id: "vacuity", description: "Do not satisfy the statement through an empty or trivial interpretation." }, + { id: "lookup", description: "Do not present a known result as a new result." }, + ], + literature: { cutoff: "2026-08-01", corpusSHA256: hash("frozen-literature-corpus") }, + noveltyFloor: "minor", + }, + minReviewers: 2, + minConfidence: 0.8, + }) +} + +function task(sessionID: string): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "semantic-intent-task", + split: "validation", + evaluator: { + name: "official-scientific-evaluator", + version: "4", + source: "benchmark", + token: evaluator, + }, + semanticAudit: { protocol: protocol(), token: reviewer }, + objective, + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + tools: ["read", "bash"], + skills: [], + budget: { steps: 20 }, + seed: 23, + intervention: "autonomous", + contamination: { policy: "review corpus and hidden cases remain withheld", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +function review( + actor: string, + sessionID: string, + overrides: Partial = {}, +): HarnessSemantic.Review { + return HarnessSemantic.Review.parse({ + actor, + sessionID, + correctness: "passed", + alignment: "intended", + novelty: "minor", + vacuous: false, + confidence: 0.95, + criteria: [ + { id: "target", status: "passed", evidence: [`artifact:${actor}-target.json`] }, + { id: "constraints", status: "passed", evidence: [`artifact:${actor}-constraints.json`] }, + ], + shortcuts: [ + { id: "vacuity", observed: false, evidence: [`artifact:${actor}-vacuity.json`] }, + { id: "lookup", observed: false, evidence: [`artifact:${actor}-lookup.json`] }, + ], + literatureRefs: [`literature:${actor}-search.json`], + evidence: [`artifact:${actor}-review.json`], + summary: `${actor} independently found the result correct, intended, substantive, and above the novelty floor.`, + reviewedAt: Date.now(), + ...overrides, + }) +} + +function panel(overrides: Partial = {}) { + return [review("reviewer-a", "semantic-session-a", overrides), review("reviewer-b", "semantic-session-b")] +} + +async function audit(contract: HarnessContract.Info, reviews = panel(), subject?: HarnessSemantic.Subject) { + const receipt = await HarnessSemantic.record( + { + sessionID: contract.sessionID, + reviewerToken: reviewer, + subject: subject ?? { type: "run", id: contract.runID }, + reviews, + }, + await HarnessAdapter.authorizeSemantic(contract.sessionID, reviewer), + ) + receipts.add(receipt.receiptID) + return receipt +} + +function evaluation(contract: HarnessContract.Info, receiptID?: string) { + const checks = HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: evaluator, + semanticReceiptID: receiptID, + status: "passed", + score: 0.91, + metrics: { score: 0.91 }, + checks, + evidence: ["official:score.json"], + evaluatedAt: Date.now(), + }) +} + +describe("semantic intent and novelty audit", () => { + test("injects the complete frozen meaning policy into main and coalition contexts", async () => { + const contract = await HarnessAdapter.bind(task("semantic-prompt")) + const prompt = await HarnessSemantic.context(contract.sessionID) + expect(prompt).toContain("Address the intended target rather than a weaker surrogate.") + expect(prompt).toContain("Do not satisfy the statement through an empty or trivial interpretation.") + expect(prompt).toContain("Minimum novelty: minor") + expect(prompt).not.toContain(reviewer) + + const state = await HarnessOrchestrator.initialize(contract.sessionID) + const work = HarnessOrchestrator.ready(state)[0]! + expect(work.prompt).toContain(prompt) + expect(work.prompt).toContain("Your output is provisional orchestration state") + }) + + test("gates a final result on a meaningful independent review panel", async () => { + const contract = await HarnessAdapter.bind(task("semantic-pass")) + await expect(HarnessAdapter.ingest(evaluation(contract))).rejects.toThrow("semantic audit receipt") + const receipt = await audit(contract) + expect(receipt).toMatchObject({ + status: "meaningful", + subject: { type: "run", id: contract.runID }, + reviewer: contract.semanticAudit?.reviewer, + failures: [], + }) + expect(receipt.reviews.map((item) => item.actor)).toEqual(["reviewer-a", "reviewer-b"]) + expect(JSON.stringify(receipt)).not.toContain(reviewer) + expect(JSON.stringify(receipt)).not.toContain(evaluator) + + const result = await HarnessAdapter.ingest(evaluation(contract, receipt.receiptID)) + expect(result.evaluation).toMatchObject({ status: "passed", semanticReceiptID: receipt.receiptID }) + const report = HarnessReport.compile({ contract, evaluations: [result.evaluation], generatedAt: Date.now() }) + expect(report.quality.semanticReceiptID).toBe(receipt.receiptID) + }) + + test("classifies correct loopholes and below-floor rediscoveries as technical only", async () => { + const cases: Array<[string, Partial, string]> = [ + ["misinterpreted", { alignment: "misinterpreted" }, "problem_misinterpreted"], + ["vacuous", { vacuous: true }, "vacuous_solution"], + ["rediscovered", { novelty: "rediscovery" }, "below_minor"], + [ + "shortcut", + { + shortcuts: [ + { id: "vacuity", observed: true, evidence: ["artifact:vacuity-found.json"] }, + { id: "lookup", observed: false, evidence: ["artifact:lookup-clear.json"] }, + ], + }, + "shortcut_vacuity_observed", + ], + ] + for (const [name, override, failure] of cases) { + const contract = await HarnessAdapter.bind(task(`semantic-${name}`)) + const receipt = await audit(contract, panel(override)) + expect(receipt.status).toBe("technical_only") + expect(receipt.failures).toContainEqual(expect.stringContaining(failure)) + await expect(HarnessAdapter.ingest(evaluation(contract, receipt.receiptID))).rejects.toThrow( + "meaningful semantic audit receipt", + ) + } + + const input = task("semantic-not-required-novelty") + const current = input.semanticAudit!.protocol + const contract = await HarnessAdapter.bind({ + ...input, + semanticAudit: { + token: reviewer, + protocol: { ...current, scope: { ...current.scope, noveltyFloor: "known" } }, + }, + }) + const receipt = await audit(contract, panel({ novelty: "not_required" })) + expect(receipt.status).toBe("technical_only") + expect(receipt.failures).toContainEqual(expect.stringContaining("not_required_below_known")) + }) + + test("preserves uncertainty and incorrectness as distinct backend outcomes", async () => { + const ambiguous = await HarnessAdapter.bind(task("semantic-ambiguous")) + const uncertain = await audit(ambiguous, panel({ confidence: 0.7 })) + expect(uncertain.status).toBe("ambiguous") + expect(uncertain.failures).toContain("reviewer-a:low_confidence") + + const incorrect = await HarnessAdapter.bind(task("semantic-incorrect")) + const failed = await audit(incorrect, panel({ correctness: "failed" })) + expect(failed.status).toBe("failed") + expect(failed.failures).toContain("reviewer-a:correctness_failed") + }) + + test("requires complete frozen checks and distinct reviewer identities", async () => { + const contract = await HarnessAdapter.bind(task("semantic-panel")) + const missing = panel() + missing[0] = { ...missing[0]!, criteria: missing[0]!.criteria.slice(0, 1) } + await expect(audit(contract, missing)).rejects.toThrow("frozen problem scope") + await expect(audit(contract, [review("same", "session-a"), review("same", "session-b")])).rejects.toThrow( + "distinct actors", + ) + await expect( + audit(contract, [review("actor-a", "same-session"), review("actor-b", "same-session")]), + ).rejects.toThrow("distinct sessions") + }) + + test("does not replay a semantic receipt across runs or subjects", async () => { + const first = await HarnessAdapter.bind(task("semantic-source")) + const receipt = await audit(first) + const second = await HarnessAdapter.bind(task("semantic-target")) + await expect(HarnessAdapter.ingest(evaluation(second, receipt.receiptID))).rejects.toThrow( + "different harness session", + ) + + await expect( + HarnessSemantic.assert({ + contract: first, + receiptID: receipt.receiptID, + subject: { type: "candidate", id: hash("candidate") }, + evaluatedAt: Date.now(), + recordedAt: Date.now(), + requirePassed: true, + }), + ).rejects.toThrow("different evaluation subject") + }) + + test("cannot pre-sign a candidate before its immutable search artifact exists", async () => { + const sessionID = "semantic-candidate-time" + const contract = await HarnessAdapter.bind({ + ...task(sessionID), + profile: "optimize", + budget: { steps: 20, candidates: 2 }, + }) + const missing = { type: "candidate" as const, id: hash("future-candidate") } + await expect(audit(contract, panel(), missing)).rejects.toThrow("does not exist in the bound search") + + const state = await HarnessSearch.initialize({ sessionID, candidates: 2 }) + const recommendation = HarnessSearch.recommend(state) + const added = await HarnessSearch.add({ + sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "semantic-candidate", + proposal: "A concrete candidate reviewed only after its artifact exists", + artifact: { uri: "candidate://semantic-time", sha256: hash("semantic-candidate-artifact") }, + }) + const candidate = (await HarnessSearch.read(sessionID)).candidates[added.id]! + await expect( + audit(contract, panel({ reviewedAt: candidate.createdAt - 1 }), { type: "candidate", id: candidate.id }), + ).rejects.toThrow("bound subject interval") + + const receipt = await audit(contract, panel(), { type: "candidate", id: candidate.id }) + const result = await HarnessAdapter.ingest({ + ...evaluation(contract, receipt.receiptID), + candidateID: candidate.id, + }) + expect(result.evaluation).toMatchObject({ + subject: { type: "candidate", id: candidate.id }, + semanticReceiptID: receipt.receiptID, + status: "passed", + }) + }) + + test("rejects objective drift, shared capabilities, and receipt tampering", async () => { + await expect( + HarnessAdapter.bind({ + ...task("semantic-objective-drift"), + semanticAudit: { protocol: protocol("different objective"), token: reviewer }, + }), + ).rejects.toThrow("objective commitment") + expect(() => + HarnessAdapter.Task.parse({ + ...task("semantic-shared-capability"), + semanticAudit: { protocol: protocol(), token: evaluator }, + }), + ).toThrow("capabilities must differ") + + const contract = await HarnessAdapter.bind(task("semantic-tamper")) + const receipt = await audit(contract) + const target = path.join(Global.Path.data, "harness", "semantics", `${receipt.receiptID}.json`) + await Bun.write(target, JSON.stringify({ ...receipt, recordedAt: receipt.recordedAt + 1 })) + expect(await HarnessSemantic.read(receipt.receiptID)).toBeNull() + await Bun.write(target, JSON.stringify({ ...receipt, status: "technical_only" })) + expect(await HarnessSemantic.read(receipt.receiptID)).toBeNull() + await expect(HarnessAdapter.ingest(evaluation(contract, receipt.receiptID))).rejects.toThrow("Unknown or corrupt") + }) + + test("rederives semantic status even after a content-addressed on-disk forgery", async () => { + const contract = await HarnessAdapter.bind(task("semantic-derived-tamper")) + const receipt = await audit(contract, panel({ vacuous: true })) + expect(receipt.status).toBe("technical_only") + const forged = { ...receipt, status: "meaningful" as const, failures: [] } + const stable = structuredClone(forged) as Record + delete stable.receiptID + const receiptID = digest(stable) + receipts.add(receiptID) + await Bun.write( + path.join(Global.Path.data, "harness", "semantics", `${receiptID}.json`), + JSON.stringify({ ...forged, receiptID }), + ) + expect(await HarnessSemantic.read(receiptID)).not.toBeNull() + await expect(HarnessAdapter.ingest(evaluation(contract, receiptID))).rejects.toThrow("backend-derived review state") + }) + + test("exposes semantic receipts only to the bound review capability", async () => { + const contract = await HarnessAdapter.bind(task("semantic-route")) + const app = HarnessRoutes() + const response = await app.request("/semantics/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionID: contract.sessionID, + reviewerToken: reviewer, + subject: { type: "run", id: contract.runID }, + reviews: panel(), + }), + }) + expect(response.status).toBe(200) + const receipt = (await response.json()) as HarnessSemantic.Receipt + receipts.add(receipt.receiptID) + + await expect(HarnessAdapter.authorizeSemantic(contract.sessionID, evaluator)).rejects.toThrow("rejected") + + const read = await app.request(`/semantics/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, reviewerToken: reviewer }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, status: "meaningful" }) + }) +}) diff --git a/backend/cli/test/session/harness-simulation.test.ts b/backend/cli/test/session/harness-simulation.test.ts new file mode 100644 index 00000000..d08180ae --- /dev/null +++ b/backend/cli/test/session/harness-simulation.test.ts @@ -0,0 +1,270 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { Global } from "../../src/global" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessSearch } from "../../src/session/harness/search" +import { HarnessSimulation } from "../../src/session/harness/simulation" + +const sessions = new Set() +const token = "simulation-evaluator-capability-token-000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations", "search", "simulations"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + await fs.rm(path.join(Global.Path.data, "harness", "retrospectives"), { recursive: true, force: true }) + sessions.clear() +}) + +function protocol() { + return HarnessContract.Simulation.parse({ + kind: "pde", + engine: { + name: "reference-solver", + version: "1.2.3", + commandSHA256: hash("reference-solver case.json"), + configSHA256: hash("effective-config"), + }, + problemSHA256: hash("equation-domain-bc-ic"), + reference: { kind: "manufactured", identity: "mms-v1", sha256: hash("manufactured-solution") }, + validation: { + errorNorm: "relative L2", + minLevels: 3, + maxLevels: 6, + expectedOrder: 2, + orderTolerance: 0.2, + maxResidual: 1e-8, + invariantTolerances: { mass_drift: 1e-6 }, + requiredStressTests: ["solver_tolerance_sensitivity", "reference_replay"], + }, + }) +} + +function task(sessionID: string, optimize = false): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: optimize ? "physics" : "pde", + version: "2026.08", + taskID: "sim-1", + split: "validation", + evaluator: { name: "simulation-evaluator", version: "4", source: "benchmark", token }, + objective: "Validate the exact numerical artifact before accepting its score", + profile: optimize ? "optimize" : "numerical", + simulation: protocol(), + packs: optimize ? ["pde"] : [], + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + tools: ["read", "bash"], + skills: [{ name: "simulator-validation", sha256: hash("simulator-validation-skill") }], + budget: { steps: 30, ...(optimize ? { candidates: 2 } : {}) }, + seed: 11, + intervention: "autonomous", + contamination: { policy: "reference outputs remain evaluator-private", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +function submit(contract: HarnessContract.Info, subject?: HarnessSimulation.Submit["subject"]) { + const simulation = contract.simulation + if (!simulation) throw new Error("Expected a simulator protocol") + return HarnessSimulation.Submit.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + subject: subject ?? { + type: "run", + id: contract.runID, + artifact: { uri: "artifact:solution", sha256: hash("solution-artifact") }, + }, + engine: simulation.engine, + problemSHA256: simulation.problemSHA256, + reference: simulation.reference, + validationInputSHA256: hash("validation-input"), + levels: [ + { label: "coarse", h: 0.1, error: 0.01, residual: 1e-9, invariants: { mass_drift: 2e-7 } }, + { label: "medium", h: 0.05, error: 0.0025, residual: 2e-9, invariants: { mass_drift: 3e-7 } }, + { label: "fine", h: 0.025, error: 0.000625, residual: 3e-9, invariants: { mass_drift: 4e-7 } }, + ], + stressTests: [ + { id: "solver_tolerance_sensitivity", status: "passed", evidence: ["artifact:tolerance-sweep.json"] }, + { id: "reference_replay", status: "passed", evidence: ["artifact:reference-replay.json"] }, + ], + evidence: ["artifact:validation-report.json"], + evaluatedAt: Date.now(), + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + +function evaluation(contract: HarnessContract.Info, receiptID?: string, candidateID?: string) { + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: token, + candidateID, + simulationReceiptID: receiptID, + status: "passed", + score: 0.91, + metrics: { score: 0.91 }, + checks: checks(contract), + evidence: ["official:numerical-result.json"], + evaluatedAt: Date.now(), + }) +} + +describe("contract-bound simulator validation", () => { + test("derives convergence and invariant checks instead of accepting a self-reported pass", async () => { + const contract = await HarnessAdapter.bind(task("simulation-run")) + const input = { ...submit(contract), evaluatedAt: contract.createdAt + 10 } + const receipt = await HarnessSimulation.record(input, await HarnessAdapter.authorize(contract.sessionID, token)) + expect(receipt.status).toBe("passed") + expect(receipt.medianObservedOrder).toBeCloseTo(2) + expect(receipt.checks).toMatchObject({ + enoughLevels: true, + resolutionDecreases: true, + errorDecreases: true, + observedOrder: true, + residualBound: true, + invariants: { mass_drift: true }, + stressTests: { solver_tolerance_sensitivity: true, reference_replay: true }, + }) + expect(JSON.stringify(receipt)).not.toContain(token) + expect(await HarnessSimulation.list(contract.sessionID)).toEqual([receipt]) + expect((await HarnessSimulation.record(input, contract)).receiptID).toBe(receipt.receiptID) + + await expect( + HarnessSimulation.record({ ...input, engine: { ...input.engine, version: "silently-changed" } }, contract), + ).rejects.toThrow("engine does not match") + expect(await HarnessSimulation.list(contract.sessionID)).toHaveLength(1) + expect(() => + HarnessContract.Info.parse({ + ...contract, + benchmark: { ...contract.benchmark, evaluatorSource: "human" }, + }), + ).toThrow("capability-authenticated evaluator source") + }) + + test("blocks final success without a passing receipt and retains failed numerical evidence", async () => { + const contract = await HarnessAdapter.bind(task("simulation-gate")) + await expect(HarnessAdapter.ingest(evaluation(contract))).rejects.toThrow("must reference") + + const input = { ...submit(contract), evaluatedAt: contract.createdAt + 10 } + const levels = input.levels.map((level, index) => + index === 1 ? { ...level, error: 0.008, residual: 1e-4 } : level, + ) + const failed = await HarnessSimulation.record( + { ...input, levels, validationInputSHA256: hash("failed-validation-input") }, + contract, + ) + expect(failed.status).toBe("failed") + expect(failed.checks.residualBound).toBe(false) + expect(failed.medianObservedOrder).toBeCloseTo(2) + expect(failed.checks.observedOrder).toBe(false) + await expect( + HarnessAdapter.ingest({ ...evaluation(contract, failed.receiptID), evaluatedAt: input.evaluatedAt + 1 }), + ).rejects.toThrow("requires a passing simulation receipt") + + const passed = await HarnessSimulation.record(input, contract) + await expect( + HarnessAdapter.ingest({ + ...evaluation(contract, passed.receiptID), + evaluatedAt: contract.createdAt + 5, + }), + ).rejects.toThrow("predates its referenced validation receipt") + const result = await HarnessAdapter.ingest({ + ...evaluation(contract, passed.receiptID), + evaluatedAt: input.evaluatedAt + 2, + }) + expect(result.evaluation).toMatchObject({ status: "passed", simulationReceiptID: passed.receiptID }) + expect(await HarnessSimulation.list(contract.sessionID)).toHaveLength(2) + }) + + test("fails closed when a persisted receipt is edited outside the append-only API", async () => { + const contract = await HarnessAdapter.bind(task("simulation-tamper")) + const receipt = await HarnessSimulation.record(submit(contract), contract) + const file = path.join(Global.Path.data, "harness", "simulations", `${encodeURIComponent(contract.sessionID)}.json`) + const journal = (await Bun.file(file).json()) as { items: Record } + journal.items[receipt.receiptID].status = "failed" + await Bun.write(file, JSON.stringify(journal)) + await expect(HarnessSimulation.list(contract.sessionID)).rejects.toThrow("content hash is invalid") + }) + + test("binds candidate receipts to the exact content-addressed artifact", async () => { + const contract = await HarnessAdapter.bind(task("simulation-candidate", true)) + const search = await HarnessSearch.initialize({ sessionID: contract.sessionID, candidates: 2 }) + const recommendation = HarnessSearch.recommend(search) + const candidate = await HarnessSearch.add({ + sessionID: contract.sessionID, + recommendationID: recommendation.id, + parentIDs: recommendation.parentIDs, + inspirationIDs: recommendation.inspirationIDs, + branch: "finite-volume", + proposal: "Validate the finite-volume implementation", + artifact: { uri: "artifact:solver-candidate", sha256: hash("candidate-artifact") }, + }) + const subject = { + type: "candidate" as const, + id: candidate.id, + artifact: candidate.state.candidates[candidate.id]!.artifact, + } + const input = submit(contract, subject) + await expect( + HarnessSimulation.record( + { ...input, subject: { ...subject, artifact: { ...subject.artifact, sha256: hash("different-artifact") } } }, + contract, + ), + ).rejects.toThrow("does not match the candidate artifact") + + const receipt = await HarnessSimulation.record(input, contract) + const result = await HarnessAdapter.ingest(evaluation(contract, receipt.receiptID, candidate.id)) + expect(result.search?.bestID).toBe(candidate.id) + }) + + test("exposes receipt recording and reads only behind the evaluator capability", async () => { + const contract = await HarnessAdapter.bind(task("simulation-route")) + const app = HarnessRoutes() + const recorded = await app.request("/simulations/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(submit(contract)), + }) + expect(recorded.status).toBe(200) + const receipt = (await recorded.json()) as HarnessSimulation.Info + expect(receipt.status).toBe("passed") + + const denied = await app.request(`/simulations/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: "short" }), + }) + expect(denied.status).not.toBe(200) + + const read = await app.request(`/simulations/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: token }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID, status: "passed" }) + }) +}) diff --git a/backend/cli/test/session/harness-skill.test.ts b/backend/cli/test/session/harness-skill.test.ts new file mode 100644 index 00000000..7a964474 --- /dev/null +++ b/backend/cli/test/session/harness-skill.test.ts @@ -0,0 +1,297 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessSkill } from "../../src/session/harness/skill" + +const names = new Set() +const sessions = new Set() +const token = "skill-evaluator-capability-token-000000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") + +afterEach(async () => { + await Promise.all( + [...names].flatMap((name) => [ + fs.rm(path.join(Global.Path.data, "learned-skill-proposals", name), { recursive: true, force: true }), + fs.rm(path.join(Global.Path.data, "learned-skills", name), { recursive: true, force: true }), + ]), + ) + await Promise.all( + [...sessions].flatMap((sessionID) => + ["bindings", "contracts", "evaluations"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ), + ) + names.clear() + sessions.clear() +}) + +function content(name: string, description = "Use when applying a verified held-out workflow.") { + return `---\nname: ${name}\ndescription: ${description}\nsource: test\n---\n\n# ${name}\n\nFollow the evidence-backed procedure.\n` +} + +async function proposal(name: string) { + names.add(name) + const description = "Use when applying a verified held-out workflow." + return HarnessSkill.propose({ + name, + description, + content: content(name, description), + origin: "conversation", + sessionID: "source-session", + createdAt: Date.now(), + }) +} + +function synthetic(taskID: string, input?: { improved?: boolean; nonregressing?: boolean; precision?: number }) { + const precision = input?.precision ?? 0.9 + return HarnessSkill.Evidence.parse({ + id: hash(`evidence-${taskID}`), + proposalSHA256: hash("proposal"), + benchmark: { + name: "statistics", + version: "1", + taskID, + split: "held_out", + metric: "score", + direction: "maximize", + }, + candidate: { + sessionID: `candidate-${taskID}`, + runID: `candidate-run-${taskID}`, + status: "passed", + score: 0.8, + evaluationSHA256: hash(`candidate-${taskID}`), + }, + control: { + sessionID: `control-${taskID}`, + runID: `control-run-${taskID}`, + status: "passed", + score: 0.7, + evaluationSHA256: hash(`control-${taskID}`), + }, + nonregressing: input?.nonregressing ?? true, + improved: input?.improved ?? true, + trigger: { + datasetSHA256: hash("trigger-set"), + split: "held_out", + examples: 20, + truePositive: Math.round(10 * precision), + falsePositive: 10 - Math.round(10 * precision), + trueNegative: Math.round(10 * precision), + falseNegative: 10 - Math.round(10 * precision), + precision, + recall: precision, + }, + evaluator: { name: "official", version: "1" }, + recordedAt: Date.now(), + }) +} + +async function pair(input: { + name: string + sha: string + taskID: string + candidateScore: number + controlScore: number +}) { + const create = async (role: "candidate" | "control") => { + const sessionID = `skill-${input.taskID}-${role}` + sessions.add(sessionID) + const score = role === "candidate" ? input.candidateScore : input.controlScore + const contract = await HarnessAdapter.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "1", + taskID: input.taskID, + split: "held_out", + evaluator: { name: "official", version: "1", source: "benchmark", token }, + objective: "Improve the fixed held-out statistical workflow", + metric: { name: "score", direction: "maximize" }, + model: { provider: "test", name: "model" }, + tools: ["read", "bash"], + skills: role === "candidate" ? [{ name: input.name, version: "candidate", sha256: input.sha }] : [], + budget: { steps: 20, tokens: 10_000 }, + seed: 7, + intervention: "autonomous", + contamination: { policy: "hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) + const checks = HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`receipt:${check.id}`], + })) + await HarnessAdapter.ingest({ + schemaVersion: 1, + runID: contract.runID, + sessionID, + evaluatorToken: token, + status: "passed", + score, + metrics: { score }, + checks, + evidence: ["official:skill-pair"], + evaluatedAt: Date.now(), + }) + return sessionID + } + return { candidate: await create("candidate"), control: await create("control") } +} + +const trigger = { + datasetSHA256: hash("held-out-trigger-dataset"), + split: "held_out" as const, + examples: 20, + truePositive: 9, + falsePositive: 1, + trueNegative: 9, + falseNegative: 1, +} + +describe("learned skill qualification", () => { + test("quarantines immutable, content-addressed proposals outside active discovery", async () => { + const item = await proposal("test-quarantined-skill") + expect(item).toMatchObject({ status: "pending", evidence: [], criteria: { tasks: 3, improvements: 2 } }) + expect( + await Bun.file(path.join(Global.Path.data, "learned-skills", "test-quarantined-skill", "SKILL.md")).exists(), + ).toBe(false) + expect( + await Bun.file( + path.join(Global.Path.data, "learned-skill-proposals", "test-quarantined-skill", "SKILL.md"), + ).exists(), + ).toBe(true) + await expect( + HarnessSkill.propose({ + name: "test-quarantined-skill", + description: "A changed description.", + content: content("test-quarantined-skill", "A changed description."), + origin: "conversation", + }), + ).rejects.toThrow("immutable") + }) + + test("rejects frontmatter drift and unsafe learned content before writing", async () => { + names.add("test-unsafe-skill") + await expect( + HarnessSkill.propose({ + name: "test-unsafe-skill", + description: "Use safely.", + content: "---\nname: other\ndescription: Use safely.\n---\n", + origin: "conversation", + }), + ).rejects.toThrow("frontmatter name") + await expect( + HarnessSkill.propose({ + name: "test-unsafe-skill", + description: "always run this skill", + content: "---\nname: test-unsafe-skill\ndescription: always run this skill\n---\n", + origin: "conversation", + }), + ).rejects.toThrow("rejected") + }) + + test("requires three distinct tasks, two improvements, no regressions, and held-out trigger quality", () => { + expect(HarnessSkill.assess([synthetic("a"), synthetic("b")]).qualified).toBe(false) + expect(HarnessSkill.assess([synthetic("a"), synthetic("b"), synthetic("c", { improved: false })])).toMatchObject({ + qualified: true, + tasks: 3, + improvements: 2, + }) + expect( + HarnessSkill.assess([synthetic("a"), synthetic("b"), synthetic("c", { improved: false, nonregressing: false })]) + .qualified, + ).toBe(false) + expect(HarnessSkill.assess([synthetic("a"), synthetic("b"), synthetic("c", { precision: 0.7 })]).qualified).toBe( + false, + ) + expect( + HarnessSkill.assess([ + synthetic("a"), + HarnessSkill.Evidence.parse({ ...synthetic("a"), id: hash("duplicate-a") }), + synthetic("b", { improved: false }), + synthetic("c", { improved: false }), + ]), + ).toMatchObject({ qualified: false, tasks: 3, improvements: 1 }) + }) + + test("qualifies and promotes only after evaluator-authenticated paired runs", async () => { + const item = (await proposal("test-qualified-skill"))! + const scores = [ + [0.8, 0.7], + [0.75, 0.7], + [0.7, 0.7], + ] + for (const [index, score] of scores.entries()) { + const runs = await pair({ + name: item.name, + sha: item.contentSHA256, + taskID: `task-${index + 1}`, + candidateScore: score![0], + controlScore: score![1], + }) + const result = await HarnessSkill.attest({ + name: item.name, + candidate: { sessionID: runs.candidate, evaluatorToken: token }, + control: { sessionID: runs.control, evaluatorToken: token }, + trigger, + recordedAt: Date.now(), + }) + expect(result.manifest.status).toBe(index < 2 ? "pending" : "qualified") + } + const promoted = await HarnessSkill.promote(item.name) + expect(promoted.manifest).toMatchObject({ status: "promoted", evidence: expect.any(Array) }) + expect(await Bun.file(promoted.path).text()).toBe(content(item.name)) + }) + + test("rejects unpaired contracts and wrong evaluator capabilities", async () => { + const item = (await proposal("test-pair-skill"))! + const runs = await pair({ + name: item.name, + sha: item.contentSHA256, + taskID: "pair", + candidateScore: 0.8, + controlScore: 0.7, + }) + await expect( + HarnessSkill.attest({ + name: item.name, + candidate: { sessionID: runs.candidate, evaluatorToken: "x".repeat(40) }, + control: { sessionID: runs.control, evaluatorToken: token }, + trigger, + }), + ).rejects.toThrow("capability was rejected") + await expect(HarnessSkill.promote(item.name)).rejects.toThrow("has not met") + }) + + test("makes evidence for one candidate/control pair immutable", async () => { + const item = (await proposal("test-evidence-skill"))! + const runs = await pair({ + name: item.name, + sha: item.contentSHA256, + taskID: "evidence", + candidateScore: 0.8, + controlScore: 0.7, + }) + const input = { + name: item.name, + candidate: { sessionID: runs.candidate, evaluatorToken: token }, + control: { sessionID: runs.control, evaluatorToken: token }, + trigger, + } + await HarnessSkill.attest(input) + await expect( + HarnessSkill.attest({ + ...input, + trigger: { ...trigger, truePositive: 8, falsePositive: 2, trueNegative: 8, falseNegative: 2 }, + }), + ).rejects.toThrow("immutable") + }) +}) diff --git a/backend/cli/test/session/harness-synthesis.test.ts b/backend/cli/test/session/harness-synthesis.test.ts new file mode 100644 index 00000000..5e7cb9fb --- /dev/null +++ b/backend/cli/test/session/harness-synthesis.test.ts @@ -0,0 +1,453 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessRoutes } from "../../src/server/routes/harness" +import { HarnessAdapter } from "../../src/session/harness/adapter" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessDomain } from "../../src/session/harness/domain" +import { HarnessJudge } from "../../src/session/harness/judge" +import { HarnessSynthesis } from "../../src/session/harness/synthesis" + +const sessions = new Set() +const judgeReceipts = new Set() +const evaluator = "scientific-synthesis-evaluator-token-000000000000000" +const auditor = "scientific-synthesis-auditor-token-00000000000000000" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const digest = (value: unknown) => new Bun.CryptoHasher("sha256").update(JSON.stringify(value)).digest("hex") + +const identity = (name: string) => ({ + name, + version: "1", + promptSHA256: hash(`${name}-prompt`), + configSHA256: hash(`${name}-config`), +}) + +function cases(): HarnessJudge.Case[] { + return [ + ...[1, 2].map((index) => ({ + id: `clean-${index}`, + commitment: hash(`clean-${index}`), + kind: "clean" as const, + decision: "accept" as const, + failureProbability: 0.05, + evidence: [`evidence://clean-${index}`], + })), + ...(["wrong_answer", "unsupported_claim", "data_leakage"] as const).flatMap((fault) => + [1, 2].map((index) => ({ + id: `${fault}-${index}`, + commitment: hash(`${fault}-${index}`), + kind: "fault" as const, + fault, + decision: "reject" as const, + failureProbability: 0.95, + evidence: [`evidence://${fault}-${index}`], + })), + ), + ] +} + +function audit() { + return HarnessContract.EvaluatorAudit.parse({ + protocolVersion: "evaluator-audit-v1", + auditor: { name: "independent-synthesis-auditor", version: "1", source: "external" }, + suite: { + name: "scientific-synthesis-fault-suite", + version: "1", + commitmentSHA256: HarnessJudge.commitment(cases()), + }, + minCleanCases: 2, + minCasesPerFault: 2, + requiredFaults: ["wrong_answer", "unsupported_claim", "data_leakage"], + minSensitivity: 0.9, + minSpecificity: 0.9, + minBalancedAccuracy: 0.9, + minFaultRecall: 0.9, + maxBrierScore: 0.1, + }) +} + +const reference = (): HarnessSynthesis.ReferenceFact[] => [ + { id: "r1", commitment: hash("salted-reference-one"), coverage: "covered", evidence: ["judge://recall-r1"] }, + { id: "r2", commitment: hash("salted-reference-two"), coverage: "missed", evidence: ["judge://recall-r2"] }, +] + +function protocol(facts = reference()) { + return HarnessContract.ScientificSynthesis.parse({ + protocolVersion: "scientific-synthesis-v1", + querySHA256: hash("public-question"), + referenceSHA256: hash("salted-hidden-reference"), + referenceFactsSHA256: HarnessSynthesis.referenceManifest(facts), + referenceFactCount: facts.length, + cutoff: "2026-01-01", + tools: ["google_search", "paper_search", "web_browse"], + traceSchemaSHA256: hash("tool-trace-schema"), + filterPolicySHA256: hash("clean-room-filter-policy"), + maxToolEvents: 8, + decomposer: identity("decomposer"), + judges: { precision: identity("precision-judge"), recall: identity("recall-judge") }, + minGeneratedFacts: 2, + minPrecision: 0.4, + minRecall: 0.5, + minF1: 0.45, + cleanRoomRequired: true, + judgeFailurePolicy: "inconclusive", + }) +} + +function task(sessionID: string, synthesis = protocol()): HarnessAdapter.Task { + sessions.add(sessionID) + return HarnessAdapter.Task.parse({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + benchmark: "statistics", + version: "2026.08", + taskID: "scientific-synthesis", + split: "validation", + evaluator: { name: "official-synthesis-evaluator", version: "1", source: "benchmark", token: evaluator }, + evaluatorAudit: { protocol: audit(), token: auditor }, + synthesis, + objective: "Synthesize a correct and comprehensive scientific conclusion without retrieving the answer key", + metric: { name: "factual_f1", direction: "maximize", target: 0.45 }, + model: { provider: "test", name: "research-agent" }, + tools: ["google_search", "paper_search", "web_browse"], + skills: [{ name: "run-clean-room-synthesis" }], + budget: { steps: 40 }, + seed: 31, + intervention: "autonomous", + contamination: { + policy: "reference review and derivative post-cutoff content remain evaluator-private", + hiddenTestsAccessible: false, + publicDataCutoff: "2026-01-01", + }, + createdAt: Date.now(), + }) +} + +const generated = (): HarnessSynthesis.GeneratedFact[] => [ + { id: "g1", commitment: hash("generated-one"), verdict: "supported", evidence: ["judge://precision-g1"] }, + { id: "g2", commitment: hash("generated-two"), verdict: "supported", evidence: ["judge://precision-g2"] }, + { + id: "g3", + commitment: hash("generated-three"), + verdict: "contradicted", + evidence: ["judge://precision-g3"], + }, +] + +const trace = () => [ + { + sequence: 1, + tool: "google_search" as const, + requestSHA256: hash("request-one"), + responseSHA256: hash("response-one"), + sourceSHA256: hash("source-one"), + publishedAt: "2025-06-01", + matches: { forbiddenDomain: false, referenceTitle: false }, + decision: "allowed" as const, + evidence: ["trace://search-one"], + }, + { + sequence: 2, + tool: "paper_search" as const, + requestSHA256: hash("request-two"), + responseSHA256: hash("response-two"), + sourceSHA256: hash("source-two"), + publishedAt: "2026-02-01", + matches: { forbiddenDomain: false, referenceTitle: false }, + decision: "blocked" as const, + evidence: ["trace://paper-two"], + }, +] + +async function qualify(contract: HarnessContract.Info) { + const receipt = await HarnessJudge.record( + { sessionID: contract.sessionID, auditorToken: auditor, cases: cases() }, + await HarnessAdapter.authorizeAuditor(contract.sessionID, auditor), + ) + judgeReceipts.add(receipt.receiptID) + return receipt +} + +function submit( + contract: HarnessContract.Info, + qualification: HarnessJudge.Receipt, + values: Partial = {}, +): HarnessSynthesis.Submit { + if (!contract.synthesis) throw new Error("Expected synthesis protocol") + return HarnessSynthesis.Submit.parse({ + sessionID: contract.sessionID, + evaluatorToken: evaluator, + subject: { type: "run", id: contract.runID }, + conclusionSHA256: hash("candidate-conclusion"), + evaluatorAuditReceiptID: qualification.receiptID, + trace: { + owner: "evaluator_runtime", + complete: true, + schemaSHA256: contract.synthesis.traceSchemaSHA256, + filterPolicySHA256: contract.synthesis.filterPolicySHA256, + events: trace(), + }, + decomposition: { + status: "passed", + outputSHA256: hash("atomic-decomposition"), + evidence: ["decomposition://report"], + }, + generatedFacts: generated(), + referenceFacts: reference(), + evaluatedAt: Math.max(Date.now(), contract.createdAt), + ...values, + }) +} + +const checks = (contract: HarnessContract.Info) => + HarnessDomain.compose(contract.packs ?? []).map((check) => ({ + id: check.id, + status: "passed" as const, + blocking: check.severity === "blocking", + evidence: [`evidence://${check.id}`], + })) + +function evaluation( + contract: HarnessContract.Info, + qualification: HarnessJudge.Receipt, + receipt?: HarnessSynthesis.Receipt, + score = receipt?.metrics.f1, +) { + return HarnessAdapter.Evaluation.parse({ + schemaVersion: 1, + runID: contract.runID, + sessionID: contract.sessionID, + evaluatorToken: evaluator, + evaluatorAuditReceiptID: qualification.receiptID, + synthesisReceiptID: receipt?.receiptID, + status: "passed", + score, + metrics: { factual_f1: score ?? 0 }, + checks: checks(contract), + evidence: ["official://scientific-synthesis-result"], + evaluatedAt: Math.max(Date.now(), receipt?.evaluatedAt ?? contract.createdAt), + }) +} + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => [ + ...["bindings", "contracts", "evaluations", "reports"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + fs.rm(path.join(Global.Path.data, "harness", "syntheses", "subjects", encodeURIComponent(sessionID)), { + recursive: true, + force: true, + }), + ]), + ) + await Promise.all( + [...judgeReceipts].map((receiptID) => + fs.rm(path.join(Global.Path.data, "harness", "judges", `${receiptID}.json`), { force: true }), + ), + ) + const receipts = await fs + .readdir(path.join(Global.Path.data, "harness", "syntheses", "receipts"), { withFileTypes: true }) + .catch(() => []) + await Promise.all( + receipts.flatMap((entry) => { + if (!entry.isFile() || !entry.name.endsWith(".json")) return [] + const file = path.join(Global.Path.data, "harness", "syntheses", "receipts", entry.name) + return [ + Bun.file(file) + .json() + .then((value) => (sessions.has(value.sessionID) ? fs.rm(file, { force: true }) : undefined)) + .catch(() => undefined), + ] + }), + ) + sessions.clear() + judgeReceipts.clear() +}) + +describe("clean-room atomic scientific synthesis", () => { + test("requires a qualified factuality/leakage evaluator and a compatible benchmark objective", async () => { + const valid = task("synthesis-contract") + const missing = HarnessAdapter.Task.parse({ ...valid, evaluatorAudit: undefined }) + await expect(HarnessAdapter.bind(missing)).rejects.toThrow("evaluator qualification") + const metric = HarnessAdapter.Task.parse({ ...valid, metric: { name: "accuracy", direction: "maximize" } }) + await expect(HarnessAdapter.bind(metric)).rejects.toThrow("factual_f1") + const tools = HarnessAdapter.Task.parse({ ...valid, tools: ["paper_search"] }) + await expect(HarnessAdapter.bind(tools)).rejects.toThrow("run tool allowlist") + const faults = HarnessAdapter.Task.parse({ + ...valid, + evaluatorAudit: { + ...valid.evaluatorAudit!, + protocol: { ...valid.evaluatorAudit!.protocol, requiredFaults: ["wrong_answer", "data_leakage"] }, + }, + }) + await expect(HarnessAdapter.bind(faults)).rejects.toThrow("unsupported claims") + const config = protocol() + expect(() => + HarnessContract.ScientificSynthesis.parse({ + ...config, + judges: { + ...config.judges, + precision: { + ...config.decomposer, + name: "relabeled-decomposer", + configSHA256: hash("different-config"), + }, + }, + }), + ).toThrow("distinct prompt commitments") + expect( + HarnessContract.ScientificSynthesis.parse({ + ...config, + minPrecision: 0.2, + minRecall: 0.2, + minF1: 0.9, + }).minF1, + ).toBe(0.9) + }) + + test("derives clean-room decisions and exact factual precision, recall, and F1", async () => { + const contract = await HarnessAdapter.bind(task("synthesis-pass")) + const qualification = await qualify(contract) + const policy = HarnessSynthesis.prompt(contract) + expect(policy).toContain("2026-01-01") + expect(policy).toContain("precision>=0.4, recall>=0.5, F1>=0.45") + expect(policy).not.toContain(contract.synthesis!.referenceSHA256) + expect(policy).not.toContain(contract.synthesis!.referenceFactsSHA256) + const receipt = await HarnessSynthesis.record(submit(contract, qualification), contract) + expect(receipt).toMatchObject({ + status: "passed", + metrics: { + toolEvents: 2, + allowedSources: 1, + blockedSources: 1, + generatedFacts: 3, + supported: 2, + contradicted: 1, + referenceFacts: 2, + covered: 1, + missed: 1, + precisionJudgeErrors: 0, + recallJudgeErrors: 0, + }, + }) + expect(receipt.metrics.violations.post_cutoff).toBe(1) + expect(receipt.metrics.precision).toBeCloseTo(4 / 9, 12) + expect(receipt.metrics.recall).toBe(0.5) + expect(receipt.metrics.f1).toBeCloseTo(8 / 17, 12) + expect(JSON.stringify(receipt)).not.toContain(evaluator) + expect(JSON.stringify(receipt)).not.toContain(auditor) + + await expect( + HarnessAdapter.ingest(evaluation(contract, qualification, undefined, receipt.metrics.f1)), + ).rejects.toThrow("scientific synthesis receipt") + await expect( + HarnessAdapter.ingest(evaluation(contract, qualification, receipt, receipt.metrics.f1! + 0.01)), + ).rejects.toThrow("backend-derived factual F1") + const result = await HarnessAdapter.ingest(evaluation(contract, qualification, receipt)) + expect(result.evaluation).toMatchObject({ + score: receipt.metrics.f1, + synthesisReceiptID: receipt.receiptID, + }) + }) + + test("rejects hidden-fact substitution, trace laundering, and receipt cherry-picking", async () => { + const contract = await HarnessAdapter.bind(task("synthesis-tamper")) + const qualification = await qualify(contract) + await expect( + HarnessSynthesis.record( + submit(contract, qualification, { subject: { type: "candidate", id: "not-created" } }), + contract, + ), + ).rejects.toThrow("candidate does not exist") + await expect( + HarnessSynthesis.record(submit(contract, qualification, { evaluatedAt: Date.now() + 60_000 }), contract), + ).rejects.toThrow("subject interval") + const changed = reference() + changed[0] = { ...changed[0]!, commitment: hash("substituted-hidden-fact") } + await expect( + HarnessSynthesis.record(submit(contract, qualification, { referenceFacts: changed }), contract), + ).rejects.toThrow("hidden manifest commitment") + + const laundered: HarnessSynthesis.Submit["trace"]["events"] = trace() + laundered[1] = { ...laundered[1]!, decision: "allowed" } + await expect( + HarnessSynthesis.record( + submit(contract, qualification, { + trace: { ...submit(contract, qualification).trace, events: laundered }, + }), + contract, + ), + ).rejects.toThrow("backend-derived clean-room decision") + + const receipt = await HarnessSynthesis.record(submit(contract, qualification), contract) + await expect( + HarnessSynthesis.record( + submit(contract, qualification, { conclusionSHA256: hash("different-conclusion") }), + contract, + ), + ).rejects.toThrow("canonical receipt") + }) + + test("keeps judge failures inconclusive instead of scoring them as unsupported", async () => { + const contract = await HarnessAdapter.bind(task("synthesis-judge-error")) + const qualification = await qualify(contract) + const facts = generated() + facts[1] = { ...facts[1]!, verdict: "judge_error", evidence: ["judge://provider-error"] } + const receipt = await HarnessSynthesis.record(submit(contract, qualification, { generatedFacts: facts }), contract) + expect(receipt.status).toBe("inconclusive") + expect(receipt.metrics.precisionJudgeErrors).toBe(1) + expect(receipt.metrics.precision).toBeUndefined() + expect(receipt.metrics.f1).toBeUndefined() + expect(receipt.metrics.unsupported).toBe(0) + await expect(HarnessAdapter.ingest(evaluation(contract, qualification, receipt, 0))).rejects.toThrow( + "passing scientific synthesis receipt", + ) + }) + + test("exposes canonical receipts only through the evaluator capability and fails closed on disk tampering", async () => { + const contract = await HarnessAdapter.bind(task("synthesis-route")) + const qualification = await qualify(contract) + const app = HarnessRoutes() + const response = await app.request("/syntheses/receipts", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(submit(contract, qualification)), + }) + expect(response.status).toBe(200) + const receipt = HarnessSynthesis.Receipt.parse(await response.json()) + const read = await app.request(`/syntheses/receipts/${receipt.receiptID}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sessionID: contract.sessionID, evaluatorToken: evaluator }), + }) + expect(read.status).toBe(200) + expect(await read.json()).toMatchObject({ receiptID: receipt.receiptID }) + + const target = path.join(Global.Path.data, "harness", "syntheses", "receipts", `${receipt.receiptID}.json`) + await Bun.write(target, JSON.stringify({ ...receipt, status: "failed" })) + expect(await HarnessSynthesis.readReceipt(receipt.receiptID)).toBeNull() + + const forged = structuredClone(receipt) as HarnessSynthesis.Receipt + forged.metrics.supported += 1 + const stable = structuredClone(forged) as Record + delete stable.receiptID + delete stable.recordedAt + forged.receiptID = digest(stable) + const receiptFile = path.join(Global.Path.data, "harness", "syntheses", "receipts", `${forged.receiptID}.json`) + const subjectFile = path.join( + Global.Path.data, + "harness", + "syntheses", + "subjects", + encodeURIComponent(contract.sessionID), + `${encodeURIComponent(`run:${contract.runID}`)}.json`, + ) + await Promise.all([Bun.write(receiptFile, JSON.stringify(forged)), Bun.write(subjectFile, JSON.stringify(forged))]) + await expect(HarnessSynthesis.read(forged.receiptID, contract)).rejects.toThrow( + "backend-derived factuality metrics", + ) + }) +}) diff --git a/backend/cli/test/session/harness-world.test.ts b/backend/cli/test/session/harness-world.test.ts new file mode 100644 index 00000000..e0e2857f --- /dev/null +++ b/backend/cli/test/session/harness-world.test.ts @@ -0,0 +1,262 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessWorld } from "../../src/session/harness/world" + +const sessions = new Set() + +afterEach(async () => { + await Promise.all( + [...sessions].flatMap((sessionID) => + ["contracts", "worlds"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { + force: true, + }), + ), + ), + ) + sessions.clear() +}) + +async function bind(sessionID: string) { + sessions.add(sessionID) + return HarnessContract.bind({ + schemaVersion: 1, + runID: `run-${sessionID}`, + sessionID, + objective: "Discover a testable mechanism and preserve calibrated uncertainty", + benchmark: { + name: "local-discovery-suite", + title: "Local discovery evaluation", + family: "custom", + task: "Generate and verify a novel scientific hypothesis", + version: "1", + taskID: "case-1", + split: "validation", + evaluator: "local-evaluator", + evaluatorVersion: "1", + evaluatorSource: "external", + metric: "score", + direction: "maximize", + }, + profile: "optimize", + search: HarnessContract.adaptiveSearch, + packs: [], + model: { provider: "test", name: "model" }, + tools: [], + skills: [], + budget: { steps: 20, candidates: 4 }, + seed: 11, + intervention: "autonomous", + contamination: { policy: "External evaluator state is unavailable", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +describe("continual harness world model", () => { + test("anchors state to the immutable contract and preserves analysis context", async () => { + const contract = await bind("world-analysis") + const initial = await HarnessWorld.read(contract.sessionID) + expect(initial).toMatchObject({ revision: 0, contextEpoch: 0, entries: {} }) + + const analysis = await HarnessWorld.event({ + sessionID: contract.sessionID, + type: "analysis", + summary: "Compared two candidate mechanisms", + changed: false, + }) + expect(analysis.contextEpoch).toBe(0) + expect(analysis.eventsSinceRefine).toBe(1) + + await expect( + HarnessWorld.event({ + sessionID: contract.sessionID, + type: "analysis", + summary: "Invalid analysis mutation", + changed: true, + }), + ).rejects.toThrow("cannot advance the context epoch") + }) + + test("recommends reset-free refinement at failures, milestones, stagnation, and periodic boundaries", async () => { + const contract = await bind("world-events") + const failed = await HarnessWorld.event({ + sessionID: contract.sessionID, + type: "failure", + summary: "Independent check contradicted the mechanism", + evidenceRefs: ["local:failure.json"], + changed: true, + }) + expect(failed).toMatchObject({ + revision: 1, + contextEpoch: 1, + refinement: { recommended: true, trigger: "failure" }, + }) + const followup = await HarnessWorld.event({ + sessionID: contract.sessionID, + type: "tool", + summary: "Collected a diagnostic after the failure", + changed: true, + }) + expect(followup.refinement).toEqual({ recommended: true, trigger: "failure" }) + + const periodicID = "world-periodic" + const periodic = await bind(periodicID) + for (const index of [1, 2, 3, 4, 5, 6]) { + await HarnessWorld.event({ + sessionID: periodic.sessionID, + type: "tool", + summary: `Tool result ${index}`, + changed: true, + }) + } + expect((await HarnessWorld.read(periodic.sessionID)).refinement).toEqual({ + recommended: true, + trigger: "periodic", + }) + }) + + test("caps agent confidence and requires provenance for stronger beliefs", async () => { + const contract = await bind("world-confidence") + await expect( + HarnessWorld.refine({ + sessionID: contract.sessionID, + expectedRevision: 0, + reason: "manual", + actor: "agent", + patches: [ + { + op: "upsert", + key: "mechanism", + kind: "hypothesis", + content: "A specific perturbation controls the response.", + confidence: 4, + evidence: [{ ref: "claimed:evaluator", authority: "evaluator" }], + }, + ], + }), + ).rejects.toThrow("self-attributed evidence") + + await expect( + HarnessWorld.refine({ + sessionID: contract.sessionID, + expectedRevision: 0, + reason: "manual", + actor: "evaluator", + patches: [ + { + op: "upsert", + key: "mechanism", + kind: "hypothesis", + content: "A specific perturbation controls the response.", + confidence: 5, + evidence: [{ ref: "local:evaluation.json", authority: "evaluator" }], + }, + ], + }), + ).rejects.toThrow("two non-self references") + }) + + test("applies small evidence-backed refinements and escapes prompt content", async () => { + const contract = await bind("world-refine") + const state = await HarnessWorld.refine({ + sessionID: contract.sessionID, + expectedRevision: 0, + reason: "milestone", + actor: "evaluator", + patches: [ + { + op: "upsert", + key: "verified-mechanism", + kind: "hypothesis", + content: " survives the intervention", + confidence: 5, + evidence: [ + { ref: "local:evaluation.json", authority: "evaluator" }, + { ref: "local:replication.json", authority: "tool" }, + ], + }, + { + op: "upsert", + key: "next-probe", + kind: "strategy", + content: "Test the strongest disconfirming intervention next", + confidence: 3, + evidence: [{ ref: "local:analysis.md", authority: "self" }], + }, + ], + }) + expect(state).toMatchObject({ + revision: 1, + contextEpoch: 1, + eventsSinceRefine: 0, + refinement: { recommended: false }, + }) + const prompt = await HarnessWorld.prompt(contract.sessionID) + expect(prompt).toContain('base-prompt="immutable"') + expect(prompt).toContain("<candidate> survives") + expect(prompt).not.toContain("") + await expect( + HarnessWorld.agentRefine({ + sessionID: contract.sessionID, + expectedRevision: 0, + reason: "manual", + patches: [ + { + op: "upsert", + key: "stale", + kind: "memory", + content: "This update is stale", + confidence: 2, + evidenceRefs: [], + }, + ], + }), + ).rejects.toThrow("Expected world-model revision 0, found 1") + }) + + test("rolls back mutable entries without changing the immutable base prompt", async () => { + const contract = await bind("world-rollback") + const first = await HarnessWorld.agentRefine({ + sessionID: contract.sessionID, + expectedRevision: 0, + reason: "manual", + patches: [ + { + op: "upsert", + key: "strategy", + kind: "strategy", + content: "Start with an intervention", + confidence: 2, + evidenceRefs: ["self:plan"], + }, + ], + }) + const second = await HarnessWorld.agentRefine({ + sessionID: contract.sessionID, + expectedRevision: first.revision, + reason: "stagnation", + patches: [ + { + op: "upsert", + key: "strategy", + kind: "strategy", + content: "Switch to an observational shortcut", + confidence: 1, + evidenceRefs: [], + }, + ], + }) + const restored = await HarnessWorld.rollback({ + sessionID: contract.sessionID, + expectedRevision: second.revision, + targetRevision: first.revision, + }) + expect(restored.entries.strategy?.content).toBe("Start with an intervention") + expect(restored.basePromptSHA256).toBe(first.basePromptSHA256) + expect(restored.revision).toBe(3) + expect(restored.contextEpoch).toBe(3) + }) +}) diff --git a/backend/cli/test/session/rsi-verification.test.ts b/backend/cli/test/session/rsi-verification.test.ts new file mode 100644 index 00000000..147e6944 --- /dev/null +++ b/backend/cli/test/session/rsi-verification.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, test } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { Global } from "../../src/global" +import { RSICritic } from "../../src/session/rsi/critic" +import { RSIDistill } from "../../src/session/rsi/distill" +import type { RSITrajectory } from "../../src/session/rsi/trajectory" + +const proposals = new Set() + +afterEach(async () => { + await Promise.all( + [...proposals].map((name) => + fs.rm(path.join(Global.Path.data, "learned-skill-proposals", name), { recursive: true, force: true }), + ), + ) + proposals.clear() +}) + +function trajectory(): RSITrajectory.Trajectory { + return { + sessionId: "session-12345678", + timestamp: Date.now(), + agent: "research", + hypothesis: "A verified workflow improves a benchmark metric on held-out tasks.", + steps: [ + { tool: "read", inputSummary: "data", outputSummary: "schema" }, + { tool: "bash", inputSummary: "baseline", outputSummary: "0.5" }, + { tool: "edit", inputSummary: "candidate", outputSummary: "saved" }, + { tool: "bash", inputSummary: "evaluate", outputSummary: "0.8" }, + ], + reportedOutcome: "success", + outcome: "unverified", + tokenCost: 1000, + } +} + +describe("RSI verification boundary", () => { + test("assigns no correctness credit to self-reported completion", () => { + const score = RSICritic.evaluate(trajectory()) + expect(score.correctness).toBe(0) + expect(score.notes).toContain("verification=unverified") + }) + + test("drafts only externally verified skill proposals", async () => { + const unverified = trajectory() + unverified.score = 100 + expect(await RSIDistill.propose(unverified)).toBeNull() + + const verified = trajectory() + verified.outcome = "success" + verified.verification = { + runID: "run-1", + evaluator: "official-evaluator", + status: "passed", + score: 0.8, + evaluatedAt: Date.now(), + } + verified.score = RSICritic.evaluate(verified).total + const name = await RSIDistill.propose(verified) + expect(name).not.toBeNull() + proposals.add(name!) + const skill = await Bun.file(path.join(Global.Path.data, "learned-skill-proposals", name!, "SKILL.md")).text() + expect(skill).toContain("source: rsi-proposal") + expect(skill).toContain("status: pending") + expect(skill).toContain("not active until held-out evaluation") + }) +}) diff --git a/backend/cli/test/session/trace.test.ts b/backend/cli/test/session/trace.test.ts index 99ca5112..3404a57c 100644 --- a/backend/cli/test/session/trace.test.ts +++ b/backend/cli/test/session/trace.test.ts @@ -205,8 +205,19 @@ test("builds one local observable harness trace without reasoning or copied outp message: "provider overloaded", delayMs: 50, }) + await SessionTraceStore.recordProfile({ + sessionID: session.id, + messageID: user.id, + id: "reproduce", + source: "heuristic", + confidence: 0.94, + reasons: ["reproduction-language"], + }) const trace = await SessionTrace.build(session.id) + expect(trace.profiles).toEqual([ + expect.objectContaining({ messageID: user.id, id: "reproduce", source: "heuristic", confidence: 0.94 }), + ]) expect(trace.summary).toMatchObject({ cost: 0.42, toolCalls: 7, @@ -249,7 +260,7 @@ test("builds one local observable harness trace without reasoning or copied outp expect(trace.turns[0].timeToFirstUsefulOutputMs).toBe(100) await Session.remove(session.id) - expect(await SessionTraceStore.read(session.id)).toEqual({ approvals: {}, retries: [] }) + expect(await SessionTraceStore.read(session.id)).toEqual({ approvals: {}, retries: [], profiles: {} }) }, }) }) diff --git a/backend/cli/test/skill/bundled-skills.test.ts b/backend/cli/test/skill/bundled-skills.test.ts index 1f76e7e6..825f85d6 100644 --- a/backend/cli/test/skill/bundled-skills.test.ts +++ b/backend/cli/test/skill/bundled-skills.test.ts @@ -8,7 +8,7 @@ const root = path.join(import.meta.dir, "..", "..", "skills") const files = await Array.fromAsync(new Bun.Glob("**/SKILL.md").scan({ cwd: root, absolute: true })) test("every bundled skill with frontmatter parses and validates", async () => { - expect(files.length).toBe(293) + expect(files.length).toBe(311) const broken = await Promise.all( files.map(async (file) => { const raw = await Bun.file(file).text() diff --git a/backend/cli/test/skill/harness-native-skills.test.ts b/backend/cli/test/skill/harness-native-skills.test.ts new file mode 100644 index 00000000..614a1ae7 --- /dev/null +++ b/backend/cli/test/skill/harness-native-skills.test.ts @@ -0,0 +1,1671 @@ +import { expect, test } from "bun:test" +import fs from "fs/promises" +import os from "os" +import path from "path" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessAutonomy } from "../../src/session/harness/autonomy" +import { HarnessBlueprint } from "../../src/session/harness/blueprint" +import { HarnessEvolution } from "../../src/session/harness/evolution" +import { HarnessFormal } from "../../src/session/harness/formal" + +const skills = path.resolve(import.meta.dir, "../../skills") + +const hash = (value: string | Uint8Array) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const canon = (value: unknown): string => { + if (Array.isArray(value)) return `[${value.map(canon).join(",")}]` + if (value && typeof value === "object") { + const record = value as Record + return `{${Object.keys(record) + .toSorted() + .map((key) => `${JSON.stringify(key)}:${canon(record[key])}`) + .join(",")}}` + } + return JSON.stringify(value) +} +const digest = (value: unknown) => hash(canon(value)) +const fileHash = async (file: string) => hash(new Uint8Array(await Bun.file(file).arrayBuffer())) + +async function run(script: string, args: string[]) { + const process = Bun.spawn(["python", path.join(skills, script), ...args], { + stdout: "pipe", + stderr: "pipe", + }) + const [code, stdout, stderr] = await Promise.all([ + process.exited, + new Response(process.stdout).text(), + new Response(process.stderr).text(), + ]) + return { code, stdout, stderr } +} + +async function bun(script: string, args: string[]) { + const process = Bun.spawn(["bun", path.join(skills, script), ...args], { + stdout: "pipe", + stderr: "pipe", + }) + const [code, stdout, stderr] = await Promise.all([ + process.exited, + new Response(process.stdout).text(), + new Response(process.stderr).text(), + ]) + return { code, stdout, stderr } +} + +test("active-failure-audit builds an opaque committed probe manifest", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-audit-")) + const source = path.join(dir, "private.jsonl") + const output = path.join(dir, "manifest.json") + await Bun.write( + source, + [ + JSON.stringify({ + id: "hard-1", + hidden: { prompt: "PRIVATE_ALPHA", target: "answer-a" }, + features: [0.1, 0.9], + stratum: "rare", + }), + JSON.stringify({ + id: "hard-2", + hidden: { prompt: "PRIVATE_BETA", target: "answer-b" }, + features: [0.8, 0.2], + stratum: "common", + weight: 2, + priorLoss: 0.7, + }), + ].join("\n"), + ) + + try { + const result = await run("research/active-failure-audit/scripts/build_probe_manifest.py", [source, output]) + expect(result.code).toBe(0) + const manifest = await Bun.file(output).text() + expect(manifest).not.toContain("PRIVATE_ALPHA") + expect(manifest).not.toContain("PRIVATE_BETA") + const parsed = JSON.parse(manifest) + expect(parsed.probes).toHaveLength(2) + expect(parsed.probes[0].commitment).toMatch(/^[0-9a-f]{64}$/) + expect(parsed.manifestSHA256).toMatch(/^[0-9a-f]{64}$/) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("active-failure-audit rejects duplicate hidden probes", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-audit-")) + const source = path.join(dir, "private.jsonl") + const output = path.join(dir, "manifest.json") + await Bun.write( + source, + [ + JSON.stringify({ id: "a", hidden: { value: 1 }, features: [0], stratum: "x" }), + JSON.stringify({ id: "b", hidden: { value: 1 }, features: [1], stratum: "y" }), + ].join("\n"), + ) + + try { + const result = await run("research/active-failure-audit/scripts/build_probe_manifest.py", [source, output]) + expect(result.code).toBe(2) + expect(result.stderr).toContain("hidden probe commitments must be unique") + expect(await Bun.file(output).exists()).toBe(false) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("verify-benchmark-integrity derives observable violations from an evaluator-owned trace", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-integrity-")) + const trace = path.join(dir, "trace.jsonl") + const contract = path.join(dir, "contract.json") + const subject = path.join(dir, "subject.json") + const model = path.join(dir, "model.json") + const audits = path.join(dir, "audits.json") + const output = path.join(dir, "submission.json") + const report = path.join(dir, "report.json") + const canaries = hash("canaries") + const commitments = await run("research/verify-benchmark-integrity/scripts/verify_integrity.py", ["commitments"]) + expect(commitments.code).toBe(0) + const pins = JSON.parse(commitments.stdout) + const kinds = ["test_item_contamination", "external_model_use", "benchmark_lookup"] + const identities = kinds.map((kind) => ({ + kind, + name: `${kind}-auditor`, + version: "1", + promptSHA256: hash(`${kind}-prompt`), + })) + await Promise.all([ + Bun.write( + contract, + JSON.stringify({ + protocolVersion: "benchmark-integrity-v1", + validatorSHA256: pins.validatorSHA256, + traceSchemaSHA256: pins.traceSchemaSHA256, + minEvents: 4, + minCoverage: 0.9, + assignedModel: { + name: "assigned-model", + baseArtifactSHA256: hash("base"), + configSHA256: hash("config"), + }, + forbiddenModelArtifacts: [hash("forbidden")], + policy: { + testItemDerivation: "forbidden", + unapprovedExternalModels: "forbidden", + benchmarkLookup: "forbidden", + }, + auditors: identities, + hiddenCanaryManifestSHA256: canaries, + minHiddenCanaries: 2, + }), + ), + Bun.write( + trace, + [ + { seq: 0, at: 1_000, kind: "command" }, + { seq: 1, at: 1_001, kind: "model_call", approved: true }, + { seq: 2, at: 1_002, kind: "model_call", approved: false }, + { seq: 3, at: 1_003, kind: "benchmark_lookup" }, + { seq: 4, at: 1_004, kind: "hidden_canary", manifestSHA256: canaries, canaryID: "a", violation: false }, + { seq: 5, at: 1_005, kind: "hidden_canary", manifestSHA256: canaries, canaryID: "b", violation: true }, + { seq: 6, at: 1_006, kind: "trace_gap", dropped: 1 }, + ] + .map((item) => JSON.stringify(item)) + .join("\n"), + ), + Bun.write( + subject, + JSON.stringify({ type: "run", id: "run-1", artifact: { uri: "artifact:output", sha256: hash("output") } }), + ), + Bun.write( + model, + JSON.stringify({ + name: "assigned-model", + baseArtifactSHA256: hash("base"), + configSHA256: hash("config"), + outputArtifactSHA256: hash("fine-tuned"), + lineageVerified: true, + }), + ), + Bun.write( + audits, + JSON.stringify( + identities.map((identity) => ({ + ...identity, + decision: "clean", + confidence: 0.99, + evidence: [`artifact:${identity.kind}.json`], + })), + ), + ), + ]) + + try { + const result = await run("research/verify-benchmark-integrity/scripts/verify_integrity.py", [ + "build", + "--contract", + contract, + "--trace", + trace, + "--subject", + subject, + "--model", + model, + "--audits", + audits, + "--run-id", + "run-1", + "--session-id", + "session-1", + "--evaluated-at", + "1100", + "--output", + output, + "--report", + report, + ]) + expect(result.code).toBe(0) + const submission = JSON.parse(await Bun.file(output).text()) + expect(submission.evaluatorToken).toBeUndefined() + expect(submission.trace).toMatchObject({ events: 7, dropped: 1, schemaSHA256: pins.traceSchemaSHA256 }) + expect(submission.activity).toMatchObject({ + unapprovedExternalModelCalls: 1, + benchmarkLookupEvents: 1, + hiddenCanariesTested: 2, + hiddenCanaryViolations: 1, + }) + expect(JSON.parse(await Bun.file(report).text()).traceCoverage).toBe(0.875) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("verify-benchmark-integrity rejects a non-contiguous trace", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-integrity-gap-")) + const trace = path.join(dir, "trace.jsonl") + await Bun.write( + trace, + [ + { seq: 0, at: 1, kind: "command" }, + { seq: 2, at: 2, kind: "command" }, + ] + .map((item) => JSON.stringify(item)) + .join("\n"), + ) + try { + const result = await run("research/verify-benchmark-integrity/scripts/verify_integrity.py", [ + "check-trace", + "--trace", + trace, + "--canary-manifest", + hash("canaries"), + ]) + expect(result.code).toBe(2) + expect(result.stderr).toContain("trace sequence must be contiguous") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("trace-evolutionary-candidate captures exact snapshots and deterministic parent deltas", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-evolution-")) + const parentRoot = path.join(dir, "parent") + const childRoot = path.join(dir, "child") + const parentArtifacts = path.join(dir, "parent-artifacts") + const childArtifacts = path.join(dir, "child-artifacts") + const contract = path.join(dir, "contract.json") + const parentSubject = path.join(dir, "parent-subject.json") + const childSubject = path.join(dir, "child-subject.json") + const parentOutput = path.join(dir, "parent-submission.json") + const childOutput = path.join(dir, "child-submission.json") + const parentSpec = path.join(dir, "parent.json") + const report = path.join(dir, "child-report.json") + await Promise.all([ + fs.mkdir(path.join(parentRoot, "src"), { recursive: true }), + fs.mkdir(path.join(childRoot, "src"), { recursive: true }), + ]) + await Promise.all([ + Bun.write(path.join(parentRoot, "src", "main.py"), "alpha\nold\n"), + Bun.write(path.join(childRoot, "src", "main.py"), "alpha\nnew\n"), + ]) + const commitments = await run("research/trace-evolutionary-candidate/scripts/trace_candidate.py", ["commitments"]) + expect(commitments.code).toBe(0) + const pins = JSON.parse(commitments.stdout) + const protocol = HarnessContract.Evolution.parse({ + protocolVersion: "evolution-trace-v1", + validatorSHA256: pins.validatorSHA256, + manifestSchemaSHA256: pins.manifestSchemaSHA256, + lineAlgorithm: "sha256-exact-line-v1", + roots: ["src"], + extensions: [".py"], + exclude: [], + maxFiles: 10, + maxFileBytes: 10_000, + maxTotalBytes: 100_000, + maxSourceLines: 1_000, + maxChangedLines: 100, + }) + await Promise.all([ + Bun.write(contract, JSON.stringify(protocol)), + Bun.write( + parentSubject, + JSON.stringify({ + type: "candidate", + id: hash("parent-id"), + artifact: { uri: "artifact:parent.tar", sha256: hash("parent-artifact") }, + }), + ), + Bun.write( + childSubject, + JSON.stringify({ + type: "candidate", + id: hash("child-id"), + artifact: { uri: "artifact:child.tar", sha256: hash("child-artifact") }, + }), + ), + ]) + + try { + const root = await run("research/trace-evolutionary-candidate/scripts/trace_candidate.py", [ + "build", + "--contract", + contract, + "--subject", + parentSubject, + "--candidate-root", + parentRoot, + "--artifact-dir", + parentArtifacts, + "--run-id", + "run-1", + "--session-id", + "session-1", + "--evaluated-at", + "1000", + "--output", + parentOutput, + ]) + expect(root.code).toBe(0) + const parent = JSON.parse(await Bun.file(parentOutput).text()) + expect(parent.parents).toEqual([]) + expect(parent.snapshot.files).toEqual([ + expect.objectContaining({ + path: "src/main.py", + lineHashes: [hash("alpha"), hash("old")], + }), + ]) + expect(parent.snapshot.artifact.sha256).toBe(await fileHash(parent.snapshot.artifact.uri)) + expect(parent.snapshot.artifact.sha256).toBe(HarnessEvolution.manifestSHA256(protocol, parent.snapshot.files)) + await Bun.write( + parentSpec, + JSON.stringify({ + id: parent.subject.id, + artifact: parent.subject.artifact, + receiptID: hash("parent-receipt"), + snapshot: parent.snapshot.artifact, + root: parentRoot, + }), + ) + const child = await run("research/trace-evolutionary-candidate/scripts/trace_candidate.py", [ + "build", + "--contract", + contract, + "--subject", + childSubject, + "--candidate-root", + childRoot, + "--parent", + parentSpec, + "--artifact-dir", + childArtifacts, + "--run-id", + "run-1", + "--session-id", + "session-1", + "--evaluated-at", + "1100", + "--output", + childOutput, + "--report", + report, + ]) + expect(child.code).toBe(0) + const submission = JSON.parse(await Bun.file(childOutput).text()) + expect(submission.evaluatorToken).toBeUndefined() + expect(submission.parents).toHaveLength(1) + const delta = JSON.parse(await Bun.file(submission.parents[0].delta.uri).text()) + expect(delta).toMatchObject({ + parent: { id: parent.subject.id, snapshotSHA256: parent.snapshot.artifact.sha256 }, + candidate: { id: submission.subject.id, snapshotSHA256: submission.snapshot.artifact.sha256 }, + addedLineHashes: [hash("new")], + deletedLineHashes: [hash("old")], + }) + expect(submission.parents[0].delta.sha256).toBe(digest(delta)) + expect(submission.parents[0].delta.sha256).toBe( + HarnessEvolution.deltaSHA256({ + subject: submission.subject, + snapshot: submission.snapshot, + parent: { subject: parent.subject, snapshot: parent.snapshot }, + }), + ) + expect(JSON.parse(await Bun.file(report).text())).toMatchObject({ + files: 1, + sourceLines: 2, + parents: [{ filesChanged: 1, addedLines: 1, deletedLines: 1 }], + }) + + await Bun.write(path.join(parentRoot, "src", "main.py"), "substituted\n") + const rejected = await run("research/trace-evolutionary-candidate/scripts/trace_candidate.py", [ + "build", + "--contract", + contract, + "--subject", + childSubject, + "--candidate-root", + childRoot, + "--parent", + parentSpec, + "--artifact-dir", + path.join(dir, "rejected-artifacts"), + "--run-id", + "run-1", + "--session-id", + "session-1", + ]) + expect(rejected.code).toBe(2) + expect(rejected.stderr).toContain("does not match its immutable snapshot") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("operate-adaptive-search rejects inconsistent controller decisions", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-adaptive-lease-")) + const script = "research/operate-adaptive-search/scripts/validate_lease.py" + const source = path.join(dir, "lease.json") + const lease = { + id: hash("lease"), + revision: 7, + strategy: "exploit", + mode: "diff", + parentIDs: [hash("parent")], + inspirationIDs: [], + targetIsland: 1, + contextIDs: [hash("parent")], + reasons: ["adaptive-exploitation"], + control: { + protocolVersion: "adaptive-search-v1", + policySHA256: hash(JSON.stringify(HarnessContract.adaptiveSearch)), + eventCount: 6, + stalled: 0, + selectedIsland: 1, + targetIsland: 1, + visits: 3, + accumulatedImprovement: 0.04, + rewardMean: 0.02, + intensity: 0.3, + draw: 0.8, + explore: false, + globalStagnation: false, + }, + } + try { + await Bun.write(source, JSON.stringify(lease)) + const valid = await run(script, [source]) + expect(valid.code).toBe(0) + expect(JSON.parse(valid.stdout)).toMatchObject({ + valid: true, + strategy: "exploit", + targetIsland: 1, + eventCount: 6, + explore: false, + }) + await Bun.write(source, JSON.stringify({ ...lease, control: { ...lease.control, explore: true } })) + const rejected = await run(script, [source]) + expect(rejected.code).toBe(1) + expect(rejected.stderr).toContain("deterministic intensity draw") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("run-verifier-routed-research enforces clean restart context isolation", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-verifier-unit-")) + const script = "research/run-verifier-routed-research/scripts/validate_unit.py" + const source = path.join(dir, "work.json") + const review = (id: string) => ({ id: hash(id), role: "verification", status: "completed" }) + const work = { + id: hash("clean-restart"), + status: "pending", + role: "generation", + label: "clean-restart-2", + prompt: 'role="generation" topology="verifier_loop" return one complete candidate artifact', + context: [review("review-a"), review("review-b")], + allocation: { steps: 1, tokens: 1000, costUSD: 0.01, wallTimeMs: 1000 }, + } + try { + await Bun.write(source, JSON.stringify(work)) + const valid = await run(script, [source]) + expect(valid.code).toBe(0) + expect(JSON.parse(valid.stdout)).toMatchObject({ + valid: true, + role: "generation", + label: "clean-restart-2", + }) + + await Bun.write( + source, + JSON.stringify({ + ...work, + context: [{ id: hash("rejected-candidate"), role: "generation", status: "completed" }, ...work.context], + }), + ) + const rejected = await run(script, [source]) + expect(rejected.code).toBe(1) + expect(rejected.stderr).toContain("clean restart may receive verifier summaries only") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("audit-scientific-meaning derives status without persisting review capabilities", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-semantic-audit-")) + const script = "research/audit-scientific-meaning/scripts/validate_submission.py" + const contractFile = path.join(dir, "contract.json") + const submissionFile = path.join(dir, "submission.json") + const objective = "Resolve the intended open problem without a vacuous interpretation" + const scope = { + objectiveSHA256: hash(objective), + criteria: [{ id: "target", requirement: "Address the intended target." }], + forbiddenShortcuts: [{ id: "vacuity", description: "Do not use a trivial interpretation." }], + literature: { cutoff: "2026-08-01", corpusSHA256: hash("semantic-corpus") }, + noveltyFloor: "minor", + } + const contract = { + sessionID: "semantic-session", + runID: "semantic-run", + objective, + semanticAudit: { + protocolVersion: "semantic-audit-v1", + reviewer: { name: "expert-panel", version: "1", source: "external" }, + scope, + minReviewers: 2, + minConfidence: 0.8, + }, + } + const review = (actor: string, sessionID: string) => ({ + actor, + sessionID, + correctness: "passed", + alignment: "intended", + novelty: "minor", + vacuous: false, + confidence: 0.9, + criteria: [{ id: "target", status: "passed", evidence: [`artifact:${actor}-target`] }], + shortcuts: [{ id: "vacuity", observed: false, evidence: [`artifact:${actor}-vacuity`] }], + literatureRefs: [`literature:${actor}`], + evidence: [`artifact:${actor}-review`], + summary: "Independent substantive review", + reviewedAt: Date.now(), + }) + const submission = { + sessionID: contract.sessionID, + subject: { type: "run", id: contract.runID }, + reviews: [review("reviewer-a", "review-session-a"), review("reviewer-b", "review-session-b")], + } + try { + await Promise.all([ + Bun.write(contractFile, JSON.stringify(contract)), + Bun.write(submissionFile, JSON.stringify(submission)), + ]) + const valid = await run(script, [contractFile, submissionFile]) + expect(valid.code).toBe(0) + expect(JSON.parse(valid.stdout)).toMatchObject({ + valid: true, + derivedStatus: "meaningful", + reviewers: 2, + subject: submission.subject, + }) + + await Bun.write(submissionFile, JSON.stringify({ ...submission, reviewerToken: "must-not-touch-disk" })) + const rejected = await run(script, [contractFile, submissionFile]) + expect(rejected.code).toBe(1) + expect(rejected.stderr).toContain("token-free on disk") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("run-replicated-evaluation preflights the exact frozen independent-unit grid", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-replicated-evaluation-")) + const script = "research/run-replicated-evaluation/scripts/preflight.py" + const contractFile = path.join(dir, "contract.json") + const observationsFile = path.join(dir, "observations.json") + const contract = { + sessionID: "replicated-session", + runID: "replicated-run", + replication: { + protocolVersion: "replicated-evaluation-v1", + environmentSHA256: hash("locked-environment"), + sampling: { + strata: [{ id: "task-a", commitmentSHA256: hash("task-a") }], + clusters: ["seed-a", "seed-b", "seed-c", "seed-d", "seed-e"].map((id) => ({ + id, + commitmentSHA256: hash(id), + })), + }, + estimator: "iqm", + }, + } + const observations = contract.replication.sampling.clusters.map((cluster, index) => ({ + stratumID: "task-a", + clusterID: cluster.id, + stratumSHA256: contract.replication.sampling.strata[0]!.commitmentSHA256, + clusterSHA256: cluster.commitmentSHA256, + status: "passed", + score: index + 1, + outputSHA256: hash(`${cluster.id}:output`), + environmentSHA256: hash("locked-environment"), + evidence: [`artifact:${cluster.id}.json`], + evaluatedAt: Date.now(), + })) + try { + await Promise.all([ + Bun.write(contractFile, JSON.stringify(contract)), + Bun.write( + observationsFile, + JSON.stringify({ + sessionID: contract.sessionID, + subject: { type: "run", id: contract.runID }, + observations, + }), + ), + ]) + const valid = await run(script, [contractFile, observationsFile]) + expect(valid.code).toBe(0) + expect(JSON.parse(valid.stdout)).toEqual({ + valid: true, + units: 5, + strata: 1, + clusters: 5, + estimator: "iqm", + statuses: { passed: 5, failed: 0, inconclusive: 0 }, + }) + + await Bun.write(observationsFile, JSON.stringify({ observations: observations.slice(1) })) + const missing = await run(script, [contractFile, observationsFile]) + expect(missing.code).toBe(1) + expect(missing.stderr).toContain("frozen grid mismatch") + + await Bun.write(observationsFile, JSON.stringify({ evaluatorToken: "must-not-touch-disk", observations })) + const token = await run(script, [contractFile, observationsFile]) + expect(token.code).toBe(1) + expect(token.stderr).toContain("token-free") + + await Bun.write( + observationsFile, + JSON.stringify({ + observations: [ + { ...observations[0], environmentSHA256: hash("drifted-environment") }, + ...observations.slice(1), + ], + }), + ) + const drift = await run(script, [contractFile, observationsFile]) + expect(drift.code).toBe(1) + expect(drift.stderr).toContain("frozen environment") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("run-sealed-confirmation preflights one token-free terminal claim result", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-sealed-confirmation-")) + const script = "research/run-sealed-confirmation/scripts/preflight.py" + const protocolFile = path.join(dir, "protocol.json") + const selectionFile = path.join(dir, "selection.json") + const resultFile = path.join(dir, "result.json") + const outputFile = path.join(dir, "payload.json") + const protocol = { + protocolVersion: "sealed-confirmation-v1", + optimization: { split: "validation", manifestSHA256: hash("optimization-manifest") }, + claim: { + taskID: "official-hidden", + split: "held_out", + manifestSHA256: hash("claim-manifest"), + validatorSHA256: hash("claim-validator"), + environmentSHA256: hash("claim-environment"), + evaluator: { name: "claim-evaluator", version: "2", source: "benchmark" }, + metric: "score", + direction: "maximize", + target: 0.8, + }, + selection: { rule: "terminal-verified-best-v1", subjects: 1 }, + exposure: { policy: "terminal-receipt-only", searchFeedback: false, memoryCapture: false }, + failurePolicy: "fail-closed", + } + const stable = { + schemaVersion: 1, + protocolVersion: "terminal-verified-best-selection-v1", + contractSHA256: hash("contract"), + protocolSHA256: hash(JSON.stringify(protocol)), + sourceSessionID: "confirmation-session", + runID: "confirmation-run", + searchRevision: 4, + stopReason: "objective_met", + candidateID: hash("candidate-id"), + candidateArtifact: { uri: "candidate://winner", sha256: hash("candidate-artifact") }, + candidateCreatedAt: 100, + optimizationResultSHA256: hash("optimization-result"), + optimizationEvaluationSHA256: hash("optimization-evaluation"), + selectedAt: 200, + } + const selection = { ...stable, selectionID: hash(JSON.stringify(stable)) } + const result = { + candidateSHA256: selection.candidateArtifact.sha256, + manifestSHA256: protocol.claim.manifestSHA256, + validatorSHA256: protocol.claim.validatorSHA256, + environmentSHA256: protocol.claim.environmentSHA256, + outcome: "completed", + score: 0.85, + metrics: { score: 0.85 }, + checks: [{ id: "official-gate", status: "passed", blocking: true, evidence: ["claim:gate.json"] }], + evidence: ["claim:result.json"], + outputSHA256: hash("claim-output"), + evaluatedAt: 201, + } + try { + await Promise.all([ + Bun.write(protocolFile, JSON.stringify(protocol)), + Bun.write(selectionFile, JSON.stringify(selection)), + Bun.write(resultFile, JSON.stringify(result)), + ]) + const valid = await run(script, [ + "--protocol", + protocolFile, + "--selection", + selectionFile, + "--result", + resultFile, + "--out", + outputFile, + ]) + expect(valid.code).toBe(0) + expect(JSON.parse(valid.stdout)).toMatchObject({ valid: true, tokenFree: true, derivedTargetReached: true }) + expect(JSON.parse(await Bun.file(outputFile).text())).toEqual({ + schemaVersion: 1, + sessionID: selection.sourceSessionID, + ...result, + }) + + await Bun.write(resultFile, JSON.stringify({ ...result, candidateSHA256: hash("alternate") })) + const changed = await run(script, [ + "--protocol", + protocolFile, + "--selection", + selectionFile, + "--result", + resultFile, + "--out", + outputFile, + ]) + expect(changed.code).toBe(1) + expect(changed.stderr).toContain("candidate substitution") + + await Bun.write(resultFile, JSON.stringify({ ...result, confirmationToken: "must-not-touch-disk" })) + const token = await run(script, [ + "--protocol", + protocolFile, + "--selection", + selectionFile, + "--result", + resultFile, + "--out", + outputFile, + ]) + expect(token.code).toBe(1) + expect(token.stderr).toContain("secret field") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("design-replay-interventions freezes exact one-difference evaluator pairs", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-interventions-")) + const script = "research/design-replay-interventions/scripts/design_interventions.py" + const contract = path.join(dir, "contract.json") + const spec = path.join(dir, "spec.json") + const output = path.join(dir, "initialize.json") + const report = path.join(dir, "targets.json") + const commitments = await run(script, ["commitments"]) + expect(commitments.code).toBe(0) + const validatorSHA256 = JSON.parse(commitments.stdout).validatorSHA256 + const artifact = (name: string) => ({ uri: `artifact:${name}`, sha256: hash(name) }) + const subject = { type: "candidate", id: hash("winner"), artifact: artifact("winner") } + const condition = (seed: number) => ({ + seed, + model: { provider: "test", name: "primary", version: "1" }, + context: artifact("context"), + evaluator: { name: "official", version: "1", source: "benchmark" }, + split: { name: "held_out", manifest: artifact("split") }, + environment: artifact("environment"), + budget: artifact("budget"), + }) + const pairs = [0, 1, 2].flatMap((index) => { + const base = condition(index) + const winner = { artifact: subject.artifact, condition: base } + return [ + { + family: "model_transfer", + index, + control: winner, + arm: { + artifact: subject.artifact, + condition: { ...base, model: { provider: "test", name: "transfer", version: "2" } }, + }, + change: artifact(`model-${index}`), + }, + { family: "replay", index, control: winner, arm: winner, change: artifact(`replay-${index}`) }, + { + family: "retune", + index, + control: { artifact: artifact(`retuned-${index}`), condition: base }, + arm: winner, + change: artifact(`retune-${index}`), + }, + ] + }) + await Promise.all([ + Bun.write( + contract, + JSON.stringify({ + protocolVersion: "intervention-study-v1", + validatorSHA256, + requiredForPromotion: true, + minPairs: 3, + maxPairs: 4, + maxTotalPairs: 12, + confidence: 0.95, + required: ["model_transfer", "replay", "retune"], + rules: [ + { family: "model_transfer", mode: "max_regression", threshold: 0.05 }, + { family: "replay", mode: "max_absolute_effect", threshold: 0.01 }, + { family: "retune", mode: "min_effect", threshold: 0.1 }, + ], + }), + ), + Bun.write( + spec, + JSON.stringify({ + schemaVersion: 1, + runID: "run-1", + sessionID: "session-1", + subject, + evolutionReceiptID: hash("evolution-receipt"), + pairs, + }), + ), + ]) + + try { + const result = await run(script, [ + "build", + "--contract", + contract, + "--spec", + spec, + "--output", + output, + "--report", + report, + ]) + expect(result.code).toBe(0) + const request = JSON.parse(await Bun.file(output).text()) + const targets = JSON.parse(await Bun.file(report).text()) + expect(request).toMatchObject({ + schemaVersion: 1, + subject, + validator: { name: "design-replay-interventions", version: 1, scriptSHA256: validatorSHA256 }, + }) + expect(request.evaluatorToken).toBeUndefined() + expect(request.pairs).toHaveLength(9) + expect(targets).toMatchObject({ + candidateID: subject.id, + families: { model_transfer: 3, replay: 3, retune: 3 }, + }) + expect(targets.targets[0].controlSHA256).toBe(digest(request.pairs[0].control)) + + const invalid = structuredClone(JSON.parse(await Bun.file(spec).text())) + invalid.pairs[0].arm.condition.context = artifact("substituted-context") + await Bun.write(spec, JSON.stringify(invalid)) + const rejected = await run(script, [ + "build", + "--contract", + contract, + "--spec", + spec, + "--output", + output, + "--report", + report, + ]) + expect(rejected.code).toBe(2) + expect(rejected.stderr).toContain("may change only model") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("simulator-validation accepts a convergent invariant-preserving study", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-simulator-")) + const source = path.join(dir, "study.json") + const output = path.join(dir, "report.json") + await Bun.write( + source, + JSON.stringify({ + simulator: { + name: "reference-solver", + version: "1.0.0", + command: "reference-solver case.json", + configSHA256: "0".repeat(64), + }, + expectedOrder: 2, + orderTolerance: 0.1, + maxResidual: 1e-8, + invariantTolerances: { mass_drift: 1e-6 }, + levels: [ + { label: "coarse", h: 0.1, error: 0.01, residual: 1e-9, invariants: { mass_drift: 2e-7 } }, + { label: "medium", h: 0.05, error: 0.0025, residual: 2e-9, invariants: { mass_drift: 3e-7 } }, + { label: "fine", h: 0.025, error: 0.000625, residual: 3e-9, invariants: { mass_drift: 4e-7 } }, + ], + }), + ) + + try { + const result = await run("physics/simulator-validation/scripts/validate_convergence.py", [ + source, + "--output", + output, + ]) + expect(result.code).toBe(0) + const report = JSON.parse(await Bun.file(output).text()) + expect(report.passed).toBe(true) + expect(report.medianObservedOrder).toBeCloseTo(2) + expect(report.checks["invariant:mass_drift"]).toBe(true) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("simulator-validation fails a refinement study with an excessive residual", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-simulator-")) + const source = path.join(dir, "study.json") + await Bun.write( + source, + JSON.stringify({ + simulator: { name: "solver", version: "1", command: "solver case", configSHA256: "a".repeat(64) }, + expectedOrder: 1, + orderTolerance: 0, + maxResidual: 1e-8, + levels: [ + { label: "coarse", h: 0.1, error: 0.1, residual: 1e-9 }, + { label: "medium", h: 0.05, error: 0.05, residual: 2e-8 }, + { label: "fine", h: 0.025, error: 0.025, residual: 1e-9 }, + ], + }), + ) + + try { + const result = await run("physics/simulator-validation/scripts/validate_convergence.py", [source]) + expect(result.code).toBe(1) + expect(JSON.parse(result.stdout).checks.residual_bound).toBe(false) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("scientific-ablation-design accepts matched one-factor contrasts", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-ablation-")) + const source = path.join(dir, "plan.json") + const output = path.join(dir, "report.json") + const context = { seeds: [1, 2, 3], budget: { candidates: 30 }, split: "held-out", evaluator: "eval-sha" } + await Bun.write( + source, + JSON.stringify({ + metric: { name: "score", direction: "maximize" }, + baseline: { id: "full", config: { memory: "verified", search: "ucb" }, ...context }, + claims: [{ id: "memory-value", factor: "memory", from: "verified", to: "none" }], + arms: [{ id: "no-memory", config: { memory: "none", search: "ucb" }, ...context }], + }), + ) + + try { + const result = await run("research/scientific-ablation-design/scripts/validate_ablation_plan.py", [ + source, + "--output", + output, + ]) + expect(result.code).toBe(0) + const report = JSON.parse(await Bun.file(output).text()) + expect(report.contrasts).toEqual([{ claim: "memory-value", baseline: "full", arm: "no-memory", factor: "memory" }]) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("scientific-ablation-design rejects budget drift", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-ablation-")) + const source = path.join(dir, "plan.json") + await Bun.write( + source, + JSON.stringify({ + metric: { name: "score", direction: "maximize" }, + baseline: { + id: "full", + config: { memory: "verified" }, + seeds: [1, 2], + budget: { candidates: 30 }, + split: "held-out", + evaluator: "eval-sha", + }, + claims: [{ id: "memory-value", factor: "memory", from: "verified", to: "none" }], + arms: [ + { + id: "no-memory", + config: { memory: "none" }, + seeds: [1, 2], + budget: { candidates: 100 }, + split: "held-out", + evaluator: "eval-sha", + }, + ], + }), + ) + + try { + const result = await run("research/scientific-ablation-design/scripts/validate_ablation_plan.py", [source]) + expect(result.code).toBe(2) + expect(result.stderr).toContain("drifts seed, budget, split, or evaluator") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("run-proactive-evaluation commits a token-free score-history pool", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-proactive-evaluation-")) + const script = "research/run-proactive-evaluation/scripts/preflight.ts" + const input = path.join(dir, "private.jsonl") + const protocolFile = path.join(dir, "protocol.json") + const output = path.join(dir, "public.json") + const rows = [ + { id: "case-b", hidden: { prompt: "secret-b", target: 2 }, sourceLosses: [0.2, 0.3, 0.4], stratum: "b" }, + { id: "case-a", hidden: { prompt: "secret-a", target: 1 }, sourceLosses: [0.1, 0.2, 0.3], stratum: "a", weight: 2 }, + { id: "case-c", hidden: { prompt: "secret-c", target: 3 }, sourceLosses: [0.5, 0.4, 0.3], stratum: "c" }, + ] + const protocol = { + sourceModels: ["source-a", "source-b", "source-c"], + selectionSHA256: hash("gmm-selection"), + selectionMethod: "pca-gmm-profile-v1", + calibrationSamples: 2, + maxCalibrationMAE: 0.1, + } + try { + await Promise.all([ + Bun.write(input, `${rows.map((row) => JSON.stringify(row)).join("\n")}\n`), + Bun.write(protocolFile, JSON.stringify(protocol)), + ]) + const valid = await bun(script, ["--input", input, "--protocol", protocolFile, "--out", output]) + expect(valid.code).toBe(0) + expect(JSON.parse(valid.stdout)).toMatchObject({ valid: true, tokenFree: true, probes: 3 }) + const payload = JSON.parse(await Bun.file(output).text()) + expect(payload.probes.map((probe: { id: string }) => probe.id)).toEqual(["case-a", "case-b", "case-c"]) + expect(JSON.stringify(payload)).not.toContain("secret-") + expect(payload.transfer.poolSHA256).toBe(hash(JSON.stringify(payload.probes))) + expect(payload.transfer.sourceManifestSHA256).toBe( + hash( + JSON.stringify({ + sourceModels: protocol.sourceModels, + scores: payload.probes.map((probe: { id: string; sourceLosses: number[] }) => ({ + id: probe.id, + sourceLosses: probe.sourceLosses, + })), + }), + ), + ) + expect( + HarnessContract.Audit.parse({ + mode: "performance", + budget: 2, + minSamples: 2, + transfer: payload.transfer, + promotionRequired: true, + }).transfer, + ).toEqual(payload.transfer) + + await Bun.write(input, `${JSON.stringify({ ...rows[0], sourceLosses: [0.1, 0.2] })}\n${JSON.stringify(rows[1])}\n`) + const drift = await bun(script, ["--input", input, "--protocol", protocolFile, "--out", output]) + expect(drift.code).toBe(1) + expect(drift.stderr).toContain("source dimension drifted") + + await Bun.write(protocolFile, JSON.stringify({ ...protocol, evaluatorToken: "must-not-touch-disk" })) + const token = await bun(script, ["--input", input, "--protocol", protocolFile, "--out", output]) + expect(token.code).toBe(1) + expect(token.stderr).toContain("unknown fields") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("run-topic-aware-failure-discovery salts definitions and emits a bindable protocol", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-topic-failure-")) + const script = "research/run-topic-aware-failure-discovery/scripts/preflight.ts" + const manifestFile = path.join(dir, "manifest.json") + const firstSalt = "a".repeat(64) + const definitions = ["molecular and organismal biology", "mechanics and field theory"] + const names = [ + "topic-model", + "generator", + "correctness-validator", + "topic-validator", + "novelty-validator", + "embedding-model", + ] + const identity = (name: string) => ({ + name, + version: "1", + promptPath: `${name}.prompt.txt`, + configPath: `${name}.config.json`, + }) + const manifest = { + sourcePoolSHA256: hash("audit-pool"), + topicSaltPath: "salt.txt", + topics: [ + { id: "biology", definition: definitions[0] }, + { id: "physics", definition: definitions[1] }, + ], + topicModel: { kind: "predefined", ...identity("topic-model") }, + generator: identity("generator"), + validators: { + correctness: identity("correctness-validator"), + topic: identity("topic-validator"), + novelty: identity("novelty-validator"), + }, + embedding: { ...identity("embedding-model"), dimensions: 8 }, + budget: 6, + anchorsPerAttempt: 2, + failureThreshold: 0.5, + } + try { + await Promise.all([ + ...names.flatMap((name) => [ + Bun.write(path.join(dir, `${name}.prompt.txt`), `private prompt for ${name}`), + Bun.write(path.join(dir, `${name}.config.json`), JSON.stringify({ actor: name, seed: 7 })), + ]), + Bun.write(path.join(dir, "salt.txt"), firstSalt), + Bun.write(manifestFile, JSON.stringify(manifest)), + ]) + const valid = await bun(script, [manifestFile]) + expect(valid.code).toBe(0) + const payload = JSON.parse(valid.stdout) + expect(HarnessContract.FailureDiscovery.parse(payload.protocol)).toEqual(payload.protocol) + expect(payload.protocol.topics.map((topic: { id: string }) => topic.id)).toEqual(["biology", "physics"]) + const privateValues = [ + ...definitions, + firstSalt, + ...names.flatMap((name) => [`private prompt for ${name}`, JSON.stringify({ actor: name, seed: 7 })]), + ] + for (const secret of privateValues) expect(valid.stdout).not.toContain(secret) + + await Bun.write(path.join(dir, "salt.txt"), "b".repeat(64)) + const salted = await bun(script, [manifestFile]) + expect(salted.code).toBe(0) + expect(JSON.parse(salted.stdout).protocol.topics).not.toEqual(payload.protocol.topics) + + await Bun.write(path.join(dir, "salt.txt"), "too-short") + const short = await bun(script, [manifestFile]) + expect(short.code).toBe(1) + expect(short.stderr).toContain("at least 32 bytes") + + await Bun.write(path.join(dir, "salt.txt"), firstSalt) + await Bun.write(manifestFile, JSON.stringify({ ...manifest, evaluatorToken: "must-not-touch-disk" })) + const unknown = await bun(script, [manifestFile]) + expect(unknown.code).toBe(1) + expect(unknown.stderr).toContain("unknown fields: evaluatorToken") + + await Bun.write( + manifestFile, + JSON.stringify({ ...manifest, topics: [{ id: "__proto__", definition: definitions[0] }, manifest.topics[1]] }), + ) + const unsafe = await bun(script, [manifestFile]) + expect(unsafe.code).toBe(1) + expect(unsafe.stderr).toContain("opaque safe identifier") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("run-clean-room-synthesis hides answer facts and emits a bindable factuality protocol", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-clean-room-synthesis-")) + const script = "research/run-clean-room-synthesis/scripts/preflight.ts" + const manifestFile = path.join(dir, "manifest.json") + const salt = "private-random-salt-material-00000000000000000000000000000000" + const reference = "The private systematic-review conclusion must never enter the candidate process." + const facts = [ + { id: "fact-1", text: "The intervention reduced the primary endpoint." }, + { id: "fact-2", text: "The evidence certainty was moderate." }, + ] + const names = ["decomposer", "precision", "recall"] + const identity = (name: string) => ({ + name, + version: "1", + promptPath: `${name}.prompt.txt`, + configPath: `${name}.config.json`, + }) + const manifest = { + query: "What conclusion follows from the pre-cutoff evidence?", + referenceTextPath: "reference.txt", + referenceFacts: facts, + factSaltPath: "salt.txt", + cutoff: "2026-01-31", + tools: ["google_search", "paper_search", "web_browse"], + traceSchemaPath: "trace.schema.json", + filterPolicyPath: "filter.policy.json", + maxToolEvents: 40, + decomposer: identity("decomposer"), + judges: { precision: identity("precision"), recall: identity("recall") }, + minGeneratedFacts: 2, + minPrecision: 0.4, + minRecall: 0.4, + minF1: 0.4, + } + try { + await Promise.all([ + Bun.write(path.join(dir, "reference.txt"), reference), + Bun.write(path.join(dir, "salt.txt"), salt), + Bun.write(path.join(dir, "trace.schema.json"), JSON.stringify({ type: "array", owner: "evaluator" })), + Bun.write(path.join(dir, "filter.policy.json"), JSON.stringify({ forbidden: ["cochrane.org"] })), + ...names.flatMap((name) => [ + Bun.write(path.join(dir, `${name}.prompt.txt`), `private ${name} prompt`), + Bun.write(path.join(dir, `${name}.config.json`), JSON.stringify({ actor: name, seed: 19 })), + ]), + Bun.write(manifestFile, JSON.stringify(manifest)), + ]) + const valid = await bun(script, [manifestFile]) + expect(valid.code).toBe(0) + const payload = JSON.parse(valid.stdout) + expect(HarnessContract.ScientificSynthesis.parse(payload.protocol)).toEqual(payload.protocol) + expect(payload.referenceManifest).toHaveLength(2) + expect(payload.referenceManifest.map((fact: { id: string }) => fact.id)).toEqual(["fact-1", "fact-2"]) + for (const secret of [ + reference, + salt, + ...facts.map((fact) => fact.text), + ...names.flatMap((name) => [`private ${name} prompt`, JSON.stringify({ actor: name, seed: 19 })]), + ]) { + expect(valid.stdout).not.toContain(secret) + } + + await Bun.write(path.join(dir, "salt.txt"), "different-private-random-salt-material-000000000000000000000000") + const salted = await bun(script, [manifestFile]) + expect(salted.code).toBe(0) + expect(JSON.parse(salted.stdout).referenceManifest).not.toEqual(payload.referenceManifest) + + await Bun.write(path.join(dir, "salt.txt"), "too-short") + const short = await bun(script, [manifestFile]) + expect(short.code).toBe(1) + expect(short.stderr).toContain("at least 32 bytes") + + await Bun.write(path.join(dir, "salt.txt"), salt) + await Bun.write(manifestFile, JSON.stringify({ ...manifest, cutoff: "2026-02-31" })) + const date = await bun(script, [manifestFile]) + expect(date.code).toBe(1) + expect(date.stderr).toContain("ISO calendar date") + + await Bun.write( + manifestFile, + JSON.stringify({ + ...manifest, + judges: { precision: identity("precision"), recall: identity("precision") }, + }), + ) + const duplicate = await bun(script, [manifestFile]) + expect(duplicate.code).toBe(1) + expect(duplicate.stderr).toContain("distinct prompt commitments") + + await Bun.write(manifestFile, JSON.stringify({ ...manifest, evaluatorToken: "must-not-touch-disk" })) + const unknown = await bun(script, [manifestFile]) + expect(unknown.code).toBe(1) + expect(unknown.stderr).toContain("unknown fields: evaluatorToken") + + await Bun.write( + manifestFile, + JSON.stringify({ ...manifest, referenceFacts: [{ id: "__proto__", text: facts[0]!.text }, facts[1]] }), + ) + const unsafe = await bun(script, [manifestFile]) + expect(unsafe.code).toBe(1) + expect(unsafe.stderr).toContain("opaque safe identifier") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("record-human-ai-autonomy hashes private interactions and emits a token-free trace", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-human-ai-autonomy-")) + const script = "research/record-human-ai-autonomy/scripts/preflight.ts" + const manifestFile = path.join(dir, "manifest.json") + const protocolFile = path.join(dir, "preflight.json") + const traceFile = path.join(dir, "trace.json") + const startedAt = Date.now() + const problem = "Private held-out benchmark problem" + const result = "Private autonomous scientific solution" + const token = "private-evaluator-capability-must-never-be-emitted" + const manifest = { + claimedLevel: "essentially_autonomous", + recorder: { name: "interaction-recorder", version: "1", artifactPath: "recorder.bin" }, + traceSchemaPath: "trace.schema.json", + classificationPolicyPath: "classification.policy.md", + maxEvents: 12, + disclosure: "evaluator_retained", + } + const trace = { + sessionID: "native-autonomy", + subject: { type: "run", id: "run-native-autonomy" }, + artifactPath: "solution.txt", + rawLogPath: "raw-log.jsonl", + startedAt, + endedAt: startedAt + 1, + events: [ + { + sequence: 1, + at: startedAt, + actor: "benchmark", + kind: "problem_statement", + contribution: "problem", + contentPath: "problem.txt", + evidence: ["private://raw-log#1"], + }, + { + sequence: 2, + at: startedAt + 1, + actor: "agent", + kind: "artifact_edit", + contribution: "core", + contentPath: "solution.txt", + artifactAfterPath: "solution.txt", + evidence: ["private://raw-log#2"], + }, + ], + } + try { + await Promise.all([ + Bun.write(path.join(dir, "recorder.bin"), "frozen recorder executable"), + Bun.write(path.join(dir, "trace.schema.json"), JSON.stringify({ owner: "evaluator_runtime" })), + Bun.write(path.join(dir, "classification.policy.md"), "Aletheia contribution classes"), + Bun.write(path.join(dir, "problem.txt"), problem), + Bun.write(path.join(dir, "solution.txt"), result), + Bun.write(path.join(dir, "raw-log.jsonl"), `${problem}\n${result}\n${token}\n`), + Bun.write(manifestFile, JSON.stringify(manifest)), + Bun.write(traceFile, JSON.stringify(trace)), + ]) + const frozen = await bun(script, ["protocol", manifestFile]) + expect(frozen.code).toBe(0) + const protocol = JSON.parse(frozen.stdout) + expect(HarnessContract.HumanAIAutonomy.parse(protocol.protocol)).toEqual(protocol.protocol) + await Bun.write(protocolFile, frozen.stdout) + + const prepared = await bun(script, ["submission", protocolFile, traceFile]) + expect(prepared.code).toBe(0) + const payload = JSON.parse(prepared.stdout) + expect(HarnessAutonomy.Submit.parse({ ...payload.submission, evaluatorToken: "x".repeat(32) })).toMatchObject({ + sessionID: "native-autonomy", + artifactSHA256: hash(result), + trace: { complete: true, events: [{ sequence: 1 }, { sequence: 2 }] }, + }) + expect(payload.preview).toMatchObject({ + claimedLevel: "essentially_autonomous", + derivedLevel: "essentially_autonomous", + humanSubstantiveEvents: 0, + agentSubstantiveEvents: 1, + }) + for (const secret of [problem, result, token]) expect(prepared.stdout).not.toContain(secret) + + const gapped = structuredClone(trace) + gapped.events[1]!.sequence = 3 + await Bun.write(traceFile, JSON.stringify(gapped)) + const invalid = await bun(script, ["submission", protocolFile, traceFile]) + expect(invalid.code).toBe(1) + expect(invalid.stderr).toContain("contiguous") + + await Bun.write(manifestFile, JSON.stringify({ ...manifest, evaluatorToken: token })) + const secret = await bun(script, ["protocol", manifestFile]) + expect(secret.code).toBe(1) + expect(secret.stderr).toContain("unknown fields: evaluatorToken") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("verify-formal-proof freezes a full external checker stack and emits token-free proof evidence", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-formal-proof-")) + const script = "research/verify-formal-proof/scripts/preflight.ts" + const manifestFile = path.join(dir, "manifest.json") + const protocolFile = path.join(dir, "preflight.json") + const evidenceFile = path.join(dir, "evidence.json") + const startedAt = Date.now() + const challenge = "private trusted theorem challenge" + const proof = "theorem target : True := by trivial" + const transcript = "private checker output accepted" + const token = "private-formal-evaluator-capability-must-never-be-emitted" + const verifiers = HarnessContract.FormalVerifierRole.options.map((role) => ({ + role, + name: `native-${role}`, + version: "1", + artifactPath: `${role}.bin`, + })) + const manifest = { + tier: "external_crosscheck", + relation: "exact_proof", + challengePath: "challenge.lean", + statementPath: "statement.lean", + declaration: "Native.target", + module: "Native.Proof", + leanVersion: "4.33.0", + leanToolchainPath: "lean-toolchain", + lakeManifestPath: "lake-manifest.json", + dependencyTreePath: "deps.json", + verifiers, + sandboxImagePath: "sandbox.img", + allowedAxioms: ["Classical.choice", "Quot.sound", "propext"], + maxFiles: 16, + } + const evidence = { + sessionID: "native-formal-proof", + subject: { type: "run", id: "run-native-formal-proof" }, + artifactPath: "proof.lean", + manifest: { + complete: true, + files: [ + { path: "statement.lean", role: "statement" }, + { path: "proof.lean", role: "proof" }, + { path: "lean-toolchain", role: "lean_toolchain" }, + { path: "lake-manifest.json", role: "lake_manifest" }, + { path: "deps.json", role: "dependency_tree" }, + { path: "challenge.lean", role: "challenge" }, + ], + }, + verification: { + startedAt, + endedAt: startedAt + 1, + build: { exitCode: 0, warnings: 0, transcriptPath: "build.log" }, + source: { complete: true, findings: [], transcriptPath: "source.log" }, + axioms: { + complete: true, + typesTraversed: true, + observed: ["propext", "Classical.choice", "Quot.sound"], + transcriptPath: "axioms.log", + }, + fresh: { fresh: true, exitCode: 0, transcriptPath: "fresh.log" }, + external: { + sandboxed: true, + challengeMatched: true, + proofTermPath: "proof.term", + transcriptPath: "comparator.log", + checks: [ + { role: "lean_kernel", accepted: true, transcriptPath: "external-lean.log" }, + { role: "external_checker", accepted: true, transcriptPath: "external-independent.log" }, + ], + }, + }, + } + try { + await Promise.all([ + Bun.write(path.join(dir, "challenge.lean"), challenge), + Bun.write(path.join(dir, "statement.lean"), "Native.target : True"), + Bun.write(path.join(dir, "proof.lean"), proof), + Bun.write(path.join(dir, "lean-toolchain"), "leanprover/lean4:v4.33.0"), + Bun.write(path.join(dir, "lake-manifest.json"), JSON.stringify({ packages: [] })), + Bun.write(path.join(dir, "deps.json"), JSON.stringify({ closure: ["mathlib"] })), + Bun.write(path.join(dir, "sandbox.img"), "frozen formal verification sandbox"), + Bun.write(path.join(dir, "proof.term"), "serialized proof term"), + ...verifiers.map((item) => Bun.write(path.join(dir, item.artifactPath), `binary:${item.role}`)), + ...[ + "build.log", + "source.log", + "axioms.log", + "fresh.log", + "comparator.log", + "external-lean.log", + "external-independent.log", + ].map((name) => Bun.write(path.join(dir, name), `${transcript}:${name}:${token}`)), + Bun.write(manifestFile, JSON.stringify(manifest)), + Bun.write(evidenceFile, JSON.stringify(evidence)), + ]) + const frozen = await bun(script, ["protocol", manifestFile]) + expect(frozen.code).toBe(0) + const preflight = JSON.parse(frozen.stdout) + expect(HarnessContract.FormalProof.parse(preflight.protocol)).toEqual(preflight.protocol) + await Bun.write(protocolFile, frozen.stdout) + + const prepared = await bun(script, ["submission", protocolFile, evidenceFile]) + expect(prepared.code).toBe(0) + const payload = JSON.parse(prepared.stdout) + const submission = HarnessFormal.Submit.parse({ ...payload.submission, evaluatorToken: "x".repeat(32) }) + expect(submission).toMatchObject({ + sessionID: "native-formal-proof", + relation: "exact_proof", + artifactSHA256: hash(proof), + manifest: { complete: true }, + verification: { + build: { exitCode: 0, warnings: 0 }, + source: { complete: true, findings: [] }, + axioms: { complete: true, typesTraversed: true }, + fresh: { fresh: true, exitCode: 0 }, + external: { sandboxed: true, challengeMatched: true }, + }, + }) + expect(submission.manifest.files[0]?.path).toBe("challenge.lean") + expect(payload.preview).toMatchObject({ + tier: "external_crosscheck", + relation: "exact_proof", + files: 6, + artifactSHA256: hash(proof), + }) + for (const secret of [challenge, proof, transcript, token]) expect(prepared.stdout).not.toContain(secret) + + await Bun.write(manifestFile, JSON.stringify({ ...manifest, allowedAxioms: ["sorryAx"] })) + const sorry = await bun(script, ["protocol", manifestFile]) + expect(sorry.code).toBe(1) + expect(sorry.stderr).toContain("never include sorryAx") + + await Bun.write(manifestFile, JSON.stringify({ ...manifest, evaluatorToken: token })) + const secret = await bun(script, ["protocol", manifestFile]) + expect(secret.code).toBe(1) + expect(secret.stderr).toContain("unknown fields: evaluatorToken") + + await Bun.write(manifestFile, JSON.stringify({ ...manifest, challengePath: "../private-challenge.lean" })) + const escaped = await bun(script, ["protocol", manifestFile]) + expect(escaped.code).toBe(1) + expect(escaped.stderr).toContain("escapes its evidence directory") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test("operate-proof-blueprint freezes architecture and emits token-free exact-placeholder attempts", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openscience-proof-blueprint-")) + const script = "research/operate-proof-blueprint/scripts/preflight.ts" + const manifestFile = path.join(dir, "manifest.json") + const protocolFile = path.join(dir, "preflight.json") + const leaseFile = path.join(dir, "lease.json") + const evidenceFile = path.join(dir, "evidence.json") + const now = Date.now() + const token = "private-blueprint-evaluator-token" + const files = { + "schema.json": "proof-blueprint-schema-v1", + "lean.bin": "frozen-lean-kernel", + "validator.bin": "frozen-sketch-validator", + "reviewer.bin": "frozen-decomposition-reviewer", + "rubric.txt": "relevant easier plausible", + "child.lean": "theorem Native.child : True := by trivial", + "sketch.lean": "theorem Native.root (h : Native.child) : True := by trivial", + "plan.txt": "reduce root to Native.child", + "compiler.log": `compiler accepted ${token}`, + "validator.log": `placeholders exactly Native.child ${token}`, + "reviewer.log": `all rubric checks passed ${token}`, + "feedback.txt": `no failure ${token}`, + } + const manifest = { + graphSchemaPath: "schema.json", + compilerPath: "lean.bin", + sketchValidatorPath: "validator.bin", + reviewerPath: "reviewer.bin", + reviewerPromptPath: "rubric.txt", + maxNodes: 32, + maxDepth: 6, + maxParallel: 4, + maxAttemptsPerGoal: 3, + maxRefinementsPerGoal: 2, + leaseDurationMs: 60_000, + } + const lease = { + id: hash("lease"), + goalID: hash("goal"), + revision: 1, + ordinal: 0, + status: "open", + issuedAt: now, + expiresAt: now + 60_000, + } + const evidence = { + sessionID: "native-proof-blueprint", + kind: "decomposition", + artifactPath: "sketch.lean", + informalPlanPath: "plan.txt", + children: [{ statementPath: "child.lean", declaration: "Native.child", module: "Native.Blueprint" }], + compiler: { + artifactPath: "lean.bin", + statementMatched: true, + exitCode: 0, + warnings: 0, + transcriptPath: "compiler.log", + feedbackPath: "feedback.txt", + startedAt: now, + endedAt: now + 1, + }, + validator: { artifactPath: "validator.bin", transcriptPath: "validator.log" }, + review: { + artifactPath: "reviewer.bin", + promptPath: "rubric.txt", + relevant: true, + easier: true, + plausible: true, + transcriptPath: "reviewer.log", + }, + } + try { + await Promise.all([ + ...Object.entries(files).map(([name, value]) => Bun.write(path.join(dir, name), value)), + Bun.write(manifestFile, JSON.stringify(manifest)), + Bun.write(leaseFile, JSON.stringify(lease)), + Bun.write(evidenceFile, JSON.stringify(evidence)), + ]) + const frozen = await bun(script, ["protocol", manifestFile]) + expect(frozen.code).toBe(0) + const preflight = JSON.parse(frozen.stdout) + expect(HarnessContract.ProofBlueprint.parse(preflight.blueprint)).toEqual(preflight.blueprint) + await Bun.write(protocolFile, frozen.stdout) + + const prepared = await bun(script, ["attempt", protocolFile, leaseFile, evidenceFile]) + expect(prepared.code).toBe(0) + const payload = JSON.parse(prepared.stdout) + const submission = HarnessBlueprint.DecompositionSubmit.parse({ + ...payload.submission, + evaluatorToken: "x".repeat(32), + }) + expect(submission).toMatchObject({ + sessionID: "native-proof-blueprint", + kind: "decomposition", + leaseID: lease.id, + children: [{ declaration: "Native.child", module: "Native.Blueprint" }], + verification: { placeholderDeclarations: ["Native.child"], statementMatched: true, exitCode: 0 }, + review: { relevant: true, easier: true, plausible: true }, + }) + for (const secret of [files["child.lean"], files["sketch.lean"], token]) { + expect(prepared.stdout).not.toContain(secret) + } + + await Bun.write(path.join(dir, "other-validator.bin"), "substituted-validator") + await Bun.write( + evidenceFile, + JSON.stringify({ ...evidence, validator: { ...evidence.validator, artifactPath: "other-validator.bin" } }), + ) + const changed = await bun(script, ["attempt", protocolFile, leaseFile, evidenceFile]) + expect(changed.code).toBe(1) + expect(changed.stderr).toContain("does not match the frozen blueprint") + + await Bun.write(manifestFile, JSON.stringify({ ...manifest, compilerPath: "../private-lean.bin" })) + const escaped = await bun(script, ["protocol", manifestFile]) + expect(escaped.code).toBe(1) + expect(escaped.stderr).toContain("escapes its evidence directory") + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) diff --git a/backend/cli/test/tool/harness.test.ts b/backend/cli/test/tool/harness.test.ts new file mode 100644 index 00000000..46cae825 --- /dev/null +++ b/backend/cli/test/tool/harness.test.ts @@ -0,0 +1,558 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { HarnessContract } from "../../src/session/harness/contract" +import { HarnessEvaluation } from "../../src/session/harness/evaluation" +import { HarnessMemory } from "../../src/session/harness/memory" +import { HarnessOrchestrator } from "../../src/session/harness/orchestrator" +import { HarnessSearch } from "../../src/session/harness/search" +import { HarnessTool } from "../../src/tool/harness" +import { TaskParameters } from "../../src/tool/task" + +const sessionID = "harness-tool-session" +const hash = (value: string) => new Bun.CryptoHasher("sha256").update(value).digest("hex") +const context = { + sessionID, + messageID: "message", + callID: "call", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata() {}, + async ask() {}, +} + +afterEach(async () => { + await Promise.all( + ["contracts", "evaluations", "search", "orchestration", "worlds"].map((name) => + fs.rm(path.join(Global.Path.data, "harness", name, `${encodeURIComponent(sessionID)}.json`), { force: true }), + ), + ) + await fs.rm(path.join(Global.Path.data, "harness", "retrospectives"), { recursive: true, force: true }) +}) + +async function bind(orchestration?: HarnessContract.Orchestration) { + return HarnessContract.bind({ + schemaVersion: 1, + runID: "run-harness-tool", + sessionID, + objective: "Maximize the official score", + benchmark: { + name: "pde", + title: "PDE evaluation", + family: "physics", + task: "Improve and verify a numerical PDE solution", + version: "1", + taskID: "task", + split: "held_out", + evaluator: "official-evaluator", + metric: "score", + direction: "maximize", + target: 0.9, + }, + profile: "optimize", + ...(orchestration ? { orchestration } : {}), + model: { provider: "test", name: "model" }, + tools: ["harness"], + skills: [], + budget: { candidates: 2, wallTimeMs: 60_000 }, + seed: 3, + intervention: "autonomous", + contamination: { policy: "hidden tests stay hidden", hiddenTestsAccessible: false }, + createdAt: Date.now(), + }) +} + +const turns: string[] = [] +const attest = async ( + work: { id: string; agent: HarnessOrchestrator.WorkerReceipt["agent"]; prompt: string }, + worker: string, +) => { + const state = await HarnessOrchestrator.read(sessionID) + const completedAt = Date.now() + return HarnessOrchestrator.attest({ + sessionID, + workID: work.id, + workerSessionID: worker, + turnID: `harness-tool-turn-${turns.push(work.id)}`, + agent: work.agent, + prompt: `Execute:\n${work.prompt}`, + outcome: "completed", + usage: { steps: 1, tokens: 100, costUSD: 0.01, wallTimeMs: 10 }, + toolCalls: 1, + failedToolCalls: 0, + startedAt: Math.max(state.createdAt, completedAt - 10), + completedAt, + }) +} + +describe("harness tool", () => { + test("exposes persisted provisional coalition work without evaluator authority", async () => { + await bind() + const tool = await HarnessTool.init() + const started = await tool.execute({ action: "coalition_start" }, context) + const state = JSON.parse(started.output) + expect(state).toMatchObject({ + status: "active", + workerPolicy: "task-attested-v1", + minIndependentVerifiers: 1, + revision: 0, + }) + expect(state.ready.length).toBeGreaterThan(0) + expect(state.ready.length).toBeLessThanOrEqual(state.maxWorkers) + + const work = state.ready[0] + await attest(work, "fresh-child-session") + const completed = await tool.execute( + { + action: "coalition_complete", + work_id: work.id, + worker_session_id: "fresh-child-session", + result_summary: "produced a bounded proposal", + artifact_refs: ["artifact://proposal"], + evidence_refs: ["evidence://trace"], + }, + context, + ) + expect(completed.metadata).toMatchObject({ workID: work.id, provisional: true, revision: 2 }) + expect(JSON.parse(completed.output).revision).toBe(2) + expect(tool.parameters.safeParse({ action: "coalition_verify", work_id: work.id }).success).toBe(false) + + expect( + TaskParameters.safeParse({ + description: "Execute coalition work", + prompt: work.prompt, + subagent_type: work.agent, + harness_work_id: work.id, + }).success, + ).toBe(true) + expect( + TaskParameters.safeParse({ + description: "Execute coalition work", + prompt: work.prompt, + subagent_type: work.agent, + harness_work_id: "fabricated", + }).success, + ).toBe(false) + }) + + test("carries verifier severity through the tool and exposes backend-owned repair routing", async () => { + await bind({ + topology: "verifier_loop", + maxWorkers: 2, + maxRounds: 1, + minIndependentVerifiers: 2, + repair: { protocolVersion: "verifier-routed-v1", minConfidence: 0.7 }, + }) + const tool = await HarnessTool.init() + const started = JSON.parse((await tool.execute({ action: "coalition_start" }, context)).output) + expect(started).toMatchObject({ + protocolVersion: "coalition-v3", + topology: "verifier_loop", + repair: { protocolVersion: "verifier-routed-v1", phase: "producing", routes: [] }, + }) + const author = started.ready[0] + await attest(author, "repair-tool-author") + const proposed = JSON.parse( + ( + await tool.execute( + { + action: "coalition_complete", + work_id: author.id, + worker_session_id: "repair-tool-author", + result_summary: "candidate", + artifact_refs: ["artifact://candidate"], + }, + context, + ) + ).output, + ) + expect(proposed.repair).toMatchObject({ phase: "verifying" }) + expect(proposed.ready).toHaveLength(2) + + const verify = async (work: (typeof proposed.ready)[number], worker: string) => { + await attest(work, worker) + return JSON.parse( + ( + await tool.execute( + { + action: "coalition_complete", + work_id: work.id, + worker_session_id: worker, + result_summary: "independent support", + evidence_refs: [`evidence://${worker}`], + verdict: "support", + verdict_severity: "none", + verdict_confidence: 0.9, + verdict_checks: [ + { id: "observable-check", status: "passed", evidence_refs: [`evidence://${worker}/check`] }, + ], + }, + context, + ) + ).output, + ) + } + const first = await verify(proposed.ready[0], "repair-tool-verifier-a") + const settled = await verify(first.ready[0], "repair-tool-verifier-b") + expect(settled).toMatchObject({ + status: "completed", + repair: { phase: "completed", stopReason: "accepted", routes: [{ decision: "accept" }] }, + consensus: { status: "supported", verifierCount: 2, support: 2 }, + }) + }) + + test("publishes the exact Task session contract for persistent producer lanes", async () => { + await bind({ topology: "evolution", maxWorkers: 2, maxRounds: 1, minIndependentVerifiers: 1 }) + const tool = await HarnessTool.init() + const started = JSON.parse((await tool.execute({ action: "coalition_start" }, context)).output) + expect(started).toMatchObject({ sessionPolicy: "producer-lanes-v1", topology: "evolution" }) + expect( + started.ready.map((work: { lane?: string; resumeSessionID?: string }) => [work.lane, work.resumeSessionID]), + ).toEqual([ + ["producer-a", undefined], + ["producer-b", undefined], + ]) + + const complete = async ( + state: { + ready: Array<{ + id: string + label: string + agent: HarnessOrchestrator.WorkerReceipt["agent"] + prompt: string + }> + }, + worker: string, + ) => { + await attest(state.ready[0]!, worker) + return JSON.parse( + ( + await tool.execute( + { + action: "coalition_complete", + work_id: state.ready[0]!.id, + worker_session_id: worker, + result_summary: state.ready[0]!.label, + artifact_refs: [`artifact://${state.ready[0]!.label}`], + evidence_refs: [`evidence://${state.ready[0]!.label}`], + }, + context, + ) + ).output, + ) + } + const seededA = await complete(started, "lane-a-session") + const seededB = await complete(seededA, "lane-b-session") + const mapped = await complete(seededB, "fresh-map") + const reflected = await complete(mapped, "fresh-reflection") + const ranked = await complete(reflected, "fresh-ranking") + expect( + ranked.ready.map((work: { lane: string; resumeSessionID: string }) => [work.lane, work.resumeSessionID]), + ).toEqual([ + ["producer-a", "lane-a-session"], + ["producer-b", "lane-b-session"], + ]) + }) + + test("exposes resumable candidate control without an agent verification action", async () => { + await bind() + const tool = await HarnessTool.init() + const started = await tool.execute({ action: "start", stall: 2 }, context) + const snapshot = JSON.parse(started.output) + expect(snapshot).toMatchObject({ + status: "active", + proposalPolicy: "leased-v3", + budget: { candidates: 2, wallTimeMs: 60_000, stall: 2 }, + used: 0, + recommendation: { strategy: "seed", mode: "single-pass", contextIDs: [] }, + recommendationContext: [], + }) + + const invalid = await tool.execute( + { + action: "propose", + branch: "baseline", + proposal: "proposal without a lease", + artifact_uri: "artifact://invalid", + artifact_sha256: hash("invalid"), + }, + context, + ) + expect(invalid.title).toBe("Invalid proposal") + + const proposed = await tool.execute( + { + action: "propose", + recommendation_id: snapshot.recommendation.id, + branch: "baseline", + proposal: "baseline candidate", + artifact_uri: "artifact://baseline", + artifact_sha256: hash("baseline"), + }, + context, + ) + const candidateID = proposed.metadata.candidateID as string + expect(proposed.metadata).toMatchObject({ accepted: true, candidateID }) + + const duplicate = await tool.execute( + { + action: "propose", + recommendation_id: snapshot.recommendation.id, + parent_ids: [hash("fabricated-parent")], + inspiration_ids: [hash("fabricated-inspiration")], + branch: "renamed", + proposal: "same bytes under a different wrapper", + artifact_uri: "artifact://baseline-mirror", + artifact_sha256: hash("baseline"), + }, + context, + ) + expect(duplicate.metadata).toMatchObject({ accepted: true, deduplicated: true, candidateID }) + + const observed = await tool.execute( + { + action: "observe", + candidate_id: candidateID, + status: "passed", + score: 999, + metrics: { proxy: 999 }, + feedback: "provisional feedback stays visibly unverified", + }, + context, + ) + expect(observed.title).toBe("Unverified observation recorded") + expect(observed.metadata).toMatchObject({ verified: false }) + expect("bestID" in JSON.parse(observed.output)).toBe(false) + expect(tool.parameters.safeParse({ action: "verify", candidate_id: candidateID }).success).toBe(false) + + const checkpoint = await tool.execute({ action: "status" }, context) + expect(JSON.parse(checkpoint.output)).toMatchObject({ + revision: 2, + candidates: [ + { + id: candidateID, + proposal: "baseline candidate", + artifact: { uri: "artifact://baseline", sha256: hash("baseline") }, + source: "observed", + score: 999, + metrics: { proxy: 999 }, + feedback: "provisional feedback stays visibly unverified", + lease: { id: snapshot.recommendation.id, mode: "single-pass", contextIDs: [] }, + }, + ], + }) + }) + + test("returns exact verified trajectory payloads with an adaptive recommendation", async () => { + await bind() + const tool = await HarnessTool.init() + const started = JSON.parse((await tool.execute({ action: "start" }, context)).output) + const proposed = await tool.execute( + { + action: "propose", + recommendation_id: started.recommendation.id, + branch: "baseline", + proposal: "candidate retained for focused refinement", + artifact_uri: "artifact://trajectory", + artifact_sha256: hash("trajectory"), + }, + context, + ) + const candidateID = proposed.metadata.candidateID as string + await HarnessEvaluation.record({ + schemaVersion: 1, + runID: "run-harness-tool", + sessionID, + subject: { type: "candidate", id: candidateID }, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status: "passed", + score: 0.8, + metrics: { score: 0.8, stability: 0.7 }, + checks: [{ id: "official", status: "passed", blocking: true, evidence: ["metric:score"] }], + evidence: ["report:trajectory"], + evaluatedAt: Date.now(), + notes: "improve the boundary residual", + }) + await HarnessSearch.verify({ sessionID, candidateID }) + const checkpoint = JSON.parse((await tool.execute({ action: "status" }, context)).output) + expect(checkpoint.recommendation).toMatchObject({ + strategy: "exploit", + mode: "diff", + parentIDs: [candidateID], + contextIDs: [candidateID], + }) + expect(checkpoint.recommendationContext).toEqual([ + expect.objectContaining({ + id: candidateID, + artifact: { uri: "artifact://trajectory", sha256: hash("trajectory") }, + score: 0.8, + metrics: { score: 0.8, stability: 0.7 }, + feedback: "improve the boundary residual", + }), + ]) + }) + + test("dispatches, releases, and consumes budget-backed parallel variations", async () => { + await bind() + const tool = await HarnessTool.init() + const started = JSON.parse((await tool.execute({ action: "start" }, context)).output) + const dispatched = await tool.execute({ action: "dispatch", count: 8 }, context) + expect(dispatched.metadata).toMatchObject({ issued: 2 }) + const batch = JSON.parse(dispatched.output) + expect(batch.reservations).toMatchObject({ open: 2, consumed: 0, released: 0 }) + const [first, second] = batch.reservations.ready + expect(first.mandate).toMatchObject({ protocol: "agentic-variation-v1", operator: "architectural-change" }) + expect(second.mandate).toMatchObject({ protocol: "agentic-variation-v1", operator: "composition" }) + expect(first.mandate.id).not.toBe(second.mandate.id) + + const invalid = await tool.execute( + { + action: "propose", + recommendation_id: started.recommendation.id, + reservation_id: second.id, + branch: "invalid", + proposal: "two admission capabilities", + artifact_uri: "artifact://invalid-admission", + artifact_sha256: hash("invalid-admission"), + }, + context, + ) + expect(invalid.title).toBe("Invalid proposal") + + const released = await tool.execute({ action: "release", reservation_id: first.id }, context) + expect(JSON.parse(released.output).reservations).toMatchObject({ open: 1, consumed: 0, released: 1 }) + const proposed = await tool.execute( + { + action: "propose", + reservation_id: second.id, + parent_ids: second.parentIDs, + inspiration_ids: second.inspirationIDs, + branch: "parallel", + proposal: "consume one reserved sibling", + artifact_uri: "artifact://parallel", + artifact_sha256: hash("parallel"), + }, + context, + ) + expect(proposed.metadata.accepted).toBe(true) + const state = JSON.parse(proposed.output) + expect(state.reservations).toMatchObject({ open: 0, consumed: 1, released: 1 }) + expect(state.candidates[0]).toMatchObject({ + reservationID: second.id, + lease: second.lease, + mandate: second.mandate, + }) + }) + + test("exposes revision-safe continual state without allowing self-certified confidence", async () => { + await bind() + const tool = await HarnessTool.init() + const initial = await tool.execute({ action: "world_status" }, context) + expect(initial.metadata).toMatchObject({ revision: 0, contextEpoch: 0, refinementRecommended: false }) + + const failure = await tool.execute( + { + action: "world_event", + event_type: "failure", + event_summary: "The independent residual check failed", + evidence: ["tool:residual.json"], + state_changed: true, + }, + context, + ) + expect(failure.metadata).toMatchObject({ revision: 1, contextEpoch: 1, refinementRecommended: true }) + + const refined = await tool.execute( + { + action: "world_refine", + expected_revision: 1, + world_reason: "failure", + world_patches: [ + { + op: "upsert", + key: "residual-failure", + kind: "observation", + content: "The current discretization violates the residual tolerance", + confidence: 3, + evidenceRefs: ["tool:residual.json"], + }, + ], + }, + context, + ) + expect(refined.metadata).toMatchObject({ revision: 2, contextEpoch: 2 }) + expect(JSON.parse(refined.output).entries).toContainEqual( + expect.objectContaining({ key: "residual-failure", confidence: 3 }), + ) + expect( + tool.parameters.safeParse({ + action: "world_refine", + expected_revision: 2, + world_reason: "manual", + world_patches: [ + { + op: "upsert", + key: "self-certified", + kind: "hypothesis", + content: "Unverified claim", + confidence: 5, + evidenceRefs: ["self:claim"], + }, + ], + }).success, + ).toBe(false) + + const rolledBack = await tool.execute( + { action: "world_rollback", expected_revision: 2, target_revision: 1 }, + context, + ) + expect(rolledBack.metadata).toMatchObject({ revision: 3, contextEpoch: 3 }) + expect(JSON.parse(rolledBack.output).entries).toEqual([]) + }) + + test("surfaces external hindsight only after backend verification", async () => { + await bind() + const tool = await HarnessTool.init() + const started = await tool.execute({ action: "start" }, context) + const recommendation = JSON.parse(started.output).recommendation + const proposed = await tool.execute( + { + action: "propose", + recommendation_id: recommendation.id, + branch: "stable", + proposal: "conservative spectral step", + artifact_uri: "artifact://spectral", + artifact_sha256: hash("spectral"), + }, + context, + ) + const candidateID = proposed.metadata.candidateID as string + await HarnessEvaluation.record({ + schemaVersion: 1, + runID: "run-harness-tool", + sessionID, + subject: { type: "candidate", id: candidateID }, + evaluator: { name: "official-evaluator", version: "1", source: "benchmark" }, + status: "passed", + score: 0.91, + metrics: { score: 0.91 }, + checks: [{ id: "official", status: "passed", blocking: true, evidence: ["metric:score"] }], + evidence: ["report:official"], + evaluatedAt: Date.now(), + notes: "passed the held-out stability gate", + }) + const state = await HarnessSearch.verify({ sessionID, candidateID }) + expect(state).toMatchObject({ status: "completed", stopReason: "objective_met", bestID: candidateID }) + await HarnessMemory.capture({ sessionID, candidateID, stage: "evaluation" }) + + const hindsight = await tool.execute( + { action: "hindsight", query: "spectral stability", stage: "planning" }, + context, + ) + expect(hindsight.title).toBe("Verified hindsight") + expect(hindsight.output).toContain("conservative spectral step") + expect(hindsight.output).toContain("passed the held-out stability gate") + }) +}) diff --git a/backend/cli/test/tool/learn.test.ts b/backend/cli/test/tool/learn.test.ts new file mode 100644 index 00000000..7049ced4 --- /dev/null +++ b/backend/cli/test/tool/learn.test.ts @@ -0,0 +1,42 @@ +import { afterEach, describe, expect, test } from "bun:test" +import fs from "fs/promises" +import path from "path" +import { Global } from "../../src/global" +import { LearnTool } from "../../src/tool/learn" + +const name = "test-learn-proposal" +const context = { + sessionID: "learn-proposal-session", + messageID: "message", + callID: "call", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata() {}, + async ask() {}, +} + +afterEach(async () => { + await fs.rm(path.join(Global.Path.data, "learned-skill-proposals", name), { recursive: true, force: true }) + await fs.rm(path.join(Global.Path.data, "learned-skills", name), { recursive: true, force: true }) +}) + +describe("learn tool", () => { + test("creates an inactive proposal instead of self-promoting or uploading", async () => { + const description = "Use when testing a learned workflow safely." + const tool = await LearnTool.init() + const result = await tool.execute( + { + name, + description, + content: `---\nname: ${name}\ndescription: ${description}\n---\n\n# Safe workflow\n`, + }, + context, + ) + expect(result.title).toBe(`Skill proposal: ${name}`) + expect(result.metadata).toMatchObject({ name, status: "pending" }) + expect(result.output).toContain("quarantined and inactive") + expect(await Bun.file(path.join(Global.Path.data, "learned-skills", name, "SKILL.md")).exists()).toBe(false) + expect(await Bun.file(path.join(Global.Path.data, "learned-skill-proposals", name, "SKILL.md")).exists()).toBe(true) + }) +}) diff --git a/backend/cli/test/tool/plan-mode.test.ts b/backend/cli/test/tool/plan-mode.test.ts index db5075b9..a41c970f 100644 --- a/backend/cli/test/tool/plan-mode.test.ts +++ b/backend/cli/test/tool/plan-mode.test.ts @@ -5,6 +5,8 @@ import { Instance } from "../../src/project/instance" import { Session } from "../../src/session" import { ApplyPatchTool } from "../../src/tool/apply_patch" import { ArtifactTool } from "../../src/tool/artifact" +import { HarnessTool } from "../../src/tool/harness" +import { ClaimTool } from "../../src/tool/claim" import { AtlasTool } from "../../src/tool/atlas" import { AtlasRecordTool } from "../../src/tool/atlas-record" import { BashTool } from "../../src/tool/bash" @@ -124,6 +126,18 @@ describe("tool.plan-mode", () => { { action: "register", type: "text", content: "must not persist" }, context("plan"), ), + async () => (await HarnessTool.init()).execute({ action: "start" }, context("plan")), + async () => + (await ClaimTool.init()).execute( + { + action: "declare", + text: "must not persist", + kind: "descriptive", + importance: "supporting", + subject_uri: "artifact://blocked", + }, + context("plan"), + ), async () => (await PlanExitTool.init()).execute({}, context("plan")), ] @@ -141,6 +155,8 @@ describe("tool.plan-mode", () => { "atlas", "atlas_record", "artifact", + "harness", + "claim", "plan_exit", ]) expect(await Bun.file(marker).exists()).toBe(false) diff --git a/backend/cli/test/tool/registry-agents.test.ts b/backend/cli/test/tool/registry-agents.test.ts index 2e8a90a4..c2902d3d 100644 --- a/backend/cli/test/tool/registry-agents.test.ts +++ b/backend/cli/test/tool/registry-agents.test.ts @@ -17,6 +17,9 @@ describe("tool registry agent boundaries", () => { expect(ids).toContain("notebook") expect(ids).toContain("compute_job") + expect(ids).toContain("artifact") + expect(ids).toContain("harness") + expect(ids).toContain("claim") expect(ids).not.toContain("query_uniprot") } }, @@ -33,6 +36,8 @@ describe("tool registry agent boundaries", () => { const ids = tools.map((tool) => tool.id) expect(ids).toContain("notebook") + expect(ids).toContain("harness") + expect(ids).toContain("claim") expect(ids).toContain("query_uniprot") }, }) diff --git a/docs/HARNESS.md b/docs/HARNESS.md new file mode 100644 index 00000000..c884e013 --- /dev/null +++ b/docs/HARNESS.md @@ -0,0 +1,341 @@ +# Scientific research harness + +OpenScience ships a generic, evaluation-bound harness for scientific reasoning, discovery, implementation, and verification. It does not ship benchmark repositories, source pins, dataset manifests, launch recipes, runner scripts, local environment files, or score tables. + +The product boundary is deliberate: + +- OpenScience owns the agent runtime, immutable research contract, orchestration, search, working memory, evidence rules, verification protocols, and reports. +- A caller-owned local or external evaluator owns tasks, data, hidden state, execution environments, scoring, evaluator credentials, and final result submission. +- The agent never receives the evaluator capability. OpenScience stores only its hash. +- Self-reported progress can guide search but cannot become verified evidence. + +This keeps private and local evaluation setup out of the product while allowing any compatible evaluator to bind a run. + +## System model + +```mermaid +flowchart LR + E["Local or external evaluator"] -->|"bind immutable contract"| C["Harness contract"] + C --> P["Profile and methodology routing"] + C --> O["Scientific orchestration"] + C --> S["Adaptive candidate search"] + P --> A["Agent session"] + O --> A + S --> A + A -->|"provisional artifacts and observations"| S + A --> W["Session-local world model"] + E -->|"authenticated evaluations and evidence"| V["Verification journal"] + V --> S + V --> W + V --> R["Quality, cost, provenance, and claim reports"] +``` + +The direct ReAct path remains the default for ordinary work. Additional machinery activates only when the bound contract and inferred task profile require it. + +## Immutable run contract + +`HarnessAdapter.bind` accepts a generic caller-defined evaluation identity: + +- `benchmark`: an opaque local or external suite identifier; +- `title`, `family`, and `task`: human-readable routing context; +- task version, task ID, and split; +- evaluator name, version, source class, and secret capability; +- objective, primary metric, direction, optional target, and optional secondary objectives; +- model, tools, skills, budgets, seed, and intervention mode; +- contamination policy with `hiddenTestsAccessible: false`; +- optional orchestration, search, audit, simulation, proof, replication, confirmation, and other verification protocols; +- generic methodology packs. + +No identifier is resolved through a built-in catalog. Arbitrary caller-owned suite names are accepted. The adapter defaults to the neutral `custom` family and `react` profile when the caller does not declare them. + +The stored contract is immutable. A second, byte-different bind for the same session is rejected. Evaluator, auditor, semantic-reviewer, claim-evaluator, and meta-harness capabilities are distinct, hashed, and timing-safe compared. + +Numeric optimization requires: + +- a named primary metric; +- an explicit maximize or minimize direction; +- a finite candidate budget; +- an `optimize` profile. + +Secondary objectives require a preflighted objective-audit commitment. They never silently replace the primary metric. + +## Continual world model + +Each bound session has a small, editable world model inspired by reset-free continual-agent systems. + +The model stores up to 48 typed entries: + +- hypotheses; +- observations; +- strategies; +- memories; +- reusable skills; +- subagent roles. + +Every entry has a stable key, content hash identity, confidence from 1 to 5, provenance-tagged evidence, update revision, and timestamp. + +Confidence is authority-gated: + +- agent-authored entries are capped at confidence 3; +- confidence 4 requires evidence beyond self-report; +- confidence 5 requires at least two non-self references, including evaluator or human evidence; +- evaluator-backed confidence can enter only through an authenticated evaluator route. + +The base prompt is immutable and content-addressed from the contract. Refinement changes only the supplemental working state. + +### Event boundaries + +Events distinguish reasoning from changes in the external world: + +- `analysis` preserves the current context epoch; +- tool results, evaluations, failures, milestones, stagnation, and manual events may advance it; +- failures, milestones, stagnation, and manual boundaries request immediate refinement; +- six events without refinement request a periodic refinement. + +This allows a reasoning chain to continue through analysis calls while rebuilding mutable context only after relevant state changes. + +### Refinement and rollback + +A refinement: + +- requires the exact current revision; +- accepts at most six small upserts or removals; +- limits added content to 12,000 characters; +- records the trigger and referenced evidence; +- snapshots the prior state; +- increments the context epoch; +- rejects stale concurrent writes. + +The harness tool exposes self-attributed `world_status`, `world_event`, `world_refine`, and `world_rollback` actions. The evaluator API can apply stronger, authenticated refinements. Rollback restores a content-verified snapshot without changing the immutable base prompt. + +The world model is injected into scientific agent prompts as escaped, bounded, explicitly mutable context. It never overrides higher-authority external evidence. + +## Scientific profiles and methodology packs + +Profiles select the lightest useful reasoning mode: + +- `react`; +- `optimize`; +- `reproduce`; +- `theory`; +- `numerical`; +- `training`; +- `forecast`. + +Methodology packs are composable product checks, not evaluator setup: + +- statistics; +- biology; +- physics; +- PDE and numerical simulation; +- chemistry and materials; +- machine learning; +- forecasting; +- formal proof. + +Each pack defines blocking and advisory checks. A passing external evaluation must include every blocking check selected by the contract, mark it blocking, and attach evidence. Duplicate, missing, failed, or evidence-free blocking checks are rejected. + +## Adaptive scientific orchestration + +`HarnessOrchestrator` derives task traits for decomposability, sequentiality, tool intensity, uncertainty, verification risk, novelty, and cross-domain scope. It selects one bounded topology: + +- solo; +- centralized review; +- fork/join; +- tournament; +- evolutionary rounds; +- verifier loop. + +The topology, reasons, worker limit, round limit, and independent-verifier requirement are persisted. Work units form a restart-safe DAG. + +Task execution is receipt-bound. A worker completion must match: + +- the exact work ID; +- assigned agent; +- canonical prompt; +- child session; +- measured tool and usage telemetry; +- artifact and evidence references; +- timestamps and outcome. + +Producer lanes may resume only where the backend explicitly permits it. Critics, rankers, investigators, and verifiers remain fresh. Verification panels are blinded; one verdict cannot establish consensus. + +Verifier-loop repair is backend-routed from structured verdicts. Adaptive evolution pauses at evaluator-authenticated marginal-utility checkpoints, so workers cannot self-score their way into more budget. + +## Adaptive candidate search + +`HarnessSearch` persists a content-addressed candidate graph with deterministic quality-diversity islands. It supports: + +- independent roots; +- exploitation; +- lineage fusion; +- cross-island migration; +- paradigm divergence; +- branch and artifact deduplication; +- bounded parallel reservations. + +Every recommendation is a lease bound to the exact state revision, lineage, search mode, island, and verified context. Stale leases are rejected transactionally. + +Parallel dispatch reserves candidate-budget slots before work begins. Each reservation receives a distinct variation mandate. A reservation can be consumed once, released on failure, or automatically released when it rediscovers known artifact bytes. + +Search state distinguishes: + +- provisional agent observations; +- externally verified evaluations; +- final and non-final fidelity stages; +- Pareto archive membership; +- target attainment; +- stagnation and budget exhaustion. + +Only externally authenticated final evaluations can: + +- become the best candidate; +- enter the verified archive; +- authorize parent or inspiration lineages; +- populate retrospective memory; +- satisfy the run target. + +## Evidence and verification protocols + +The harness includes composable protocols for difficult scientific claims. + +### Active audit + +The evaluator commits an opaque probe pool. The backend selects probes using uncertainty reduction, failure UCB, failure-region diversity, and stratum coverage. Probe identities remain capability-protected. A terminal receipt binds the pool, observations, estimate, stopping state, transfer qualification, and audited artifact. + +### Failure discovery + +Topic-aware adversarial generation allocates attempts through deterministic UCB1. Correctness, topic fit, novelty, and target outcome are frozen before attempts. Generated cases can reveal failures but cannot silently alter the original population estimate. + +### Runtime integrity + +Integrity receipts bind the evaluated subject to an evaluator-owned event trace. The backend checks continuity, hidden-boundary violations, candidate mutation, evaluator identity, and exact terminal state. + +### Evolution trace + +Candidate lineage captures exact parent and child snapshots, source-file limits, canonical line hashes, changed-line counts, reintroduction, and cycles. Novelty diagnostics guide search but never become fitness. + +### Evaluator qualification + +An independent auditor uses a committed hidden fault suite to measure evaluator discrimination and calibration. The audited evaluator cannot authenticate its own qualification. + +### Semantic audit and clean-room synthesis + +Independent semantic review separates numerical success from scientific meaning. Clean-room synthesis isolates answer facts, checks citations and claim support, and binds a factuality receipt without leaking reviewer capabilities into the agent session. + +### Simulation validation + +Simulation contracts pin engine, command/config hashes, problem identity, reference solution, convergence levels, expected order, residual tolerance, invariants, and stress tests. Visual plausibility alone cannot pass. + +### Formal proof + +Formal validation binds the exact statement, proof relation, toolchain, dependency closure, source policy, transitive axiom closure, and verification tier. Compiler success alone is insufficient, and a proof of a repaired statement cannot masquerade as an exact proof. + +### Replication and sealed confirmation + +Replication freezes independent sampling units, strata, environments, and aggregation before results. Sealed confirmation separates search fitness from terminal claim evaluation. Cherry-picked replicates and agent-authored success claims cannot authorize promotion. + +### Human-AI autonomy + +Autonomy receipts derive provenance from a hash-chained interaction and artifact trace. The coarse intervention label is not treated as proof of autonomy. + +### Controlled interventions and ablations + +Matched replay, retuning, component ablation, repair, and transfer pairs are frozen before final evaluation. The backend verifies that paired contracts differ only in declared factors and derives direction-aware effects and intervals. + +### Meta-harness qualification + +Prompts, memories, skills, tools, middleware, subagents, and scaffolds can be treated as candidate harness components. Qualification uses frozen tasks, model panels, activation-required cases, cost accounting, and independent evidence. Meta-harness outcomes cannot self-authorize their own deployment. + +## Retrospective memory and learned skills + +Verified candidate outcomes enter task-scoped retrospective memory with evaluator identity, proposal, feedback, metrics, artifacts, and evidence. Retrieval returns a bounded mixture of relevant successes and failures. The prompt labels these as precedents, not instructions. + +Learned skills follow a quarantine pipeline: + +1. create an inactive proposal; +2. preserve exact content and origin; +3. attach paired held-out candidate/control evidence; +4. verify compatible immutable contracts and evaluator authority; +5. promote only an unchanged proposal that meets the qualification policy. + +An agent cannot activate a skill by writing convincing prose about its own success. + +## Evaluator interface + +The core generic endpoints are: + +| Method | Path | Purpose | +| ------ | ---------------------------------------------------- | --------------------------------------------- | +| `POST` | `/harness/runs` | Bind an immutable caller-owned evaluation run | +| `GET` | `/harness/runs/:sessionID/contract` | Read the bound contract | +| `POST` | `/harness/evaluations` | Record an authenticated result | +| `GET` | `/harness/runs/:sessionID/evaluations` | Read the evaluation journal | +| `GET` | `/harness/runs/:sessionID/world` | Read continual working state | +| `POST` | `/harness/runs/:sessionID/world/refinements` | Apply evaluator-backed world-state evidence | +| `POST` | `/harness/runs/:sessionID/orchestration` | Initialize orchestration | +| `GET` | `/harness/runs/:sessionID/orchestration` | Resume orchestration state | +| `POST` | `/harness/runs/:sessionID/orchestration/checkpoints` | Submit external utility evidence | +| `GET` | `/harness/runs/:sessionID/report` | Build a quality-cost report | +| `POST` | `/harness/compare` | Compare compatible runs | + +Additional routes expose the product verification protocols described above. Every capability-protected write validates its bound contract before mutation. + +## Local storage + +Harness state is stored under the OpenScience data directory, separated by protocol: + +- contracts and hashed capability bindings; +- candidate search and orchestration state; +- continual world models; +- evaluation journals; +- audit, integrity, evolution, simulation, proof, replication, confirmation, and intervention receipts; +- retrospective memory; +- learned-skill proposals and qualification evidence; +- reports. + +Writes use validated JSON state and revision checks. Content-addressed receipts make evaluator evidence replayable and resistant to substitution. + +## Security and scientific integrity invariants + +- Hidden evaluator state is never part of the agent contract. +- Evaluator secrets are hashed and never returned. +- Independent roles require independent capabilities. +- Agent observations remain provisional. +- High-confidence working beliefs require non-self evidence. +- The base prompt and run contract are immutable. +- Final promotion requires every bound blocking protocol. +- Candidate bytes, evidence references, and receipt identities are content-addressed where applicable. +- Search novelty, orchestration completion, and model confidence are not scientific proof. +- Reports separate measured results from qualified claims. + +## Deliberately absent + +This repository intentionally contains no: + +- benchmark catalog; +- upstream repository pins; +- dataset download or mount instructions; +- source-audit manifest; +- execution recipe; +- launcher or pilot runner; +- local secret or environment template; +- benchmark-specific result table; +- claim that the harness is state of the art before external measurement. + +Those belong in private or local evaluation workspaces. OpenScience remains the reusable product. + +## Verification + +From `backend/cli`: + +```bash +bun run typecheck +bun test +``` + +When the API changes, regenerate the JavaScript SDK from the repository root: + +```bash +./tooling/repo/generate.ts +``` diff --git a/docs/notes/clean-room-scientific-synthesis.md b/docs/notes/clean-room-scientific-synthesis.md new file mode 100644 index 00000000..5fbbc4d3 --- /dev/null +++ b/docs/notes/clean-room-scientific-synthesis.md @@ -0,0 +1,27 @@ +# Clean-room scientific synthesis + +## Why this protocol exists + +Long-form scientific conclusions are not adequately graded by a single answer-level similarity score. A system can retrieve the hidden review, copy a plausible conclusion, omit important reference facts, or benefit from a judge/provider failure while still producing an attractive scalar. OpenScience therefore treats clean-room provenance and atomic-fact scoring as one evaluator-owned protocol. + +The immediate methodological source is a 2026 clean-room scientific-synthesis study built from systematic reviews. Its setting blocks answer-title matches and post-publication evidence. It decomposes conclusions into atomic facts, measures factual precision as support rate multiplied by one minus contradiction rate, measures reference-fact recall, and reports their harmonic-mean F1. The reported clean-room gap makes answer-key retrieval a first-class evaluation threat rather than a footnote. + +## OpenScience protocol + +`scientific-synthesis-v1` freezes before execution: + +- salted commitments for the hidden conclusion and sorted atomic reference facts; +- the public-question hash, publication cutoff, canonical retrieval-tool subset, event budget, trace schema, and filter policy; +- distinct decomposer, precision-judge, and recall-judge prompt commitments plus frozen configurations; +- minimum generated facts, factual precision, recall, and F1; and +- a separately qualified evaluator suite covering wrong answers, unsupported claims, and data leakage. + +The external evaluator keeps the answer, fact text, salt, prompts, raw blocked outputs, and bearer capabilities private. It submits a complete tool trace and atomic judgments. The backend replays source decisions, verifies manifests and chronology, derives all counts and scores, and freezes one receipt per run or candidate. Unknown dates, post-cutoff sources, forbidden domains, reference-title matches, and repeated results are blocked. Decomposer or judge failures yield `inconclusive`; they are never converted to ordinary unsupported or missed facts. + +The `run-clean-room-synthesis` native skill constructs token-free commitments and validates the private manifest. A passing benchmark result must cite the canonical receipt, and its score must exactly equal backend-derived F1. + +## Honest boundary + +This is a reusable product protocol, not a benchmark score or a claim of state-of-the-art performance. Evaluator execution, private data, and scoring remain outside OpenScience. + +Hashes and authentication prove identity, immutability, provenance, and arithmetic. They do not prove that an atomic fact is scientifically true. That remains the responsibility of the qualified judges, their evidence, and benchmark-owner acceptance of the comparison protocol. diff --git a/docs/notes/formal-proof-trust.md b/docs/notes/formal-proof-trust.md new file mode 100644 index 00000000..170fb61f --- /dev/null +++ b/docs/notes/formal-proof-trust.md @@ -0,0 +1,98 @@ +# Formal proof trust receipts + +Date: 2026-08-05 + +## Decision + +Add an opt-in `formal-proof-v1` contract and evaluator-authenticated receipt +for Lean 4 results. Bind theorem identity, proof relation, exact artifact, +complete source and environment manifests, verifier identities, source policy, +transitive axiom policy, and the chosen trust tier before proof search. Never +promote an exact refutation or repaired statement as a proof of the original +challenge. + +The tiers are `kernel`, `fresh_recheck`, and `external_crosscheck`. Strict +public benchmark and research claims should use the last tier. A passing final +evaluation must cite the sole canonical passing receipt for the exact run or +candidate bytes. + +## Research basis + +DeepMind's [LEAP](https://arxiv.org/abs/2606.03303) couples decomposition and +blueprints with compiler feedback and reports 70% on Lean-IMO-Bench plus 12/12 +on the 2025 Putnam problems. [AlphaProof +Nexus](https://arxiv.org/abs/2605.22763) combines parallel Lean agents with +evolutionary coordination for open mathematical research. The open +[Goedel-Architect](https://arxiv.org/abs/2606.06468) makes lemma dependencies +explicit and uses failed proof obligations to refine the blueprint. These +systems motivate stronger search, but their search traces are not substitutes +for result-side verification. + +Lean's official [proof-validation +guide](https://lean-lang.org/doc/reference/latest/ValidatingProofs/) describes +an escalating ladder from ordinary builds through axiom inspection and fresh +rechecking to a sandboxed gold-standard comparison with an external checker. +Lean [issue #8840](https://github.com/leanprover/lean4/issues/8840) shows why a +plain `#print axioms` result is insufficient in a hostile setting: dependencies +can occur in axiom types. The protocol therefore requires transitive traversal +of both proof bodies and axiom types. + +The [Formal Conjectures](https://github.com/google-deepmind/formal-conjectures) +project uses immutable benchmark snapshots tied to a Lean version. The +kernel-checked [Erdős problem exclusion +result](https://arxiv.org/abs/2607.25628) demonstrates the value of a CI gate +that excludes `sorry`, unchecked native decisions, and solver additions to the +trusted base. The protocol freezes a source auditor and rejects `sorry`, +`admit`, `debug.skipKernelTC`, and `native_decide`, while separately enforcing +the axiom closure. + +[MechGeo](https://arxiv.org/abs/2608.02295) formally refutes two geometry +statements in Lean-IMO-Bench and proves corrected replacements. That result is +the reason claim relation is part of receipt identity: repaired formalization +is valuable evidence, but it is not an exact benchmark solve. + +## Backend derivation + +The evaluator supplies hashes and transcripts, never raw hidden challenges or +capabilities. OpenScience recomputes and binds: + +- exact challenge, statement, declaration, module, relation, and artifact; +- one canonically ordered complete manifest containing challenge, statement, + proof, toolchain, Lake manifest, and dependency closure; +- distinct frozen verifier artifacts required by the selected tier; +- a complete source audit with no forbidden construct findings; +- a complete, canonically ordered transitive axiom inventory whose types were + traversed and whose entries all appear in the frozen allowlist; +- warning-free kernel acceptance, optional fresh replay, and optional exact + sandboxed comparison accepted by Lean plus an external checker; and +- subject creation and verification timestamps that prevent a post-hoc or + cross-candidate receipt. + +The receipt is content-addressed without its recording timestamp. A subject +can freeze only one receipt, including a failed one, so unfavorable verification +cannot be replaced by a later retry. + +## Attacks covered + +- Swap in an easier statement or silently prove a repaired theorem. +- Attach a valid proof from another task, run, candidate, or Lean environment. +- Omit support files or change dependency, toolchain, or verifier bytes. +- Pass a build containing `sorry`, `admit`, `debug.skipKernelTC`, or + `native_decide`. +- Hide a custom axiom through a dependency in an axiom's type. +- Claim fresh or independent verification using the ordinary build process. +- Reuse a failed subject with a favorable second receipt. +- Record proof evidence after the final benchmark evaluation. + +## Explicit limits + +Artifact commitments prove identity inside the evaluator boundary; they do not +make an unqualified evaluator honest. The kernel and fresh tiers still depend +on Lean's implementation and the frozen audit wrappers. The external tier +reduces correlated implementation risk but does not remove it. + +Most importantly, a kernel can validate a vacuous or mistranslated statement. +Formal acceptance establishes only the frozen Lean proposition relative to its +reported axioms and environment. Informal correspondence, definition quality, +novelty, significance, benchmark comparability, and SOTA all need separate +semantic and empirical evidence. diff --git a/docs/notes/human-ai-autonomy.md b/docs/notes/human-ai-autonomy.md new file mode 100644 index 00000000..ec796711 --- /dev/null +++ b/docs/notes/human-ai-autonomy.md @@ -0,0 +1,82 @@ +# Human-AI autonomy receipts + +Date: 2026-08-05 + +## Decision + +Treat benchmark autonomy as a provenance result derived from a complete, +evaluator-owned interaction trace. Do not accept the existing caller-supplied +`autonomous` or `human_reprompted` field as evidence of who produced the +scientific content. + +The opt-in `human-ai-autonomy-v1` protocol predeclares a claimed contribution +level, evaluator-runtime recorder artifact, trace schema, semantic +classification policy, raw-retention and disclosure policies, and event +ceiling. A passing final evaluation must cite the single canonical receipt for +the exact run or candidate artifact. + +## Research basis + +Google DeepMind's [Towards Autonomous Mathematics +Research](https://arxiv.org/abs/2602.10177) defines three contribution levels: +primarily human (core content human-generated), human-AI collaboration (both +contribute essentially), and essentially autonomous (core content AI-generated +without essential human intervention). It treats posing the question, +exposition, and genuinely minor corrections as compatible with essentially +autonomous work, acknowledges ambiguous cases, and proposes Human-AI +Interaction cards plus access to important raw prompts and outputs for +essential contributions. + +The same paper warns that the evaluation gap encourages misleading autonomy +claims. DeepMind's [Conjecture Machines and the new validation bottleneck in +science](https://deepmind.google/public-policy/conjecture-machines-ai-agents-and-the-new-validation-bottleneck-in-science/) +argues that independent validation becomes the limiting resource as agents +generate more hypotheses. Together these motivate a capability-separated +recorder and a fail-closed promotion gate, not a self-authored model card. + +## Backend derivation + +Actors are `benchmark`, `human`, and `agent`. Events use one of ten observable +interaction kinds and one contribution class: `problem`, `auxiliary`, +`essential`, `core`, or `unclear`. + +- Substantive agent content with no substantive human content derives + `essentially_autonomous`. +- Substantive human and agent content derives `human_ai_collaboration`. +- Substantive human content with only auxiliary agent content derives + `primarily_human`. +- Any unclear classification yields `inconclusive`. + +The backend additionally requires a frozen problem statement, at least one +agent event, an exact contract-start timestamp, contiguous and monotonic events, +a real event-ID hash chain, a continuous artifact transition chain, and a last +transition equal to the evaluated artifact. Candidate traces must enclose the +server-recorded candidate creation time and match the candidate's registered +artifact SHA-256. + +## Attacks covered + +- Relabel an essential human strategy hint as an autonomous pass. +- Supply only the favorable suffix of a run after human guidance. +- Reorder, delete, or alter events after receipt creation. +- Attach a receipt from another run, candidate, or artifact. +- Link the expected artifact once and then apply an unreported final edit. +- Submit a future or post-hoc trace or evaluate before receipt creation. +- Replace a failed or inconclusive trace with a favorable retry. +- Let the candidate call receipt routes without the evaluator capability. + +## Explicit limits + +Cryptographic commitments prove byte identity and ordering inside the captured +boundary. They do not prove that the evaluator omitted no off-platform human +conversation, or that an `auxiliary` versus `essential` judgment is +semantically correct. Credible use therefore still requires an evaluator-owned +recorder, controlled communication boundary, retained raw prompts and outputs, +and qualified expert review. The receipt reports contribution provenance; it +does not establish scientific correctness, novelty, significance, or benchmark +SOTA. + +`public_essential_after_release` can commit a run to disclose essential +interactions once hidden material may safely be released. Until then, +`evaluator_retained` keeps raw benchmark content private while preserving +auditable hashes and evidence references. diff --git a/docs/notes/proactive-evaluation.md b/docs/notes/proactive-evaluation.md new file mode 100644 index 00000000..a2c8a67f --- /dev/null +++ b/docs/notes/proactive-evaluation.md @@ -0,0 +1,89 @@ +# Proactive evaluation: research basis and harness boundary + +This note records the research basis for `proactive-audit-v2`. It deliberately +separates mechanisms supported by ProEval from controls added by OpenScience. +The latter are security and provenance requirements for an adversarial benchmark +harness, not claims made by the paper. + +## Primary sources + +- DeepMind, [ProEval: Proactive Failure Discovery and Efficient Performance + Estimation for Generative AI Evaluation](https://deepmind.google/research/publications/238239/) +- Hiranandani et al., [ProEval, arXiv:2604.23099v2](https://arxiv.org/html/2604.23099v2) +- Google DeepMind, [ProEval reference implementation](https://github.com/google-deepmind/proeval) +- DeepMind, [Conjecture Machines and the validation bottleneck in + science](https://deepmind.google/public-policy/conjecture-machines-ai-agents-and-the-new-validation-bottleneck-in-science/) + +## Directly supported by ProEval + +ProEval models per-example scores with a Gaussian process. Historical source +model score profiles can define a score-feature prior: the target prior mean is +the source-model mean and the prior covariance between two examples is their +empirical covariance across source models. Performance estimation is Bayesian +quadrature over the evaluation population, with acquisition based on reduction +in posterior integral variance. Because that acquisition does not depend on the +observed target scores, a batch can be selected before its outcomes are known. + +Failure discovery uses a probabilistic superlevel-set acquisition: prioritize +points whose upper confidence bound crosses a predeclared failure threshold and +whose posterior uncertainty remains high. ProEval also describes source-profile +selection with dimensionality reduction and mixture clustering to reduce +negative transfer. Its implementation abstains from transfer when too few +source models are available in the selected cluster. Generated failure cases +are a separate mechanism: seeded generation anchors likely failures, while +topic-aware generation separates failure patterns from topics and uses a bandit +to allocate generation effort. + +These mechanisms support four design invariants: + +1. A score-history prior is derived from a frozen matrix; it is not a caller- + authored mean plus arbitrary features. +2. Population estimation and targeted failure discovery are different + acquisitions over the same committed static pool. +3. Generated/adversarial cases are useful for discovery but are not samples + from the benchmark population and cannot enter its quadrature estimate. +4. Transfer requires qualified source profiles; insufficient or mismatched + sources require abstention or a cold-start path. + +## OpenScience engineering extrapolations + +The paper does not define a hostile multi-tenant receipt protocol. OpenScience +therefore adds the following controls: + +- The contract freezes the exact committed probe pool, source-model identities, + source-score manifest, source-selection artifact, selection method, + calibration size, and rejection threshold before target observations exist. +- The evaluator sends only committed probe metadata and source loss vectors. + OpenScience derives the mean and covariance features and never receives hidden + probe bytes. +- Initial calibration probes are selected by an outcome-independent digest + order. If their mean absolute prior error exceeds the frozen threshold, + transfer is rejected, acquisition falls back to an outcome-independent order, + and the population estimate permanently abstains. +- A completed audit is promoted only through a content-addressed receipt bound + to the contract, exact subject artifact, committed pool, terminal revision, + derived estimate, and timestamps. A passing final evaluation may cite that + receipt only when the contract explicitly requires it and the receipt is + completed, transfer-qualified, and non-abstaining. +- Synthetic failure-generation streams, when implemented, must use separate + commitments, budgets, observations, and receipts. They may create robustness + evidence but cannot change the official population score estimate. + +The calibration rule is a conservative harness guard, not a proof that accepted +transfer is statistically correct. The receipt proves protocol completion and +provenance, not state of the art. Benchmark claims still require held-out runs, +baselines, uncertainty reporting, and independent reproduction. + +## Acceptance tests + +- Changing a source loss changes the pool commitment and audit identity. +- A v2 caller cannot provide or override derived prior means or covariance + features. +- Calibration order is unchanged when observed target losses change. +- High calibration error produces `rejected` transfer and a permanently + abstaining estimate. +- A terminal receipt fails after any content mutation and cannot be replayed + across contracts, subjects, pools, or timestamps. +- A passing final evaluation cannot omit a required audit receipt or cite a + rejected, incomplete, abstaining, future, or mismatched receipt. +- `active-audit-v1` remains backward compatible. diff --git a/docs/notes/topic-aware-failure-discovery.md b/docs/notes/topic-aware-failure-discovery.md new file mode 100644 index 00000000..14d18e27 --- /dev/null +++ b/docs/notes/topic-aware-failure-discovery.md @@ -0,0 +1,97 @@ +# Topic-aware adversarial failure discovery + +This note records the research and adversarial design boundary for +`topic-aware-failure-v1`. It distinguishes mechanisms supported by ProEval from +the receipt, capability, and contamination controls added by OpenScience. + +## Primary sources inspected + +- Huang, Zeng, Kumaresan, and Wang, [ProEval: Proactive Failure Discovery and + Efficient Performance Estimation for Generative AI + Evaluation](https://arxiv.org/html/2604.23099v2), arXiv v2, 1 June 2026. +- Google DeepMind, [ProEval source + repository](https://github.com/google-deepmind/proeval), inspected at commit + `8c0422db6a5b6d655712bae63e852e90219f7ddc` (16 June 2026). +- DeepMind, [Conjecture Machines and the validation bottleneck in + science](https://deepmind.google/public-policy/conjecture-machines-ai-agents-and-the-new-validation-bottleneck-in-science/). + +## Directly supported mechanisms + +ProEval separates population performance estimation from failure discovery. +For generated discovery, SS-Gen selects likely-failure anchors using the +superlevel-set acquisition and asks an LLM to create a harder case with a +similar failure pattern. TSS addresses semantic collapse by separating the +failure pattern from a target topic. Topics come from BERTopic or a predefined +set; UCB1 treats topics as arms; topic choice is explicitly independent of the +anchors; and the generator transposes the anchor pattern into the selected +topic. + +The paper measures cumulative failures, failure rate, samples to first failure, +normalized topic entropy, and embedding log-determinant diversity. It reports +that topic-aware/anchored synthesis can find substantially more and more diverse +failures than random generation under fixed budgets. These are empirical +results on the evaluated datasets and models, not a universal SOTA guarantee. + +The paper also calls out generator validity as a limitation: a synthesized hard +question may itself be wrong. Its experiment controls the generator and +temperature across methods, which supports relative comparisons but does not +authenticate an individual generated case in a hostile benchmark setting. + +## Reference-code audit + +The inspected repository is useful executable research, but its generator loop +is not a receipt protocol. In the inspected commit: + +- `generate()` increments a topic's total count; +- the experiment calls `generator.update(score)` after target evaluation; +- `update(score)` updates the GP posterior but not the topic failure count; and +- the separate deprecated `update_stats()` is not called by that experiment and + increments failures when `score == 0.0`, while the experiment documents + `1.0` as an error/failure. + +Consequently, copying that path verbatim would not establish that UCB rewards +track observed failures. OpenScience recomputes arm pulls and rewards from the +stored attempt journal instead of trusting mutable generator state. + +## OpenScience engineering extrapolations + +The following controls are OpenScience additions, not claims made by ProEval: + +1. A bound contract freezes the source-audit pool, sorted opaque topics whose + hidden definitions use private salted commitments, + topic-model/generator/validator/embedding identities, budget, UCB constant, + threshold, and anchor count before generation. Generator/validator + separation uses prompt/config commitment pairs rather than relabelable names. +2. A stream can start only from a valid terminal active-audit receipt for the + exact subject. Anchors are derived from authenticated observed failures. +3. Every topic is forced once before UCB1 exploitation or target-based + termination. Later choices and ties are deterministic and replayed from + immutable observations; concurrent selection retries receive the same + server lease. +4. Every attempt consumes budget. Only an independently correctness-, topic-, + and novelty-valid case with a threshold-consistent target failure earns + reward. Exact duplicates cannot pass novelty. +5. The backend derives topic entropy and dimension-bounded embedding log-det + from frozen-model embeddings. It never receives hidden case bytes. +6. A content-addressed terminal receipt transitively revalidates its source + audit and derives journal semantics again, so recomputing a hash cannot turn + a threshold-inconsistent label or caller-authored reward into evidence. It + is robustness evidence only and cannot affect the official score or + population estimate. + +## Acceptance tests + +- A caller cannot choose a topic, substitute anchors, or spend a selection + twice. +- Every topic is selected once before reward-dependent allocation. +- Invalid, failed, inconclusive, duplicate, or threshold-inconsistent cases do + not earn reward; invalid attempts still consume budget. +- Validator order cannot change the attempt identity, and generator/validator + identities must be distinct. +- State, receipt, or source-audit tampering invalidates the terminal receipt. +- Adding a failure-discovery receipt leaves the active-audit state and official + score byte-for-byte unchanged. + +The receipt proves that this protocol ran. Credible SOTA claims still require +held-out benchmark execution, declared baselines, repeated runs, uncertainty, +cost reporting, and independent reproduction. diff --git a/docs/notes/verifier-grounded-proof-blueprints.md b/docs/notes/verifier-grounded-proof-blueprints.md new file mode 100644 index 00000000..538c8155 --- /dev/null +++ b/docs/notes/verifier-grounded-proof-blueprints.md @@ -0,0 +1,77 @@ +# Verifier-grounded proof blueprints + +## Design question + +How can OpenScience borrow the strongest recent formal-proof search mechanisms +without letting a search controller, language-model reviewer, or successful +partial build impersonate final proof evidence? + +## Primary-source findings + +- [LEAP](https://arxiv.org/abs/2606.03303) turns a theorem into a bipartite + AND/OR DAG. It tries a direct proof, compiles a sketch of the parent assuming + only newly proposed lemmas, treats each decomposition as an AND node, treats + each goal as an OR node, shares reusable lemmas, checks acyclicity, and uses + reviewer judgments to reject bad decompositions before expanding them. +- [Goedel-Architect](https://arxiv.org/abs/2606.06468) preserves proved lemmas + and refines around diagnosed failures. Its useful transferable mechanism is + failure-local repair; OpenScience implements that as a new immutable + alternative branch rather than rewriting or dropping an existing lemma. +- [AlphaProof Nexus](https://arxiv.org/abs/2605.22763) runs independent prover + workers, invokes Lean after edits, and separates proof search from SafeVerify + checks for statement or environment exploits. This motivates evaluator-owned + bounded leases and a separate final verifier. +- [Self-Modifying Lean Proof Agents](https://arxiv.org/abs/2607.17352) allows + the machine-readable proof context to evolve while keeping verifier + grounding fixed. OpenScience therefore versions the graph representation but + freezes all evaluator artifacts and final authority in the run contract. +- [OProver](https://arxiv.org/abs/2605.17283) emphasizes compiler-verified proof + and repair trajectories at scale. This supports preserving negative + compiler feedback as reusable search state rather than retaining only wins. + +These systems improve search. None eliminates the need to freeze theorem +identity, audit unchecked constructs and transitive axioms, or replay the final +artifact under the selected trust tier. + +## OpenScience protocol + +The optional `proof-blueprint-v1` contract freezes five distinct content +artifacts: graph schema, Lean compiler, sketch validator, decomposition +reviewer, and reviewer rubric. The compiler is exactly the `lean_kernel` +artifact already bound by `formal-proof-v1`. It also freezes hard limits for +goals, graph depth, parallel leases, direct attempts per goal, refinements per +goal, and lease lifetime. + +The backend initializes one root goal from the exact formal statement. A goal +identity hashes `(statementSHA256, declaration, module)`, which makes lemma +reuse explicit. A decomposition is admitted only after a direct attempt and +only when: + +1. the frozen compiler checked the exact leased parent without warnings; +2. the frozen sketch validator found placeholders exactly equal to the new + child declarations; and +3. the frozen reviewer marked the branch relevant, easier, and plausible. + +Reviewer failure is retained but does not create graph nodes. Verifier failure +is retained. Accepted branches and proved goals are never mutated. A blocked +branch makes its parent eligible for another bounded alternative. Every write +atomically revalidates content identities, complete contiguous histories, +lease provenance, reachability, acyclicity, longest depth, node count, and per- +goal budgets. + +## Authority boundary + +Blueprint state is search provenance. `proved` means that direct accepted +attempts and closed AND nodes form a route to the root. It does not imply a +complete source audit, transitive axiom audit, fresh kernel replay, independent +checker agreement, or semantic correspondence to the informal problem. + +Consequently: + +- blueprint status is included in reports as execution metadata; +- it never supplies `proofReceiptID`; +- it cannot make a final evaluation pass; and +- the exact root artifact must still pass `formal-proof-v1`. + +This boundary makes the architecture useful for proof search and safe to +ablate without overstating what has been formally established. diff --git a/frontend/workspace/src/atlas/session-trace-model.test.ts b/frontend/workspace/src/atlas/session-trace-model.test.ts index d2b31dcd..f07a6900 100644 --- a/frontend/workspace/src/atlas/session-trace-model.test.ts +++ b/frontend/workspace/src/atlas/session-trace-model.test.ts @@ -30,6 +30,7 @@ const trace: SessionTraceResponse = { retryCount: 1, }, turns: [], + profiles: [], inference: [ { messageID: "msg_assistant", diff --git a/tooling/sdk/js/src/v2/gen/sdk.gen.ts b/tooling/sdk/js/src/v2/gen/sdk.gen.ts index 673eb193..1baa6774 100644 --- a/tooling/sdk/js/src/v2/gen/sdk.gen.ts +++ b/tooling/sdk/js/src/v2/gen/sdk.gen.ts @@ -91,6 +91,123 @@ import type { GlobalProjectCreateErrors, GlobalProjectCreateResponses, GlobalSyncResponses, + HarnessAblationAssessErrors, + HarnessAblationAssessResponses, + HarnessAblationInitializeErrors, + HarnessAblationInitializeResponses, + HarnessAuditInitializeErrors, + HarnessAuditInitializeResponses, + HarnessAuditObserveErrors, + HarnessAuditObserveResponses, + HarnessAuditSealErrors, + HarnessAuditSealResponses, + HarnessAuditSelectErrors, + HarnessAuditSelectResponses, + HarnessAuditStatusErrors, + HarnessAuditStatusResponses, + HarnessAutonomyReceiptErrors, + HarnessAutonomyReceiptResponses, + HarnessAutonomyRecordErrors, + HarnessAutonomyRecordResponses, + HarnessBindErrors, + HarnessBindResponses, + HarnessBlueprintInitializeErrors, + HarnessBlueprintInitializeResponses, + HarnessBlueprintLeaseErrors, + HarnessBlueprintLeaseResponses, + HarnessBlueprintRecordErrors, + HarnessBlueprintRecordResponses, + HarnessBlueprintStatusErrors, + HarnessBlueprintStatusResponses, + HarnessCompareErrors, + HarnessCompareResponses, + HarnessConfirmationReceiptErrors, + HarnessConfirmationReceiptResponses, + HarnessConfirmationRecordErrors, + HarnessConfirmationRecordResponses, + HarnessConfirmationSelectionErrors, + HarnessConfirmationSelectionResponses, + HarnessContractErrors, + HarnessContractResponses, + HarnessEvaluateErrors, + HarnessEvaluateResponses, + HarnessEvaluationsErrors, + HarnessEvaluationsResponses, + HarnessEvolutionReceiptErrors, + HarnessEvolutionReceiptResponses, + HarnessEvolutionRecordErrors, + HarnessEvolutionRecordResponses, + HarnessFailureInitializeErrors, + HarnessFailureInitializeResponses, + HarnessFailureObserveErrors, + HarnessFailureObserveResponses, + HarnessFailureSealErrors, + HarnessFailureSealResponses, + HarnessFailureSelectErrors, + HarnessFailureSelectResponses, + HarnessFailureStatusErrors, + HarnessFailureStatusResponses, + HarnessFormalReceiptErrors, + HarnessFormalReceiptResponses, + HarnessFormalRecordErrors, + HarnessFormalRecordResponses, + HarnessIntegrityReceiptErrors, + HarnessIntegrityReceiptResponses, + HarnessIntegrityRecordErrors, + HarnessIntegrityRecordResponses, + HarnessInterventionAssessErrors, + HarnessInterventionAssessResponses, + HarnessInterventionInitializeErrors, + HarnessInterventionInitializeResponses, + HarnessInterventionObserveErrors, + HarnessInterventionObserveResponses, + HarnessInterventionStatusErrors, + HarnessInterventionStatusResponses, + HarnessJudgeReceiptErrors, + HarnessJudgeReceiptResponses, + HarnessJudgeRecordErrors, + HarnessJudgeRecordResponses, + HarnessMetaReceiptErrors, + HarnessMetaReceiptResponses, + HarnessMetaRecordErrors, + HarnessMetaRecordResponses, + HarnessMetaSelectionErrors, + HarnessMetaSelectionResponses, + HarnessOrchestrationCheckpointErrors, + HarnessOrchestrationCheckpointResponses, + HarnessOrchestrationStartErrors, + HarnessOrchestrationStartResponses, + HarnessOrchestrationStatusErrors, + HarnessOrchestrationStatusResponses, + HarnessReplicationReceiptErrors, + HarnessReplicationReceiptResponses, + HarnessReplicationRecordErrors, + HarnessReplicationRecordResponses, + HarnessReportErrors, + HarnessReportResponses, + HarnessSemanticReceiptErrors, + HarnessSemanticReceiptResponses, + HarnessSemanticRecordErrors, + HarnessSemanticRecordResponses, + HarnessSimulationReceiptErrors, + HarnessSimulationReceiptResponses, + HarnessSimulationRecordErrors, + HarnessSimulationRecordResponses, + HarnessSkillAttestErrors, + HarnessSkillAttestResponses, + HarnessSkillPromoteErrors, + HarnessSkillPromoteResponses, + HarnessSkillProposeErrors, + HarnessSkillProposeResponses, + HarnessSkillsResponses, + HarnessSynthesisReceiptErrors, + HarnessSynthesisReceiptResponses, + HarnessSynthesisRecordErrors, + HarnessSynthesisRecordResponses, + HarnessWorldRefineErrors, + HarnessWorldRefineResponses, + HarnessWorldStatusErrors, + HarnessWorldStatusResponses, InstanceDisposeResponses, LspStatusResponses, McpAddErrors, @@ -3851,6 +3968,4197 @@ export class Permission extends HeyApiClient { } } +export class Audit extends HeyApiClient { + /** + * Initialize an evaluator-owned active audit + * + * Commits an opaque probe pool and binds uncertainty-aware selection to the evaluator capability and audited artifact. + */ + public initialize( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + probes?: Array< + | { + id: string + commitment: string + features: Array + stratum: string + weight?: number + priorLoss?: number + } + | { + id: string + commitment: string + sourceLosses: Array + stratum: string + weight?: number + } + > + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "probes" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessAuditInitializeResponses, + HarnessAuditInitializeErrors, + ThrowOnError + >({ + url: "/harness/audits", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected active audit + */ + public status( + parameters: { + auditID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "auditID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/audits/{auditID}/status", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Select the next opaque active-audit probe + * + * Combines weighted integral-variance reduction, failure UCB, failure-region diversity, and stratum coverage. + */ + public select( + parameters: { + auditID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "auditID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/audits/{auditID}/selection", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Record an evaluator-authenticated probe outcome + * + * Updates the GP posterior and stopping rule without promoting the audit estimate into benchmark evidence. + */ + public observe( + parameters: { + auditID: string + directory?: string + sessionID?: string + evaluatorToken?: string + probeID?: string + loss?: number + failure?: boolean + evidence?: Array + note?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "auditID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "probeID" }, + { in: "body", key: "loss" }, + { in: "body", key: "failure" }, + { in: "body", key: "evidence" }, + { in: "body", key: "note" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post( + { + url: "/harness/audits/{auditID}/observations", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }, + ) + } + + /** + * Seal a terminal active-audit receipt + * + * Content-addresses the completed audit, exact subject artifact, committed pool, derived estimate, transfer qualification, and terminal revision for optional promotion gating. + */ + public seal( + parameters: { + auditID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "auditID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/audits/{auditID}/receipt", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Failure extends HeyApiClient { + /** + * Initialize a topic-aware adversarial failure stream + * + * Binds deterministic UCB1 topic allocation and server-derived failure anchors to a terminal active-audit receipt without adding generated cases to the population estimate. + */ + public initialize( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + auditReceiptID?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "auditReceiptID" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessFailureInitializeResponses, + HarnessFailureInitializeErrors, + ThrowOnError + >({ + url: "/harness/failure-streams", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected failure discovery stream + */ + public status( + parameters: { + streamID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "streamID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessFailureStatusResponses, + HarnessFailureStatusErrors, + ThrowOnError + >({ + url: "/harness/failure-streams/{streamID}/status", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Select the next topic and authenticated failure anchors + * + * Forces every frozen topic once, then derives UCB1 from the immutable attempt journal with deterministic tie-breaking. + */ + public select( + parameters: { + streamID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "streamID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessFailureSelectResponses, + HarnessFailureSelectErrors, + ThrowOnError + >({ + url: "/harness/failure-streams/{streamID}/selection", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Record a validated adversarial generation attempt + * + * Consumes one attempt budget and derives admissibility and reward from the frozen correctness, topic, and novelty validators plus the target outcome. + */ + public observe( + parameters: { + streamID: string + directory?: string + sessionID?: string + evaluatorToken?: string + selectionID?: string + generation?: + | { + status: "failed" + mode: "generator_error" | "timeout" | "invalid_output" | "other" + outputSHA256?: string + evidence: Array + } + | { + status: "generated" + caseSHA256: string + outputSHA256: string + embedding: Array + evidence: Array + } + validations?: Array<{ + kind: "correctness" | "topic" | "novelty" + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + note?: string + }> + outcome?: { + loss: number + failure: boolean + outputSHA256: string + evidence: Array + } + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "streamID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "selectionID" }, + { in: "body", key: "generation" }, + { in: "body", key: "validations" }, + { in: "body", key: "outcome" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessFailureObserveResponses, + HarnessFailureObserveErrors, + ThrowOnError + >({ + url: "/harness/failure-streams/{streamID}/attempts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Seal a terminal failure discovery receipt + * + * Content-addresses the exact audit source, subject, topic contract, attempt journal, replayed UCB statistics, failure yield, and diversity evidence. + */ + public seal( + parameters: { + streamID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "streamID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/failure-streams/{streamID}/receipt", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Ablation extends HeyApiClient { + /** + * Freeze a matched scientific ablation plan + * + * Binds at least three evaluator-authenticated seed pairs before evaluation and permits exactly one declared contract factor to differ. + */ + public initialize( + parameters?: { + directory?: string + schemaVersion?: 1 + studyID?: string + factor?: { + kind: + | "profile" + | "orchestration" + | "search" + | "audit" + | "simulation" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" + | "fidelities" + | "skill" + | "tool" + name?: string + } + minEffect?: number + maxPairRegression?: number + pairs?: Array<{ + baseline: { + sessionID: string + evaluatorToken: string + } + arm: { + sessionID: string + evaluatorToken: string + } + }> + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "studyID" }, + { in: "body", key: "factor" }, + { in: "body", key: "minEffect" }, + { in: "body", key: "maxPairRegression" }, + { in: "body", key: "pairs" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessAblationInitializeResponses, + HarnessAblationInitializeErrors, + ThrowOnError + >({ + url: "/harness/ablations", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Assess a frozen matched ablation + * + * Authenticates every paired run, verifies immutable contracts and final evaluations, then derives paired effects and a 95% interval. + */ + public assess( + parameters: { + planID: string + directory?: string + runs?: Array<{ + sessionID: string + evaluatorToken: string + }> + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "planID" }, + { in: "query", key: "directory" }, + { in: "body", key: "runs" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessAblationAssessResponses, + HarnessAblationAssessErrors, + ThrowOnError + >({ + url: "/harness/ablations/{planID}/assessment", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Intervention extends HeyApiClient { + /** + * Freeze an evaluator-owned controlled replay study + * + * Binds a candidate and exact evolution receipt to predeclared replay, retuning, ablation, repair, or transfer pairs before the candidate's final evaluation. + */ + public initialize( + parameters?: { + directory?: string + schemaVersion?: 1 + runID?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evolutionReceiptID?: string + validator?: { + name: "design-replay-interventions" + version: 1 + scriptSHA256: string + } + pairs?: Array<{ + family: + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + index: number + control: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + arm: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + change: { + uri: string + sha256: string + } + }> + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "runID" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "evolutionReceiptID" }, + { in: "body", key: "validator" }, + { in: "body", key: "pairs" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessInterventionInitializeResponses, + HarnessInterventionInitializeErrors, + ThrowOnError + >({ + url: "/harness/interventions", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Record an evaluator-authenticated intervention outcome + * + * Binds one numeric outcome to an exact frozen pair target without adding it to candidate fitness or the benchmark evaluation journal. + */ + public observe( + parameters: { + candidateID: string + directory?: string + schemaVersion?: 1 + sessionID?: string + evaluatorToken?: string + pairID?: string + role?: "control" | "arm" + targetSHA256?: string + status?: "passed" | "failed" | "inconclusive" + score?: number + evidence?: Array + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "candidateID" }, + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "pairID" }, + { in: "body", key: "role" }, + { in: "body", key: "targetSHA256" }, + { in: "body", key: "status" }, + { in: "body", key: "score" }, + { in: "body", key: "evidence" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessInterventionObserveResponses, + HarnessInterventionObserveErrors, + ThrowOnError + >({ + url: "/harness/interventions/{candidateID}/observations", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Assess a complete controlled replay study + * + * Recomputes direction-aware paired effects, confidence intervals, stability, tuning gap, component dependence, and transfer robustness from every frozen outcome. + */ + public assess( + parameters: { + candidateID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "candidateID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessInterventionAssessResponses, + HarnessInterventionAssessErrors, + ThrowOnError + >({ + url: "/harness/interventions/{candidateID}/assessment", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected controlled replay study + */ + public status( + parameters: { + candidateID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "candidateID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessInterventionStatusResponses, + HarnessInterventionStatusErrors, + ThrowOnError + >({ + url: "/harness/interventions/{candidateID}/status", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Judge extends HeyApiClient { + /** + * Qualify a bound benchmark evaluator + * + * Uses an independent auditor capability and a committed hidden fault suite to recompute evaluator discrimination and calibration metrics. + */ + public record( + parameters?: { + directory?: string + sessionID?: string + auditorToken?: string + cases?: Array<{ + id: string + commitment: string + kind: "clean" | "fault" + fault?: + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + decision: "accept" | "reject" | "abstain" + failureProbability: number + evidence: Array + }> + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "auditorToken" }, + { in: "body", key: "cases" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/evaluators/qualifications", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected evaluator qualification + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + auditorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "auditorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post( + { + url: "/harness/evaluators/qualifications/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }, + ) + } +} + +export class Replication extends HeyApiClient { + /** + * Record an evaluator-authenticated replicated evaluation + * + * Requires the complete frozen stratum-by-cluster grid, then recomputes a robust estimate, uncertainty interval, and conservative promotion verdict. + */ + public record( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "run" | "candidate" + id: string + } + observations?: Array<{ + stratumID: string + clusterID: string + stratumSHA256: string + clusterSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + outputSHA256: string + environmentSHA256: string + evidence: Array + evaluatedAt: number + }> + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "observations" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessReplicationRecordResponses, + HarnessReplicationRecordErrors, + ThrowOnError + >({ + url: "/harness/replications/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected replicated evaluation receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessReplicationReceiptResponses, + HarnessReplicationReceiptErrors, + ThrowOnError + >({ + url: "/harness/replications/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Meta extends HeyApiClient { + /** + * Resolve the terminal meta-harness qualification subject + * + * Returns the backend-selected verified winner after search termination, isolated behind the independent meta-harness qualifier capability. + */ + public selection( + parameters?: { + directory?: string + sessionID?: string + metaToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "metaToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessMetaSelectionResponses, + HarnessMetaSelectionErrors, + ThrowOnError + >({ + url: "/harness/meta/selection", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Record a one-shot continual-harness qualification + * + * Freezes the complete refinement lineage, full trace archive, cross-model held-out matrix, activation/adherence diagnostics, and backend-derived promotion verdict. + */ + public record( + parameters?: { + directory?: string + schemaVersion?: 1 + sessionID?: string + metaToken?: string + selectionID?: string + candidateArtifactSHA256?: string + candidateManifestSHA256?: string + protectedManifestSHA256?: string + validatorSHA256?: string + archive?: { + uri: string + sha256: string + schemaSHA256: string + indexSHA256: string + contents: "full-source-scores-traces" + query: "filesystem" + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + entries: Array<{ + candidateID: string + artifactSHA256: string + sourceSHA256: string + state: "evaluated" | "unevaluated" + scoresSHA256?: string + resultSHA256?: string + evaluationSHA256?: string + trace?: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + }> + } + refinements?: Array<{ + revision: number + scope: "session" + parentSnapshotSHA256: string + snapshotSHA256: string + trigger: string + diagnosis: { + kind: "implementation" | "fundamental" | "inconclusive" + rationale: string + } + rootCause: string + expectedOutcome: string + changes: Array<{ + action: "create" | "update" | "delete" | "rollback" + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + path: string + beforeSHA256?: string + afterSHA256?: string + reason: string + }> + evidence: Array<{ + candidateID: string + traceSHA256: string + messageIndex: number + excerptSHA256: string + }> + predictions: Array<{ + modelID: string + taskID: string + expected: "fail_to_pass" | "remain_pass" + }> + }> + cells?: Array< + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "baseline" + } + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "candidate" + loaded: boolean + phases: Array<{ + followed: number + violatedCommission: number + violatedOmission: number + requiredUnobserved: number + notApplicable: number + insufficientEvidence: number + phase: "loaded" | "midpoint" | "pre_final" | "final_validation" + }> + } + > + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "metaToken" }, + { in: "body", key: "selectionID" }, + { in: "body", key: "candidateArtifactSHA256" }, + { in: "body", key: "candidateManifestSHA256" }, + { in: "body", key: "protectedManifestSHA256" }, + { in: "body", key: "validatorSHA256" }, + { in: "body", key: "archive" }, + { in: "body", key: "refinements" }, + { in: "body", key: "cells" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/meta/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected meta-harness receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + metaToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "metaToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/meta/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Confirmation extends HeyApiClient { + /** + * Resolve the sealed post-search confirmation subject + * + * Returns exactly one backend-selected verified winner only after adaptive search is terminal. The endpoint is isolated behind the claim evaluator capability. + */ + public selection( + parameters?: { + directory?: string + sessionID?: string + confirmationToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "confirmationToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessConfirmationSelectionResponses, + HarnessConfirmationSelectionErrors, + ThrowOnError + >({ + url: "/harness/confirmations/selection", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Record a one-shot sealed claim evaluation + * + * Freezes the claim result for the server-selected terminal winner without feeding the result into search, adaptive control, hindsight memory, or skill learning. + */ + public record( + parameters?: { + directory?: string + schemaVersion?: 1 + sessionID?: string + confirmationToken?: string + candidateSHA256?: string + manifestSHA256?: string + validatorSHA256?: string + environmentSHA256?: string + outcome?: "completed" | "failed" | "inconclusive" + score?: number + metrics?: { + [key: string]: number + } + checks?: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + score?: number + evidence?: Array + note?: string + }> + evidence?: Array + usage?: { + wallTimeMs?: number + costUSD?: number + } + outputSHA256?: string + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "confirmationToken" }, + { in: "body", key: "candidateSHA256" }, + { in: "body", key: "manifestSHA256" }, + { in: "body", key: "validatorSHA256" }, + { in: "body", key: "environmentSHA256" }, + { in: "body", key: "outcome" }, + { in: "body", key: "score" }, + { in: "body", key: "metrics" }, + { in: "body", key: "checks" }, + { in: "body", key: "evidence" }, + { in: "body", key: "usage" }, + { in: "body", key: "outputSHA256" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessConfirmationRecordResponses, + HarnessConfirmationRecordErrors, + ThrowOnError + >({ + url: "/harness/confirmations/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected sealed confirmation receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + confirmationToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "confirmationToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessConfirmationReceiptResponses, + HarnessConfirmationReceiptErrors, + ThrowOnError + >({ + url: "/harness/confirmations/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Semantic extends HeyApiClient { + /** + * Record an independent semantic audit + * + * Derives whether one bound result is meaningful, merely technically valid, ambiguous, or incorrect from independent evidence-backed reviews of frozen intent, shortcuts, and literature-relative novelty. + */ + public record( + parameters?: { + directory?: string + sessionID?: string + reviewerToken?: string + subject?: { + type: "run" | "candidate" + id: string + } + reviews?: Array<{ + actor: string + sessionID: string + correctness: "passed" | "failed" | "inconclusive" + alignment: "intended" | "reasonable_alternative" | "misinterpreted" | "ambiguous" + novelty: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + vacuous: boolean + confidence: number + criteria: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + shortcuts: Array<{ + id: string + observed: boolean + evidence: Array + }> + literatureRefs?: Array + evidence: Array + summary: string + reviewedAt: number + }> + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "reviewerToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "reviews" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessSemanticRecordResponses, + HarnessSemanticRecordErrors, + ThrowOnError + >({ + url: "/harness/semantics/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected semantic audit receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + reviewerToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "reviewerToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessSemanticReceiptResponses, + HarnessSemanticReceiptErrors, + ThrowOnError + >({ + url: "/harness/semantics/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Synthesis extends HeyApiClient { + /** + * Record an evaluator-authenticated clean-room synthesis + * + * Binds a complete retrieval trace and hidden atomic-fact manifest, rejects clean-room policy drift, and derives factual precision, recall, contradiction penalty, and F1 without trusting caller-authored metrics. + */ + public record( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "run" | "candidate" + id: string + } + conclusionSHA256?: string + evaluatorAuditReceiptID?: string + trace?: { + owner: "evaluator_runtime" + complete: true + schemaSHA256: string + filterPolicySHA256: string + events: Array<{ + sequence: number + tool: "google_search" | "paper_search" | "web_browse" + requestSHA256: string + responseSHA256: string + sourceSHA256: string + publishedAt?: string + matches: { + forbiddenDomain: boolean + referenceTitle: boolean + } + decision: "allowed" | "blocked" + evidence: Array + }> + } + decomposition?: { + status: "passed" | "failed" + outputSHA256?: string + evidence: Array + } + generatedFacts?: Array<{ + id: string + commitment: string + verdict: "supported" | "contradicted" | "unsupported" | "judge_error" + evidence: Array + }> + referenceFacts?: Array<{ + id: string + commitment: string + coverage: "covered" | "missed" | "judge_error" + evidence: Array + }> + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "conclusionSHA256" }, + { in: "body", key: "evaluatorAuditReceiptID" }, + { in: "body", key: "trace" }, + { in: "body", key: "decomposition" }, + { in: "body", key: "generatedFacts" }, + { in: "body", key: "referenceFacts" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessSynthesisRecordResponses, + HarnessSynthesisRecordErrors, + ThrowOnError + >({ + url: "/harness/syntheses/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected scientific synthesis receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessSynthesisReceiptResponses, + HarnessSynthesisReceiptErrors, + ThrowOnError + >({ + url: "/harness/syntheses/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Autonomy extends HeyApiClient { + /** + * Record an evaluator-authenticated human-AI autonomy trace + * + * Binds a complete interaction log to the exact run or candidate artifact and derives the Aletheia-inspired essentially-autonomous, collaborative, or primarily-human contribution level without trusting the caller's claim. + */ + public record( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "run" | "candidate" + id: string + } + artifactSHA256?: string + trace?: { + owner: "evaluator_runtime" + complete: true + recorderArtifactSHA256: string + schemaSHA256: string + classificationPolicySHA256: string + rawLogSHA256: string + startedAt: number + endedAt: number + events: Array<{ + sequence: number + at: number + actor: "benchmark" | "human" | "agent" + kind: + | "problem_statement" + | "clarification" + | "resource_provision" + | "strategy" + | "technical_correction" + | "artifact_edit" + | "candidate_selection" + | "evaluation_feedback" + | "exposition" + | "other" + contribution: "problem" | "auxiliary" | "essential" | "core" | "unclear" + contentSHA256: string + artifactBeforeSHA256?: string + artifactAfterSHA256?: string + evidence: Array + }> + } + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "artifactSHA256" }, + { in: "body", key: "trace" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessAutonomyRecordResponses, + HarnessAutonomyRecordErrors, + ThrowOnError + >({ + url: "/harness/autonomy/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected human-AI autonomy receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessAutonomyReceiptResponses, + HarnessAutonomyReceiptErrors, + ThrowOnError + >({ + url: "/harness/autonomy/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Blueprint extends HeyApiClient { + /** + * Initialize an evaluator-grounded formal proof blueprint + * + * Creates the content-addressed root of a bounded LEAP-inspired AND/OR proof graph without granting the graph final proof authority. + */ + public initialize( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessBlueprintInitializeResponses, + HarnessBlueprintInitializeErrors, + ThrowOnError + >({ + url: "/harness/proofs/blueprints", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read an evaluator-grounded formal proof blueprint + */ + public status( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessBlueprintStatusResponses, + HarnessBlueprintStatusErrors, + ThrowOnError + >({ + url: "/harness/proofs/blueprints/status", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Lease bounded ready goals from a formal proof blueprint + * + * Atomically expires stale work and leases distinct deepest-ready goals up to the frozen parallelism limit. + */ + public lease( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + count?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "count" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessBlueprintLeaseResponses, + HarnessBlueprintLeaseErrors, + ThrowOnError + >({ + url: "/harness/proofs/blueprints/leases", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Record an evaluator-authenticated proof or decomposition attempt + * + * Consumes one active goal lease, retains failed verifier or reviewer outcomes, and admits only exact compiler-checked sketches into the monotone acyclic graph. + */ + public record( + parameters?: { + directory?: string + body?: + | { + sessionID: string + evaluatorToken: string + kind: "direct" + leaseID: string + artifactSHA256: string + claim: "proof" | "refutation" | "failure" + verification: { + compilerArtifactSHA256: string + statementMatched: boolean + exitCode: number + warnings: number + transcriptSHA256: string + feedbackSHA256: string + startedAt: number + endedAt: number + } + } + | { + sessionID: string + evaluatorToken: string + kind: "decomposition" + leaseID: string + informalPlanSHA256: string + artifactSHA256: string + children: Array<{ + statementSHA256: string + declaration: string + module: string + }> + verification: { + compilerArtifactSHA256: string + statementMatched: boolean + exitCode: number + warnings: number + transcriptSHA256: string + feedbackSHA256: string + startedAt: number + endedAt: number + validatorArtifactSHA256: string + placeholderDeclarations: Array + validatorTranscriptSHA256: string + } + review: { + reviewerArtifactSHA256: string + promptSHA256: string + relevant: boolean + easier: boolean + plausible: boolean + transcriptSHA256: string + } + } + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { key: "body", map: "body" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessBlueprintRecordResponses, + HarnessBlueprintRecordErrors, + ThrowOnError + >({ + url: "/harness/proofs/blueprints/attempts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Formal extends HeyApiClient { + /** + * Record an evaluator-authenticated formal proof verification + * + * Binds a trusted Lean challenge, exact proof artifact, frozen environment, transitive axiom audit, and the contract's kernel, fresh-recheck, or external-crosscheck trust tier. + */ + public record( + parameters?: { + directory?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "run" | "candidate" + id: string + } + artifactSHA256?: string + relation?: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256?: string + statementSHA256?: string + declaration?: string + module?: string + environment?: { + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + } + manifest?: { + complete: boolean + files: Array<{ + path: string + role: + | "challenge" + | "statement" + | "proof" + | "lean_toolchain" + | "lake_manifest" + | "dependency_tree" + | "config" + | "support" + sha256: string + }> + } + verification?: { + startedAt: number + endedAt: number + build: { + verifierArtifactSHA256: string + exitCode: number + warnings: number + transcriptSHA256: string + } + source: { + verifierArtifactSHA256: string + complete: boolean + findings: Array<{ + construct: "sorry" | "admit" | "debug.skipKernelTC" | "native_decide" + path: string + line: number + }> + transcriptSHA256: string + } + axioms: { + verifierArtifactSHA256: string + complete: boolean + typesTraversed: boolean + observed: Array + transcriptSHA256: string + } + fresh?: { + verifierArtifactSHA256: string + fresh: boolean + exitCode: number + transcriptSHA256: string + } + external?: { + comparatorArtifactSHA256: string + sandboxImageSHA256: string + sandboxed: boolean + challengeMatched: boolean + proofTermSHA256: string + transcriptSHA256: string + checks: [ + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + ] + } + } + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "artifactSHA256" }, + { in: "body", key: "relation" }, + { in: "body", key: "challengeSHA256" }, + { in: "body", key: "statementSHA256" }, + { in: "body", key: "declaration" }, + { in: "body", key: "module" }, + { in: "body", key: "environment" }, + { in: "body", key: "manifest" }, + { in: "body", key: "verification" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post( + { + url: "/harness/proofs/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }, + ) + } + + /** + * Read a capability-protected formal proof receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessFormalReceiptResponses, + HarnessFormalReceiptErrors, + ThrowOnError + >({ + url: "/harness/proofs/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Integrity extends HeyApiClient { + /** + * Record evaluator-authenticated runtime integrity + * + * Derives trace-completeness, model-identity, contamination, external-model, benchmark-lookup, and hidden-canary gates against an immutable protocol. + */ + public record( + parameters?: { + directory?: string + schemaVersion?: 1 + runID?: string + sessionID?: string + evaluatorToken?: string + protocol?: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + subject?: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + trace?: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + events: number + dropped: number + startedAt: number + endedAt: number + } + model?: { + name: string + baseArtifactSHA256: string + configSHA256: string + outputArtifactSHA256: string + lineageVerified: boolean + } + audits?: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + ] + activity?: { + unapprovedExternalModelCalls: number + benchmarkLookupEvents: number + hiddenCanaryManifestSHA256: string + hiddenCanariesTested: number + hiddenCanaryViolations: number + } + validator?: { + name: "verify-benchmark-integrity" + version: 1 + scriptSHA256: string + } + evidence?: Array + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "runID" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "protocol" }, + { in: "body", key: "subject" }, + { in: "body", key: "trace" }, + { in: "body", key: "model" }, + { in: "body", key: "audits" }, + { in: "body", key: "activity" }, + { in: "body", key: "validator" }, + { in: "body", key: "evidence" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessIntegrityRecordResponses, + HarnessIntegrityRecordErrors, + ThrowOnError + >({ + url: "/harness/integrity/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected runtime integrity receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessIntegrityReceiptResponses, + HarnessIntegrityReceiptErrors, + ThrowOnError + >({ + url: "/harness/integrity/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Evolution extends HeyApiClient { + /** + * Record evaluator-authenticated evolutionary provenance + * + * Binds a candidate snapshot and every parent delta to immutable search lineage, then derives replay and ancestral line-reintroduction diagnostics without changing fitness. + */ + public record( + parameters?: { + directory?: string + schemaVersion?: 1 + runID?: string + sessionID?: string + evaluatorToken?: string + protocol?: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + subject?: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + snapshot?: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + files: Array<{ + path: string + sha256: string + bytes: number + lineHashes: Array + }> + } + parents?: Array<{ + id: string + artifact: { + uri: string + sha256: string + } + receiptID: string + snapshotSHA256: string + delta: { + uri: string + sha256: string + } + }> + validator?: { + name: "trace-evolutionary-candidate" + version: 1 + scriptSHA256: string + } + evidence?: Array + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "runID" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "protocol" }, + { in: "body", key: "subject" }, + { in: "body", key: "snapshot" }, + { in: "body", key: "parents" }, + { in: "body", key: "validator" }, + { in: "body", key: "evidence" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessEvolutionRecordResponses, + HarnessEvolutionRecordErrors, + ThrowOnError + >({ + url: "/harness/evolution/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected evolution trace receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessEvolutionReceiptResponses, + HarnessEvolutionReceiptErrors, + ThrowOnError + >({ + url: "/harness/evolution/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Simulation extends HeyApiClient { + /** + * Record an evaluator-authenticated simulator validation + * + * Recomputes convergence, residual, invariant, and stress-test gates against the immutable simulator protocol and exact subject artifact. + */ + public record( + parameters?: { + directory?: string + schemaVersion?: 1 + runID?: string + sessionID?: string + evaluatorToken?: string + subject?: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + engine?: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256?: string + reference?: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validationInputSHA256?: string + levels?: Array<{ + label: string + h: number + error: number + residual: number + invariants: { + [key: string]: number + } + }> + stressTests?: Array<{ + id: + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + evidence?: Array + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "runID" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "subject" }, + { in: "body", key: "engine" }, + { in: "body", key: "problemSHA256" }, + { in: "body", key: "reference" }, + { in: "body", key: "validationInputSHA256" }, + { in: "body", key: "levels" }, + { in: "body", key: "stressTests" }, + { in: "body", key: "evidence" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessSimulationRecordResponses, + HarnessSimulationRecordErrors, + ThrowOnError + >({ + url: "/harness/simulations/receipts", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read a capability-protected simulator validation receipt + */ + public receipt( + parameters: { + receiptID: string + directory?: string + sessionID?: string + evaluatorToken?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "receiptID" }, + { in: "query", key: "directory" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessSimulationReceiptResponses, + HarnessSimulationReceiptErrors, + ThrowOnError + >({ + url: "/harness/simulations/receipts/{receiptID}", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Orchestration extends HeyApiClient { + /** + * Read scientific orchestration state + */ + public status( + parameters: { + sessionID: string + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get< + HarnessOrchestrationStatusResponses, + HarnessOrchestrationStatusErrors, + ThrowOnError + >({ + url: "/harness/runs/{sessionID}/orchestration", + ...options, + ...params, + }) + } + + /** + * Initialize contract-bound scientific orchestration + * + * Selects a bounded topology from immutable contract traits and creates a restart-safe provisional work DAG. + */ + public start( + parameters: { + sessionID: string + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessOrchestrationStartResponses, + HarnessOrchestrationStartErrors, + ThrowOnError + >({ + url: "/harness/runs/{sessionID}/orchestration", + ...options, + ...params, + }) + } + + /** + * Record an evaluator-authenticated orchestration utility checkpoint + * + * Gates the next evolution round and stops low-utility search without allowing worker self-scores to control budget. + */ + public checkpoint( + parameters: { + sessionID: string + directory?: string + evaluatorToken?: string + round?: number + utility?: number + uncertainty?: number + evidenceRefs?: Array + evaluatedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "round" }, + { in: "body", key: "utility" }, + { in: "body", key: "uncertainty" }, + { in: "body", key: "evidenceRefs" }, + { in: "body", key: "evaluatedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + HarnessOrchestrationCheckpointResponses, + HarnessOrchestrationCheckpointErrors, + ThrowOnError + >({ + url: "/harness/runs/{sessionID}/orchestration/checkpoints", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class World extends HeyApiClient { + /** + * Read the session-local continual world model + * + * Returns confidence-graded mutable working state, event boundaries, context epoch, refinement trigger, and rollback revisions without exposing evaluator capabilities. + */ + public status( + parameters: { + sessionID: string + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get({ + url: "/harness/runs/{sessionID}/world", + ...options, + ...params, + }) + } + + /** + * Apply an evaluator-authenticated world-model refinement + * + * Applies a small revision-checked patch. Evaluator evidence may raise confidence beyond the agent's self-report ceiling while the immutable base prompt remains unchanged. + */ + public refine( + parameters: { + sessionID: string + directory?: string + evaluatorToken?: string + expectedRevision?: number + reason?: "manual" | "failure" | "stagnation" | "milestone" | "periodic" + patches?: Array< + | { + op: "upsert" + key: string + kind: "hypothesis" | "observation" | "strategy" | "memory" | "skill" | "subagent" + content: string + confidence: number + evidenceRefs?: Array + } + | { + op: "remove" + key: string + } + > + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "expectedRevision" }, + { in: "body", key: "reason" }, + { in: "body", key: "patches" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/runs/{sessionID}/world/refinements", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Skill extends HeyApiClient { + /** + * Create an inactive learned skill proposal + */ + public propose( + parameters?: { + directory?: string + name?: string + description?: string + content?: string + origin?: "conversation" | "rsi" + sessionID?: string + runID?: string + createdAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "name" }, + { in: "body", key: "description" }, + { in: "body", key: "content" }, + { in: "body", key: "origin" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "runID" }, + { in: "body", key: "createdAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post( + { + url: "/harness/skills", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }, + ) + } + + /** + * Attach paired held-out skill evidence + * + * Requires both evaluator capabilities and accepts only otherwise-identical candidate/control contracts. + */ + public attest( + parameters?: { + directory?: string + name?: string + candidate?: { + sessionID: string + evaluatorToken: string + } + control?: { + sessionID: string + evaluatorToken: string + } + trigger?: { + datasetSHA256: string + split: "held_out" + examples: number + truePositive: number + falsePositive: number + trueNegative: number + falseNegative: number + } + recordedAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "name" }, + { in: "body", key: "candidate" }, + { in: "body", key: "control" }, + { in: "body", key: "trigger" }, + { in: "body", key: "recordedAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/skills/evidence", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Promote a qualified learned skill + * + * Copies only an unchanged proposal that has met every held-out qualification criterion. + */ + public promote( + parameters: { + name: string + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "name" }, + { in: "query", key: "directory" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post( + { + url: "/harness/skills/{name}/promotion", + ...options, + ...params, + }, + ) + } +} + +export class Harness extends HeyApiClient { + /** + * Bind an immutable scientific evaluation run + * + * Called by a local or external evaluator before agent execution. The evaluator capability is hashed and never returned. + */ + public bind( + parameters?: { + directory?: string + schemaVersion?: 1 + runID?: string + sessionID?: string + benchmark?: string + title?: string + family?: "data" | "biology" | "physics" | "chemistry" | "ml" | "generalist" | "custom" + task?: string + version?: string + taskID?: string + split?: "development" | "validation" | "held_out" | "release" + evaluator?: { + name: string + version: string + source: "benchmark" | "gate" | "external" + token: string + } + objective?: string + profile?: "react" | "optimize" | "reproduce" | "theory" | "numerical" | "training" | "forecast" + search?: "adaptive" | "static" + orchestration?: { + topology: "auto" | "solo" | "centralized" | "fork_join" | "tournament" | "evolution" | "verifier_loop" + traits?: { + decomposability: number + sequentiality: number + toolIntensity: number + uncertainty: number + verificationRisk: number + novelty: number + crossDomain: number + } + maxWorkers: number + maxRounds: number + roles?: Array< + | "generation" + | "proximity" + | "reflection" + | "ranking" + | "evolution" + | "revision" + | "verification" + | "investigation" + | "simulation" + | "synthesis" + > + minIndependentVerifiers: number + adaptive?: { + protocolVersion: "marginal-utility-v1" + minRounds: number + patience: number + minUtilityGain: number + maxUncertainty: number + targetUtility?: number + } + repair?: { + protocolVersion: "verifier-routed-v1" + minConfidence: number + } + } + audit?: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + failureDiscovery?: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + integrity?: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + evolution?: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + metaHarness?: { + protocol: { + protocolVersion: "meta-harness-v1" + validatorSHA256: string + archiveSchemaSHA256: string + traceSchemaSHA256: string + baseline: { + artifactSHA256: string + manifestSHA256: string + } + mutable: Array<{ + root: string + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + }> + protected: { + manifestSHA256: string + roots: Array + } + archive: { + contents: "full-source-scores-traces" + query: "filesystem" + summariesOnly: false + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + updater: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judge: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + search: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + heldout: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + thresholds: { + minSearchGain: number + minHeldoutGain: number + maxModelRegression: number + minActivationRate: number + minRequiredAdherence: number + minFinalAdherence: number + maxPhaseDrift: number + minPredictionPrecision: number + maxRiskRegressions: number + maxContextTokens: number + maxMeanContextIncrease: number + } + promotionRequired: true + } + token: string + } + interventions?: { + protocolVersion: "intervention-study-v1" + validatorSHA256: string + requiredForPromotion: boolean + minPairs: number + maxPairs: number + maxTotalPairs: number + confidence: 0.95 + required: Array< + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + > + rules: Array< + | { + family: "replay" + mode: "max_absolute_effect" + threshold: number + } + | { + family: "retune" | "ablation" | "repair" + mode: "min_effect" + threshold: number + } + | { + family: "model_transfer" | "context_transfer" | "evaluator_transfer" | "split_transfer" + mode: "max_regression" + threshold: number + } + > + } + simulation?: { + kind: "ode" | "pde" | "cfd" | "materials" | "molecular" | "agentic" + engine: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256: string + reference: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validation: { + errorNorm: string + minLevels: number + maxLevels?: number + expectedOrder: number + orderTolerance: number + maxResidual: number + invariantTolerances: { + [key: string]: number + } + requiredStressTests: Array< + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + > + } + } + evaluatorAudit?: { + protocol: { + protocolVersion: "evaluator-audit-v1" + auditor: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + suite: { + name: string + version: string + commitmentSHA256: string + } + minCleanCases: number + minCasesPerFault: number + requiredFaults: Array< + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + > + minSensitivity: number + minSpecificity: number + minBalancedAccuracy: number + minFaultRecall: number + maxBrierScore: number + } + token: string + } + semanticAudit?: { + protocol: { + protocolVersion: "semantic-audit-v1" + reviewer: { + name: string + version: string + source: "gate" | "human" | "external" + } + scope: { + objectiveSHA256: string + criteria: Array<{ + id: string + requirement: string + }> + forbiddenShortcuts: Array<{ + id: string + description: string + }> + literature: { + cutoff: string + corpusSHA256: string + } + noveltyFloor: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + } + minReviewers: number + minConfidence: number + } + token: string + } + synthesis?: { + protocolVersion: "scientific-synthesis-v1" + querySHA256: string + referenceSHA256: string + referenceFactsSHA256: string + referenceFactCount: number + cutoff: string + tools: Array<"google_search" | "paper_search" | "web_browse"> + traceSchemaSHA256: string + filterPolicySHA256: string + maxToolEvents: number + decomposer: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judges: { + precision: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + recall: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + minGeneratedFacts: number + minPrecision: number + minRecall: number + minF1: number + cleanRoomRequired: true + judgeFailurePolicy: "inconclusive" + } + autonomy?: { + protocolVersion: "human-ai-autonomy-v1" + claimedLevel: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + recorder: { + name: string + version: string + artifactSHA256: string + source: "evaluator_runtime" + } + traceSchemaSHA256: string + classificationPolicySHA256: string + maxEvents: number + rawRetention: "required" + disclosure: "evaluator_retained" | "public_essential_after_release" + completeTraceRequired: true + uncertaintyPolicy: "inconclusive" + } + formalProof?: { + protocolVersion: "formal-proof-v1" + language: "lean4" + tier: "kernel" | "fresh_recheck" | "external_crosscheck" + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256: string + statementSHA256: string + declaration: string + module: string + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + verifiers: Array<{ + role: + | "lean_kernel" + | "source_auditor" + | "axiom_auditor" + | "fresh_rechecker" + | "sandbox_comparator" + | "external_checker" + name: string + version: string + artifactSHA256: string + }> + sandboxImageSHA256?: string + forbiddenConstructs: [ + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + ] + allowedAxioms: Array + maxFiles: number + completeManifestRequired: true + warningPolicy: "fail" + semanticPolicy: "formal_statement_only" + blueprint?: { + protocolVersion: "proof-blueprint-v1" + graphSchemaSHA256: string + compilerArtifactSHA256: string + sketchValidatorArtifactSHA256: string + reviewerArtifactSHA256: string + reviewerPromptSHA256: string + nodePolicy: "and-or-monotone-v1" + failurePolicy: "preserve-and-refine" + memoization: "goal-sha256" + finalAuthority: "formal-proof-v1" + directAttemptFirst: true + verifiedSketchRequired: true + completeFailureHistoryRequired: true + maxNodes: number + maxDepth: number + maxParallel: number + maxAttemptsPerGoal: number + maxRefinementsPerGoal: number + leaseDurationMs: number + } + } + replication?: { + protocolVersion: "replicated-evaluation-v1" + validatorSHA256: string + environmentSHA256: string + sampling: { + design: "crossed-stratified-cluster-v1" + stratumKind: string + clusterKind: string + strata: Array<{ + id: string + commitmentSHA256: string + }> + clusters: Array<{ + id: string + commitmentSHA256: string + }> + } + estimator: "mean" | "median" | "iqm" | "pass_rate" + interval: + | { + method: "stratified-bootstrap-percentile-v1" + confidence: 0.95 + resamples: number + seed: number + } + | { + method: "wilson-score-v1" + confidence: 0.95 + } + decision: { + rule: "conservative-bound-v1" + direction: "maximize" | "minimize" | "pass" + target: number + maxIntervalWidth?: number + } + failurePolicy: "fail-closed" + } + confirmation?: { + protocol: { + protocolVersion: "sealed-confirmation-v1" + optimization: { + split: "development" | "validation" + manifestSHA256: string + } + claim: { + taskID: string + split: "held_out" | "release" + manifestSHA256: string + validatorSHA256: string + environmentSHA256: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + source?: { + repository: string + revision: string + } + metric: string + direction: "maximize" | "minimize" + target: number + } + selection: { + rule: "terminal-verified-best-v1" + subjects: 1 + } + exposure: { + policy: "terminal-receipt-only" + searchFeedback: false + memoryCapture: false + } + failurePolicy: "fail-closed" + } + token: string + } + packs?: Array<"statistics" | "biology" | "physics" | "pde" | "chemistry" | "ml" | "forecast" | "formal"> + metric?: { + name?: string + direction: "maximize" | "minimize" | "pass" + target?: number + } + objectives?: Array<{ + metric: string + direction: "maximize" | "minimize" + }> + objectiveAudit?: { + schemaVersion: 1 + planSHA256: string + validatorSHA256: string + contractSHA256: string + guardIDs: Array + } + fidelities?: Array<{ + id: string + final: boolean + maxWallTimeMs?: number + maxCostUSD?: number + }> + model?: { + provider: string + name: string + effort?: string + } + tools?: Array + skills?: Array<{ + name: string + version?: string + sha256?: string + }> + budget?: { + wallTimeMs?: number + steps?: number + candidates?: number + tokens?: number + costUSD?: number + cpuHours?: number + gpuHours?: number + } + seed?: number + intervention?: "autonomous" | "human_reprompted" + contamination?: { + policy: string + hiddenTestsAccessible: false + publicDataCutoff?: string + } + createdAt?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "runID" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "benchmark" }, + { in: "body", key: "title" }, + { in: "body", key: "family" }, + { in: "body", key: "task" }, + { in: "body", key: "version" }, + { in: "body", key: "taskID" }, + { in: "body", key: "split" }, + { in: "body", key: "evaluator" }, + { in: "body", key: "objective" }, + { in: "body", key: "profile" }, + { in: "body", key: "search" }, + { in: "body", key: "orchestration" }, + { in: "body", key: "audit" }, + { in: "body", key: "failureDiscovery" }, + { in: "body", key: "integrity" }, + { in: "body", key: "evolution" }, + { in: "body", key: "metaHarness" }, + { in: "body", key: "interventions" }, + { in: "body", key: "simulation" }, + { in: "body", key: "evaluatorAudit" }, + { in: "body", key: "semanticAudit" }, + { in: "body", key: "synthesis" }, + { in: "body", key: "autonomy" }, + { in: "body", key: "formalProof" }, + { in: "body", key: "replication" }, + { in: "body", key: "confirmation" }, + { in: "body", key: "packs" }, + { in: "body", key: "metric" }, + { in: "body", key: "objectives" }, + { in: "body", key: "objectiveAudit" }, + { in: "body", key: "fidelities" }, + { in: "body", key: "model" }, + { in: "body", key: "tools" }, + { in: "body", key: "skills" }, + { in: "body", key: "budget" }, + { in: "body", key: "seed" }, + { in: "body", key: "intervention" }, + { in: "body", key: "contamination" }, + { in: "body", key: "createdAt" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/runs", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Ingest an evaluator-authenticated result + * + * Records an immutable subject result, promotes a verified search candidate, and captures task-scoped hindsight. + */ + public evaluate( + parameters?: { + directory?: string + schemaVersion?: 1 + runID?: string + sessionID?: string + evaluatorToken?: string + candidateID?: string + stage?: string + simulationReceiptID?: string + integrityReceiptID?: string + evolutionReceiptID?: string + interventionReceiptID?: string + evaluatorAuditReceiptID?: string + semanticReceiptID?: string + replicationReceiptID?: string + auditReceiptID?: string + failureDiscoveryReceiptID?: string + synthesisReceiptID?: string + autonomyReceiptID?: string + proofReceiptID?: string + status?: "passed" | "failed" | "inconclusive" + score?: number + metrics?: { + [key: string]: number + } + checks?: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + score?: number + evidence?: Array + note?: string + }> + evidence?: Array + usage?: { + wallTimeMs?: number + costUSD?: number + } + evaluatedAt?: number + notes?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "schemaVersion" }, + { in: "body", key: "runID" }, + { in: "body", key: "sessionID" }, + { in: "body", key: "evaluatorToken" }, + { in: "body", key: "candidateID" }, + { in: "body", key: "stage" }, + { in: "body", key: "simulationReceiptID" }, + { in: "body", key: "integrityReceiptID" }, + { in: "body", key: "evolutionReceiptID" }, + { in: "body", key: "interventionReceiptID" }, + { in: "body", key: "evaluatorAuditReceiptID" }, + { in: "body", key: "semanticReceiptID" }, + { in: "body", key: "replicationReceiptID" }, + { in: "body", key: "auditReceiptID" }, + { in: "body", key: "failureDiscoveryReceiptID" }, + { in: "body", key: "synthesisReceiptID" }, + { in: "body", key: "autonomyReceiptID" }, + { in: "body", key: "proofReceiptID" }, + { in: "body", key: "status" }, + { in: "body", key: "score" }, + { in: "body", key: "metrics" }, + { in: "body", key: "checks" }, + { in: "body", key: "evidence" }, + { in: "body", key: "usage" }, + { in: "body", key: "evaluatedAt" }, + { in: "body", key: "notes" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/evaluations", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Compare compatible scientific evaluation runs + * + * Reports direction-aware deltas and the quality-cost Pareto frontier. + */ + public compare( + parameters?: { + directory?: string + sessionIDs?: Array + baselineRunID?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "body", key: "sessionIDs" }, + { in: "body", key: "baselineRunID" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/harness/compare", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * List quarantined learned skill proposals + */ + public skills( + parameters?: { + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams([parameters], [{ args: [{ in: "query", key: "directory" }] }]) + return (options?.client ?? this.client).get({ + url: "/harness/skills", + ...options, + ...params, + }) + } + + /** + * Read a bound harness contract + */ + public contract( + parameters: { + sessionID: string + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get({ + url: "/harness/runs/{sessionID}/contract", + ...options, + ...params, + }) + } + + /** + * List immutable harness evaluations + */ + public evaluations( + parameters: { + sessionID: string + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get({ + url: "/harness/runs/{sessionID}/evaluations", + ...options, + ...params, + }) + } + + /** + * Build an evaluation quality-cost report + */ + public report( + parameters: { + sessionID: string + directory?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get({ + url: "/harness/runs/{sessionID}/report", + ...options, + ...params, + }) + } + + private _audit?: Audit + get audit(): Audit { + return (this._audit ??= new Audit({ client: this.client })) + } + + private _failure?: Failure + get failure(): Failure { + return (this._failure ??= new Failure({ client: this.client })) + } + + private _ablation?: Ablation + get ablation(): Ablation { + return (this._ablation ??= new Ablation({ client: this.client })) + } + + private _intervention?: Intervention + get intervention(): Intervention { + return (this._intervention ??= new Intervention({ client: this.client })) + } + + private _judge?: Judge + get judge(): Judge { + return (this._judge ??= new Judge({ client: this.client })) + } + + private _replication?: Replication + get replication(): Replication { + return (this._replication ??= new Replication({ client: this.client })) + } + + private _meta?: Meta + get meta(): Meta { + return (this._meta ??= new Meta({ client: this.client })) + } + + private _confirmation?: Confirmation + get confirmation(): Confirmation { + return (this._confirmation ??= new Confirmation({ client: this.client })) + } + + private _semantic?: Semantic + get semantic(): Semantic { + return (this._semantic ??= new Semantic({ client: this.client })) + } + + private _synthesis?: Synthesis + get synthesis(): Synthesis { + return (this._synthesis ??= new Synthesis({ client: this.client })) + } + + private _autonomy?: Autonomy + get autonomy(): Autonomy { + return (this._autonomy ??= new Autonomy({ client: this.client })) + } + + private _blueprint?: Blueprint + get blueprint(): Blueprint { + return (this._blueprint ??= new Blueprint({ client: this.client })) + } + + private _formal?: Formal + get formal(): Formal { + return (this._formal ??= new Formal({ client: this.client })) + } + + private _integrity?: Integrity + get integrity(): Integrity { + return (this._integrity ??= new Integrity({ client: this.client })) + } + + private _evolution?: Evolution + get evolution(): Evolution { + return (this._evolution ??= new Evolution({ client: this.client })) + } + + private _simulation?: Simulation + get simulation(): Simulation { + return (this._simulation ??= new Simulation({ client: this.client })) + } + + private _orchestration?: Orchestration + get orchestration(): Orchestration { + return (this._orchestration ??= new Orchestration({ client: this.client })) + } + + private _world?: World + get world(): World { + return (this._world ??= new World({ client: this.client })) + } + + private _skill?: Skill + get skill(): Skill { + return (this._skill ??= new Skill({ client: this.client })) + } +} + export class Search extends HeyApiClient { /** * Search sessions, messages, and artifacts @@ -6328,7 +10636,7 @@ export class Command extends HeyApiClient { } } -export class Skill extends HeyApiClient { +export class Skill2 extends HeyApiClient { /** * Delete user skill * @@ -6479,9 +10787,9 @@ export class App extends HeyApiClient { }) } - private _skill?: Skill - get skill(): Skill { - return (this._skill ??= new Skill({ client: this.client })) + private _skill?: Skill2 + get skill(): Skill2 { + return (this._skill ??= new Skill2({ client: this.client })) } } @@ -6742,6 +11050,11 @@ export class OpenScienceClient extends HeyApiClient { return (this._permission ??= new Permission({ client: this.client })) } + private _harness?: Harness + get harness(): Harness { + return (this._harness ??= new Harness({ client: this.client })) + } + private _search?: Search get search(): Search { return (this._search ??= new Search({ client: this.client })) diff --git a/tooling/sdk/js/src/v2/gen/types.gen.ts b/tooling/sdk/js/src/v2/gen/types.gen.ts index 7a7b6f2f..a6a03c3e 100644 --- a/tooling/sdk/js/src/v2/gen/types.gen.ts +++ b/tooling/sdk/js/src/v2/gen/types.gen.ts @@ -7820,6 +7820,14 @@ export type SessionTraceResponses = { delayMs: number createdAt: number }> + profiles: Array<{ + messageID: string + id: "react" | "optimize" | "reproduce" | "theory" | "numerical" | "training" | "forecast" + source: "contract" | "heuristic" | "control" + confidence: number + reasons: Array + selectedAt: number + }> privacy: { local: true atlasRequired: false @@ -8763,6 +8771,9848 @@ export type PermissionRespondResponses = { export type PermissionRespondResponse = PermissionRespondResponses[keyof PermissionRespondResponses] +export type HarnessAuditInitializeData = { + body?: { + sessionID: string + evaluatorToken: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + probes: Array< + | { + id: string + commitment: string + features: Array + stratum: string + weight?: number + priorLoss?: number + } + | { + id: string + commitment: string + sourceLosses: Array + stratum: string + weight?: number + } + > + } + path?: never + query?: { + directory?: string + } + url: "/harness/audits" +} + +export type HarnessAuditInitializeErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessAuditInitializeError = HarnessAuditInitializeErrors[keyof HarnessAuditInitializeErrors] + +export type HarnessAuditInitializeResponses = { + /** + * Active audit state + */ + 200: { + schemaVersion: 1 + protocolVersion: "active-audit-v1" | "proactive-audit-v2" + auditID: string + runID: string + sessionID: string + evaluator: string + contractFingerprint: string + poolFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + config: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + status: "active" | "completed" + stopReason?: "budget_exhausted" | "precision_reached" | "failure_target_reached" | "pool_exhausted" + pool: { + [key: string]: { + id: string + commitment: string + features: Array + stratum: string + weight?: number + priorLoss?: number + sourceLosses?: Array + selection?: { + round: number + selectedAt: number + phase?: "calibration" | "adaptive" | "fallback" + acquisition: { + posteriorLoss: number + posteriorStd: number + failureUCB: number + varianceReduction: number + diversity: number + coverage: number + score: number + } + } + observation?: { + loss: number + failure: boolean + evidence: Array + note?: string + evaluatedAt: number + } + } + } + order: Array + estimate: { + observed: number + failures: number + meanLoss: number + standardDeviation: number + lower95: number + upper95: number + abstain: boolean + effectivePoolSize: number + stratumCoverage: number + transfer?: { + status: "not_configured" | "calibrating" | "accepted" | "rejected" + observed: number + required: number + meanAbsoluteError?: number + threshold?: number + } + } + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessAuditInitializeResponse = HarnessAuditInitializeResponses[keyof HarnessAuditInitializeResponses] + +export type HarnessAuditStatusData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + auditID: string + } + query?: { + directory?: string + } + url: "/harness/audits/{auditID}/status" +} + +export type HarnessAuditStatusErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessAuditStatusError = HarnessAuditStatusErrors[keyof HarnessAuditStatusErrors] + +export type HarnessAuditStatusResponses = { + /** + * Active audit state + */ + 200: { + schemaVersion: 1 + protocolVersion: "active-audit-v1" | "proactive-audit-v2" + auditID: string + runID: string + sessionID: string + evaluator: string + contractFingerprint: string + poolFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + config: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + status: "active" | "completed" + stopReason?: "budget_exhausted" | "precision_reached" | "failure_target_reached" | "pool_exhausted" + pool: { + [key: string]: { + id: string + commitment: string + features: Array + stratum: string + weight?: number + priorLoss?: number + sourceLosses?: Array + selection?: { + round: number + selectedAt: number + phase?: "calibration" | "adaptive" | "fallback" + acquisition: { + posteriorLoss: number + posteriorStd: number + failureUCB: number + varianceReduction: number + diversity: number + coverage: number + score: number + } + } + observation?: { + loss: number + failure: boolean + evidence: Array + note?: string + evaluatedAt: number + } + } + } + order: Array + estimate: { + observed: number + failures: number + meanLoss: number + standardDeviation: number + lower95: number + upper95: number + abstain: boolean + effectivePoolSize: number + stratumCoverage: number + transfer?: { + status: "not_configured" | "calibrating" | "accepted" | "rejected" + observed: number + required: number + meanAbsoluteError?: number + threshold?: number + } + } + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessAuditStatusResponse = HarnessAuditStatusResponses[keyof HarnessAuditStatusResponses] + +export type HarnessAuditSelectData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + auditID: string + } + query?: { + directory?: string + } + url: "/harness/audits/{auditID}/selection" +} + +export type HarnessAuditSelectErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessAuditSelectError = HarnessAuditSelectErrors[keyof HarnessAuditSelectErrors] + +export type HarnessAuditSelectResponses = { + /** + * Selected opaque probe commitment + */ + 200: unknown +} + +export type HarnessAuditObserveData = { + body?: { + sessionID: string + evaluatorToken: string + probeID: string + loss: number + failure: boolean + evidence: Array + note?: string + } + path: { + auditID: string + } + query?: { + directory?: string + } + url: "/harness/audits/{auditID}/observations" +} + +export type HarnessAuditObserveErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessAuditObserveError = HarnessAuditObserveErrors[keyof HarnessAuditObserveErrors] + +export type HarnessAuditObserveResponses = { + /** + * Updated active audit state + */ + 200: { + schemaVersion: 1 + protocolVersion: "active-audit-v1" | "proactive-audit-v2" + auditID: string + runID: string + sessionID: string + evaluator: string + contractFingerprint: string + poolFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + config: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + status: "active" | "completed" + stopReason?: "budget_exhausted" | "precision_reached" | "failure_target_reached" | "pool_exhausted" + pool: { + [key: string]: { + id: string + commitment: string + features: Array + stratum: string + weight?: number + priorLoss?: number + sourceLosses?: Array + selection?: { + round: number + selectedAt: number + phase?: "calibration" | "adaptive" | "fallback" + acquisition: { + posteriorLoss: number + posteriorStd: number + failureUCB: number + varianceReduction: number + diversity: number + coverage: number + score: number + } + } + observation?: { + loss: number + failure: boolean + evidence: Array + note?: string + evaluatedAt: number + } + } + } + order: Array + estimate: { + observed: number + failures: number + meanLoss: number + standardDeviation: number + lower95: number + upper95: number + abstain: boolean + effectivePoolSize: number + stratumCoverage: number + transfer?: { + status: "not_configured" | "calibrating" | "accepted" | "rejected" + observed: number + required: number + meanAbsoluteError?: number + threshold?: number + } + } + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessAuditObserveResponse = HarnessAuditObserveResponses[keyof HarnessAuditObserveResponses] + +export type HarnessAuditSealData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + auditID: string + } + query?: { + directory?: string + } + url: "/harness/audits/{auditID}/receipt" +} + +export type HarnessAuditSealErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessAuditSealError = HarnessAuditSealErrors[keyof HarnessAuditSealErrors] + +export type HarnessAuditSealResponses = { + /** + * Immutable active-audit receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "proactive-audit-receipt-v1" + receiptID: string + auditID: string + runID: string + sessionID: string + contractFingerprint: string + poolFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + config: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + stopReason: "budget_exhausted" | "precision_reached" | "failure_target_reached" | "pool_exhausted" + estimate: { + observed: number + failures: number + meanLoss: number + standardDeviation: number + lower95: number + upper95: number + abstain: boolean + effectivePoolSize: number + stratumCoverage: number + transfer?: { + status: "not_configured" | "calibrating" | "accepted" | "rejected" + observed: number + required: number + meanAbsoluteError?: number + threshold?: number + } + } + revision: number + qualified: boolean + completedAt: number + sealedAt: number + } +} + +export type HarnessAuditSealResponse = HarnessAuditSealResponses[keyof HarnessAuditSealResponses] + +export type HarnessFailureInitializeData = { + body?: { + sessionID: string + evaluatorToken: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + auditReceiptID: string + } + path?: never + query?: { + directory?: string + } + url: "/harness/failure-streams" +} + +export type HarnessFailureInitializeErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessFailureInitializeError = HarnessFailureInitializeErrors[keyof HarnessFailureInitializeErrors] + +export type HarnessFailureInitializeResponses = { + /** + * Topic-aware failure discovery state + */ + 200: { + schemaVersion: 1 + protocolVersion: "topic-aware-failure-v1" + streamID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + auditReceiptID: string + sourcePoolSHA256: string + anchors: Array<{ + id: string + commitment: string + loss: number + }> + config: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + status: "active" | "completed" + stopReason?: "budget_exhausted" | "failure_target_reached" + pending?: { + selectionID: string + round: number + topic: { + id: string + commitment: string + } + anchors: Array<{ + id: string + commitment: string + loss: number + }> + allocation: { + phase: "initialization" | "ucb1" + pulls: number + rewards: number + score: number + } + selectedAt: number + } + attempts: Array<{ + attemptID: string + selection: { + selectionID: string + round: number + topic: { + id: string + commitment: string + } + anchors: Array<{ + id: string + commitment: string + loss: number + }> + allocation: { + phase: "initialization" | "ucb1" + pulls: number + rewards: number + score: number + } + selectedAt: number + } + generation: + | { + status: "failed" + mode: "generator_error" | "timeout" | "invalid_output" | "other" + outputSHA256?: string + evidence: Array + } + | { + status: "generated" + caseSHA256: string + outputSHA256: string + embedding: Array + evidence: Array + } + validations: Array<{ + kind: "correctness" | "topic" | "novelty" + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + note?: string + }> + outcome?: { + loss: number + failure: boolean + outputSHA256: string + evidence: Array + } + admissible: boolean + reward: 0 | 1 + evaluatedAt: number + recordedAt: number + }> + statistics: { + attempts: number + generated: number + admissible: number + failures: number + invalid: number + samplesToFirstFailure?: number + failureRate: number + topicEntropy: number + embeddingLogDet: number + topics: { + [key: string]: { + pulls: number + rewards: number + rate: number + } + } + } + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessFailureInitializeResponse = + HarnessFailureInitializeResponses[keyof HarnessFailureInitializeResponses] + +export type HarnessFailureStatusData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + streamID: string + } + query?: { + directory?: string + } + url: "/harness/failure-streams/{streamID}/status" +} + +export type HarnessFailureStatusErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessFailureStatusError = HarnessFailureStatusErrors[keyof HarnessFailureStatusErrors] + +export type HarnessFailureStatusResponses = { + /** + * Topic-aware failure discovery state + */ + 200: { + schemaVersion: 1 + protocolVersion: "topic-aware-failure-v1" + streamID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + auditReceiptID: string + sourcePoolSHA256: string + anchors: Array<{ + id: string + commitment: string + loss: number + }> + config: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + status: "active" | "completed" + stopReason?: "budget_exhausted" | "failure_target_reached" + pending?: { + selectionID: string + round: number + topic: { + id: string + commitment: string + } + anchors: Array<{ + id: string + commitment: string + loss: number + }> + allocation: { + phase: "initialization" | "ucb1" + pulls: number + rewards: number + score: number + } + selectedAt: number + } + attempts: Array<{ + attemptID: string + selection: { + selectionID: string + round: number + topic: { + id: string + commitment: string + } + anchors: Array<{ + id: string + commitment: string + loss: number + }> + allocation: { + phase: "initialization" | "ucb1" + pulls: number + rewards: number + score: number + } + selectedAt: number + } + generation: + | { + status: "failed" + mode: "generator_error" | "timeout" | "invalid_output" | "other" + outputSHA256?: string + evidence: Array + } + | { + status: "generated" + caseSHA256: string + outputSHA256: string + embedding: Array + evidence: Array + } + validations: Array<{ + kind: "correctness" | "topic" | "novelty" + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + note?: string + }> + outcome?: { + loss: number + failure: boolean + outputSHA256: string + evidence: Array + } + admissible: boolean + reward: 0 | 1 + evaluatedAt: number + recordedAt: number + }> + statistics: { + attempts: number + generated: number + admissible: number + failures: number + invalid: number + samplesToFirstFailure?: number + failureRate: number + topicEntropy: number + embeddingLogDet: number + topics: { + [key: string]: { + pulls: number + rewards: number + rate: number + } + } + } + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessFailureStatusResponse = HarnessFailureStatusResponses[keyof HarnessFailureStatusResponses] + +export type HarnessFailureSelectData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + streamID: string + } + query?: { + directory?: string + } + url: "/harness/failure-streams/{streamID}/selection" +} + +export type HarnessFailureSelectErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessFailureSelectError = HarnessFailureSelectErrors[keyof HarnessFailureSelectErrors] + +export type HarnessFailureSelectResponses = { + /** + * Server-selected topic, anchors, and allocation evidence + */ + 200: { + selectionID: string + round: number + topic: { + id: string + commitment: string + } + anchors: Array<{ + id: string + commitment: string + loss: number + }> + allocation: { + phase: "initialization" | "ucb1" + pulls: number + rewards: number + score: number + } + selectedAt: number + } +} + +export type HarnessFailureSelectResponse = HarnessFailureSelectResponses[keyof HarnessFailureSelectResponses] + +export type HarnessFailureObserveData = { + body?: { + sessionID: string + evaluatorToken: string + selectionID: string + generation: + | { + status: "failed" + mode: "generator_error" | "timeout" | "invalid_output" | "other" + outputSHA256?: string + evidence: Array + } + | { + status: "generated" + caseSHA256: string + outputSHA256: string + embedding: Array + evidence: Array + } + validations: Array<{ + kind: "correctness" | "topic" | "novelty" + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + note?: string + }> + outcome?: { + loss: number + failure: boolean + outputSHA256: string + evidence: Array + } + evaluatedAt: number + } + path: { + streamID: string + } + query?: { + directory?: string + } + url: "/harness/failure-streams/{streamID}/attempts" +} + +export type HarnessFailureObserveErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessFailureObserveError = HarnessFailureObserveErrors[keyof HarnessFailureObserveErrors] + +export type HarnessFailureObserveResponses = { + /** + * Updated topic-aware failure discovery state + */ + 200: { + schemaVersion: 1 + protocolVersion: "topic-aware-failure-v1" + streamID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + auditReceiptID: string + sourcePoolSHA256: string + anchors: Array<{ + id: string + commitment: string + loss: number + }> + config: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + status: "active" | "completed" + stopReason?: "budget_exhausted" | "failure_target_reached" + pending?: { + selectionID: string + round: number + topic: { + id: string + commitment: string + } + anchors: Array<{ + id: string + commitment: string + loss: number + }> + allocation: { + phase: "initialization" | "ucb1" + pulls: number + rewards: number + score: number + } + selectedAt: number + } + attempts: Array<{ + attemptID: string + selection: { + selectionID: string + round: number + topic: { + id: string + commitment: string + } + anchors: Array<{ + id: string + commitment: string + loss: number + }> + allocation: { + phase: "initialization" | "ucb1" + pulls: number + rewards: number + score: number + } + selectedAt: number + } + generation: + | { + status: "failed" + mode: "generator_error" | "timeout" | "invalid_output" | "other" + outputSHA256?: string + evidence: Array + } + | { + status: "generated" + caseSHA256: string + outputSHA256: string + embedding: Array + evidence: Array + } + validations: Array<{ + kind: "correctness" | "topic" | "novelty" + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + note?: string + }> + outcome?: { + loss: number + failure: boolean + outputSHA256: string + evidence: Array + } + admissible: boolean + reward: 0 | 1 + evaluatedAt: number + recordedAt: number + }> + statistics: { + attempts: number + generated: number + admissible: number + failures: number + invalid: number + samplesToFirstFailure?: number + failureRate: number + topicEntropy: number + embeddingLogDet: number + topics: { + [key: string]: { + pulls: number + rewards: number + rate: number + } + } + } + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessFailureObserveResponse = HarnessFailureObserveResponses[keyof HarnessFailureObserveResponses] + +export type HarnessFailureSealData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + streamID: string + } + query?: { + directory?: string + } + url: "/harness/failure-streams/{streamID}/receipt" +} + +export type HarnessFailureSealErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessFailureSealError = HarnessFailureSealErrors[keyof HarnessFailureSealErrors] + +export type HarnessFailureSealResponses = { + /** + * Immutable topic-aware failure discovery receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "topic-aware-failure-receipt-v1" + receiptID: string + streamID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifactSHA256: string + } + auditReceiptID: string + sourcePoolSHA256: string + config: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + attemptIDs: Array + statistics: { + attempts: number + generated: number + admissible: number + failures: number + invalid: number + samplesToFirstFailure?: number + failureRate: number + topicEntropy: number + embeddingLogDet: number + topics: { + [key: string]: { + pulls: number + rewards: number + rate: number + } + } + } + stopReason: "budget_exhausted" | "failure_target_reached" + revision: number + completedAt: number + sealedAt: number + } +} + +export type HarnessFailureSealResponse = HarnessFailureSealResponses[keyof HarnessFailureSealResponses] + +export type HarnessAblationInitializeData = { + body?: { + schemaVersion: 1 + studyID: string + factor: { + kind: + | "profile" + | "orchestration" + | "search" + | "audit" + | "simulation" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" + | "fidelities" + | "skill" + | "tool" + name?: string + } + minEffect: number + maxPairRegression?: number + pairs: Array<{ + baseline: { + sessionID: string + evaluatorToken: string + } + arm: { + sessionID: string + evaluatorToken: string + } + }> + } + path?: never + query?: { + directory?: string + } + url: "/harness/ablations" +} + +export type HarnessAblationInitializeErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessAblationInitializeError = HarnessAblationInitializeErrors[keyof HarnessAblationInitializeErrors] + +export type HarnessAblationInitializeResponses = { + /** + * Immutable matched ablation plan + */ + 200: { + schemaVersion: 1 + plan: { + schemaVersion: 1 + planID: string + studyID: string + factor: { + kind: + | "profile" + | "orchestration" + | "search" + | "audit" + | "simulation" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" + | "fidelities" + | "skill" + | "tool" + name?: string + } + baselineValueSHA256: string + armValueSHA256: string + contextSHA256: string + benchmark: { + name: string + version: string + taskID: string + split: "held_out" | "release" + evaluator: string + evaluatorVersion: string + metric: string + direction: "maximize" | "minimize" + } + minEffect: number + maxPairRegression: number + pairs: Array<{ + seed: number + baseline: { + sessionID: string + runID: string + contractFingerprint: string + } + arm: { + sessionID: string + runID: string + contractFingerprint: string + } + }> + createdAt: number + } + receipt?: { + schemaVersion: 1 + receiptID: string + planID: string + studyID: string + factor: { + kind: + | "profile" + | "orchestration" + | "search" + | "audit" + | "simulation" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" + | "fidelities" + | "skill" + | "tool" + name?: string + } + pairs: Array<{ + seed: number + baseline: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + evaluatedAt: number + recordedAt: number + } + arm: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + evaluatedAt: number + recordedAt: number + } + effect?: number + }> + statistics: { + pairs: number + validPairs: number + meanEffect?: number + standardDeviation?: number + standardError?: number + confidence95?: [number, number] + regressions: number + minEffect: number + maxPairRegression: number + } + verdict: "supported" | "rejected" | "inconclusive" + assessedAt: number + } + } +} + +export type HarnessAblationInitializeResponse = + HarnessAblationInitializeResponses[keyof HarnessAblationInitializeResponses] + +export type HarnessAblationAssessData = { + body?: { + runs: Array<{ + sessionID: string + evaluatorToken: string + }> + } + path: { + planID: string + } + query?: { + directory?: string + } + url: "/harness/ablations/{planID}/assessment" +} + +export type HarnessAblationAssessErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessAblationAssessError = HarnessAblationAssessErrors[keyof HarnessAblationAssessErrors] + +export type HarnessAblationAssessResponses = { + /** + * Immutable matched ablation assessment + */ + 200: { + schemaVersion: 1 + plan: { + schemaVersion: 1 + planID: string + studyID: string + factor: { + kind: + | "profile" + | "orchestration" + | "search" + | "audit" + | "simulation" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" + | "fidelities" + | "skill" + | "tool" + name?: string + } + baselineValueSHA256: string + armValueSHA256: string + contextSHA256: string + benchmark: { + name: string + version: string + taskID: string + split: "held_out" | "release" + evaluator: string + evaluatorVersion: string + metric: string + direction: "maximize" | "minimize" + } + minEffect: number + maxPairRegression: number + pairs: Array<{ + seed: number + baseline: { + sessionID: string + runID: string + contractFingerprint: string + } + arm: { + sessionID: string + runID: string + contractFingerprint: string + } + }> + createdAt: number + } + receipt?: { + schemaVersion: 1 + receiptID: string + planID: string + studyID: string + factor: { + kind: + | "profile" + | "orchestration" + | "search" + | "audit" + | "simulation" + | "evaluator_audit" + | "semantic_audit" + | "synthesis" + | "autonomy" + | "formal_proof" + | "replication" + | "fidelities" + | "skill" + | "tool" + name?: string + } + pairs: Array<{ + seed: number + baseline: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + evaluatedAt: number + recordedAt: number + } + arm: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + evaluatedAt: number + recordedAt: number + } + effect?: number + }> + statistics: { + pairs: number + validPairs: number + meanEffect?: number + standardDeviation?: number + standardError?: number + confidence95?: [number, number] + regressions: number + minEffect: number + maxPairRegression: number + } + verdict: "supported" | "rejected" | "inconclusive" + assessedAt: number + } + } +} + +export type HarnessAblationAssessResponse = HarnessAblationAssessResponses[keyof HarnessAblationAssessResponses] + +export type HarnessInterventionInitializeData = { + body?: { + schemaVersion: 1 + runID: string + sessionID: string + evaluatorToken: string + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evolutionReceiptID: string + validator: { + name: "design-replay-interventions" + version: 1 + scriptSHA256: string + } + pairs: Array<{ + family: + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + index: number + control: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + arm: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + change: { + uri: string + sha256: string + } + }> + } + path?: never + query?: { + directory?: string + } + url: "/harness/interventions" +} + +export type HarnessInterventionInitializeErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessInterventionInitializeError = + HarnessInterventionInitializeErrors[keyof HarnessInterventionInitializeErrors] + +export type HarnessInterventionInitializeResponses = { + /** + * Immutable controlled intervention plan + */ + 200: { + schemaVersion: 1 + plan: { + schemaVersion: 1 + planID: string + runID: string + sessionID: string + contractFingerprint: string + protocol: { + protocolVersion: "intervention-study-v1" + validatorSHA256: string + requiredForPromotion: boolean + minPairs: number + maxPairs: number + maxTotalPairs: number + confidence: 0.95 + required: Array< + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + > + rules: Array< + | { + family: "replay" + mode: "max_absolute_effect" + threshold: number + } + | { + family: "retune" | "ablation" | "repair" + mode: "min_effect" + threshold: number + } + | { + family: "model_transfer" | "context_transfer" | "evaluator_transfer" | "split_transfer" + mode: "max_regression" + threshold: number + } + > + } + benchmark: { + name: string + version: string + taskID: string + metric: string + direction: "maximize" | "minimize" + } + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evolutionReceiptID: string + validator: { + name: "design-replay-interventions" + version: 1 + scriptSHA256: string + } + pairs: Array<{ + family: + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + index: number + control: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + arm: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + change: { + uri: string + sha256: string + } + pairID: string + }> + createdAt: number + } + outcomes: { + [key: string]: { + schemaVersion: 1 + outcomeID: string + submissionID: string + pairID: string + role: "control" | "arm" + targetSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + evaluatedAt: number + recordedAt: number + } + } + order: Array + receipt?: { + schemaVersion: 1 + receiptID: string + planID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evolutionReceiptID: string + families: Array<{ + family: + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + mode: "max_absolute_effect" | "min_effect" | "max_regression" + threshold: number + pairs: number + validPairs: number + meanEffect?: number + standardDeviation?: number + standardError?: number + confidence95?: [number, number] + maxAbsoluteEffect?: number + regressions: number + verdict: "passed" | "failed" | "inconclusive" + }> + status: "passed" | "failed" | "inconclusive" + observedAt: number + assessedAt: number + } + } +} + +export type HarnessInterventionInitializeResponse = + HarnessInterventionInitializeResponses[keyof HarnessInterventionInitializeResponses] + +export type HarnessInterventionObserveData = { + body?: { + schemaVersion: 1 + sessionID: string + evaluatorToken: string + pairID: string + role: "control" | "arm" + targetSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + evaluatedAt: number + } + path: { + candidateID: string + } + query?: { + directory?: string + } + url: "/harness/interventions/{candidateID}/observations" +} + +export type HarnessInterventionObserveErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessInterventionObserveError = HarnessInterventionObserveErrors[keyof HarnessInterventionObserveErrors] + +export type HarnessInterventionObserveResponses = { + /** + * Immutable intervention outcome + */ + 200: { + schemaVersion: 1 + outcomeID: string + submissionID: string + pairID: string + role: "control" | "arm" + targetSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessInterventionObserveResponse = + HarnessInterventionObserveResponses[keyof HarnessInterventionObserveResponses] + +export type HarnessInterventionAssessData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + candidateID: string + } + query?: { + directory?: string + } + url: "/harness/interventions/{candidateID}/assessment" +} + +export type HarnessInterventionAssessErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessInterventionAssessError = HarnessInterventionAssessErrors[keyof HarnessInterventionAssessErrors] + +export type HarnessInterventionAssessResponses = { + /** + * Immutable controlled intervention receipt + */ + 200: { + schemaVersion: 1 + receiptID: string + planID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evolutionReceiptID: string + families: Array<{ + family: + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + mode: "max_absolute_effect" | "min_effect" | "max_regression" + threshold: number + pairs: number + validPairs: number + meanEffect?: number + standardDeviation?: number + standardError?: number + confidence95?: [number, number] + maxAbsoluteEffect?: number + regressions: number + verdict: "passed" | "failed" | "inconclusive" + }> + status: "passed" | "failed" | "inconclusive" + observedAt: number + assessedAt: number + } +} + +export type HarnessInterventionAssessResponse = + HarnessInterventionAssessResponses[keyof HarnessInterventionAssessResponses] + +export type HarnessInterventionStatusData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + candidateID: string + } + query?: { + directory?: string + } + url: "/harness/interventions/{candidateID}/status" +} + +export type HarnessInterventionStatusErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessInterventionStatusError = HarnessInterventionStatusErrors[keyof HarnessInterventionStatusErrors] + +export type HarnessInterventionStatusResponses = { + /** + * Controlled intervention state + */ + 200: { + schemaVersion: 1 + plan: { + schemaVersion: 1 + planID: string + runID: string + sessionID: string + contractFingerprint: string + protocol: { + protocolVersion: "intervention-study-v1" + validatorSHA256: string + requiredForPromotion: boolean + minPairs: number + maxPairs: number + maxTotalPairs: number + confidence: 0.95 + required: Array< + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + > + rules: Array< + | { + family: "replay" + mode: "max_absolute_effect" + threshold: number + } + | { + family: "retune" | "ablation" | "repair" + mode: "min_effect" + threshold: number + } + | { + family: "model_transfer" | "context_transfer" | "evaluator_transfer" | "split_transfer" + mode: "max_regression" + threshold: number + } + > + } + benchmark: { + name: string + version: string + taskID: string + metric: string + direction: "maximize" | "minimize" + } + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evolutionReceiptID: string + validator: { + name: "design-replay-interventions" + version: 1 + scriptSHA256: string + } + pairs: Array<{ + family: + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + index: number + control: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + arm: { + artifact: { + uri: string + sha256: string + } + condition: { + seed: number + model: { + provider: string + name: string + version: string + } + context: { + uri: string + sha256: string + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + split: { + name: string + manifest: { + uri: string + sha256: string + } + } + environment: { + uri: string + sha256: string + } + budget: { + uri: string + sha256: string + } + } + } + change: { + uri: string + sha256: string + } + pairID: string + }> + createdAt: number + } + outcomes: { + [key: string]: { + schemaVersion: 1 + outcomeID: string + submissionID: string + pairID: string + role: "control" | "arm" + targetSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + evidence: Array + evaluatedAt: number + recordedAt: number + } + } + order: Array + receipt?: { + schemaVersion: 1 + receiptID: string + planID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evolutionReceiptID: string + families: Array<{ + family: + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + mode: "max_absolute_effect" | "min_effect" | "max_regression" + threshold: number + pairs: number + validPairs: number + meanEffect?: number + standardDeviation?: number + standardError?: number + confidence95?: [number, number] + maxAbsoluteEffect?: number + regressions: number + verdict: "passed" | "failed" | "inconclusive" + }> + status: "passed" | "failed" | "inconclusive" + observedAt: number + assessedAt: number + } + } | null +} + +export type HarnessInterventionStatusResponse = + HarnessInterventionStatusResponses[keyof HarnessInterventionStatusResponses] + +export type HarnessJudgeRecordData = { + body?: { + sessionID: string + auditorToken: string + cases: Array<{ + id: string + commitment: string + kind: "clean" | "fault" + fault?: + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + decision: "accept" | "reject" | "abstain" + failureProbability: number + evidence: Array + }> + } + path?: never + query?: { + directory?: string + } + url: "/harness/evaluators/qualifications" +} + +export type HarnessJudgeRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessJudgeRecordError = HarnessJudgeRecordErrors[keyof HarnessJudgeRecordErrors] + +export type HarnessJudgeRecordResponses = { + /** + * Immutable evaluator audit receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "evaluator-audit-receipt-v1" + receiptID: string + protocolSHA256: string + sourceSessionID: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + auditor: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + suite: { + name: string + version: string + commitmentSHA256: string + } + cases: Array<{ + id: string + commitment: string + kind: "clean" | "fault" + fault?: + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + decision: "accept" | "reject" | "abstain" + failureProbability: number + evidence: Array + }> + metrics: { + cases: number + cleanCases: number + faultCases: number + truePositive: number + falseNegative: number + trueNegative: number + falsePositive: number + sensitivity: number + specificity: number + balancedAccuracy: number + brierScore: number + perFault: { + [key: string]: { + cases: number + detected: number + recall: number + } + } + } + status: "passed" | "failed" + failures: Array + recordedAt: number + } +} + +export type HarnessJudgeRecordResponse = HarnessJudgeRecordResponses[keyof HarnessJudgeRecordResponses] + +export type HarnessJudgeReceiptData = { + body?: { + sessionID: string + auditorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/evaluators/qualifications/{receiptID}" +} + +export type HarnessJudgeReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessJudgeReceiptError = HarnessJudgeReceiptErrors[keyof HarnessJudgeReceiptErrors] + +export type HarnessJudgeReceiptResponses = { + /** + * Evaluator audit receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "evaluator-audit-receipt-v1" + receiptID: string + protocolSHA256: string + sourceSessionID: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + auditor: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + suite: { + name: string + version: string + commitmentSHA256: string + } + cases: Array<{ + id: string + commitment: string + kind: "clean" | "fault" + fault?: + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + decision: "accept" | "reject" | "abstain" + failureProbability: number + evidence: Array + }> + metrics: { + cases: number + cleanCases: number + faultCases: number + truePositive: number + falseNegative: number + trueNegative: number + falsePositive: number + sensitivity: number + specificity: number + balancedAccuracy: number + brierScore: number + perFault: { + [key: string]: { + cases: number + detected: number + recall: number + } + } + } + status: "passed" | "failed" + failures: Array + recordedAt: number + } +} + +export type HarnessJudgeReceiptResponse = HarnessJudgeReceiptResponses[keyof HarnessJudgeReceiptResponses] + +export type HarnessReplicationRecordData = { + body?: { + sessionID: string + evaluatorToken: string + subject: { + type: "run" | "candidate" + id: string + } + observations: Array<{ + stratumID: string + clusterID: string + stratumSHA256: string + clusterSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + outputSHA256: string + environmentSHA256: string + evidence: Array + evaluatedAt: number + }> + } + path?: never + query?: { + directory?: string + } + url: "/harness/replications/receipts" +} + +export type HarnessReplicationRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessReplicationRecordError = HarnessReplicationRecordErrors[keyof HarnessReplicationRecordErrors] + +export type HarnessReplicationRecordResponses = { + /** + * Immutable replicated evaluation receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "replicated-evaluation-receipt-v1" + receiptID: string + protocolSHA256: string + contractSHA256: string + sourceSessionID: string + subject: { + type: "run" | "candidate" + id: string + } + metric: string + protocol: { + protocolVersion: "replicated-evaluation-v1" + validatorSHA256: string + environmentSHA256: string + sampling: { + design: "crossed-stratified-cluster-v1" + stratumKind: string + clusterKind: string + strata: Array<{ + id: string + commitmentSHA256: string + }> + clusters: Array<{ + id: string + commitmentSHA256: string + }> + } + estimator: "mean" | "median" | "iqm" | "pass_rate" + interval: + | { + method: "stratified-bootstrap-percentile-v1" + confidence: 0.95 + resamples: number + seed: number + } + | { + method: "wilson-score-v1" + confidence: 0.95 + } + decision: { + rule: "conservative-bound-v1" + direction: "maximize" | "minimize" | "pass" + target: number + maxIntervalWidth?: number + } + failurePolicy: "fail-closed" + } + observations: Array<{ + stratumID: string + clusterID: string + stratumSHA256: string + clusterSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + outputSHA256: string + environmentSHA256: string + evidence: Array + evaluatedAt: number + }> + statistics: { + units: number + passed: number + failed: number + inconclusive: number + estimator: "mean" | "median" | "iqm" | "pass_rate" + estimate?: number + confidence: 0.95 + interval?: [number, number] + intervalWidth?: number + conservativeBound?: number + method: "stratified-bootstrap-percentile-v1" | "wilson-score-v1" + resamples?: number + } + status: "passed" | "failed" | "inconclusive" + failures: Array + evidence: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessReplicationRecordResponse = + HarnessReplicationRecordResponses[keyof HarnessReplicationRecordResponses] + +export type HarnessReplicationReceiptData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/replications/receipts/{receiptID}" +} + +export type HarnessReplicationReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessReplicationReceiptError = HarnessReplicationReceiptErrors[keyof HarnessReplicationReceiptErrors] + +export type HarnessReplicationReceiptResponses = { + /** + * Replicated evaluation receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "replicated-evaluation-receipt-v1" + receiptID: string + protocolSHA256: string + contractSHA256: string + sourceSessionID: string + subject: { + type: "run" | "candidate" + id: string + } + metric: string + protocol: { + protocolVersion: "replicated-evaluation-v1" + validatorSHA256: string + environmentSHA256: string + sampling: { + design: "crossed-stratified-cluster-v1" + stratumKind: string + clusterKind: string + strata: Array<{ + id: string + commitmentSHA256: string + }> + clusters: Array<{ + id: string + commitmentSHA256: string + }> + } + estimator: "mean" | "median" | "iqm" | "pass_rate" + interval: + | { + method: "stratified-bootstrap-percentile-v1" + confidence: 0.95 + resamples: number + seed: number + } + | { + method: "wilson-score-v1" + confidence: 0.95 + } + decision: { + rule: "conservative-bound-v1" + direction: "maximize" | "minimize" | "pass" + target: number + maxIntervalWidth?: number + } + failurePolicy: "fail-closed" + } + observations: Array<{ + stratumID: string + clusterID: string + stratumSHA256: string + clusterSHA256: string + status: "passed" | "failed" | "inconclusive" + score?: number + outputSHA256: string + environmentSHA256: string + evidence: Array + evaluatedAt: number + }> + statistics: { + units: number + passed: number + failed: number + inconclusive: number + estimator: "mean" | "median" | "iqm" | "pass_rate" + estimate?: number + confidence: 0.95 + interval?: [number, number] + intervalWidth?: number + conservativeBound?: number + method: "stratified-bootstrap-percentile-v1" | "wilson-score-v1" + resamples?: number + } + status: "passed" | "failed" | "inconclusive" + failures: Array + evidence: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessReplicationReceiptResponse = + HarnessReplicationReceiptResponses[keyof HarnessReplicationReceiptResponses] + +export type HarnessMetaSelectionData = { + body?: { + sessionID: string + metaToken: string + } + path?: never + query?: { + directory?: string + } + url: "/harness/meta/selection" +} + +export type HarnessMetaSelectionErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessMetaSelectionError = HarnessMetaSelectionErrors[keyof HarnessMetaSelectionErrors] + +export type HarnessMetaSelectionResponses = { + /** + * Content-addressed terminal meta-harness selection + */ + 200: { + schemaVersion: 1 + protocolVersion: "meta-harness-selection-v1" + selectionID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + searchRevision: number + stopReason: "budget_exhausted" | "objective_met" | "no_improvement" | "user_cancelled" | "runtime_error" + candidateID: string + candidateArtifact: { + uri: string + sha256: string + } + optimizationResultSHA256: string + optimizationEvaluationSHA256: string + selectedAt: number + } +} + +export type HarnessMetaSelectionResponse = HarnessMetaSelectionResponses[keyof HarnessMetaSelectionResponses] + +export type HarnessMetaRecordData = { + body?: { + schemaVersion: 1 + sessionID: string + metaToken: string + selectionID: string + candidateArtifactSHA256: string + candidateManifestSHA256: string + protectedManifestSHA256: string + validatorSHA256: string + archive: { + uri: string + sha256: string + schemaSHA256: string + indexSHA256: string + contents: "full-source-scores-traces" + query: "filesystem" + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + entries: Array<{ + candidateID: string + artifactSHA256: string + sourceSHA256: string + state: "evaluated" | "unevaluated" + scoresSHA256?: string + resultSHA256?: string + evaluationSHA256?: string + trace?: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + }> + } + refinements: Array<{ + revision: number + scope: "session" + parentSnapshotSHA256: string + snapshotSHA256: string + trigger: string + diagnosis: { + kind: "implementation" | "fundamental" | "inconclusive" + rationale: string + } + rootCause: string + expectedOutcome: string + changes: Array<{ + action: "create" | "update" | "delete" | "rollback" + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + path: string + beforeSHA256?: string + afterSHA256?: string + reason: string + }> + evidence: Array<{ + candidateID: string + traceSHA256: string + messageIndex: number + excerptSHA256: string + }> + predictions: Array<{ + modelID: string + taskID: string + expected: "fail_to_pass" | "remain_pass" + }> + }> + cells: Array< + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "baseline" + } + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "candidate" + loaded: boolean + phases: Array<{ + followed: number + violatedCommission: number + violatedOmission: number + requiredUnobserved: number + notApplicable: number + insufficientEvidence: number + phase: "loaded" | "midpoint" | "pre_final" | "final_validation" + }> + } + > + evaluatedAt: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/meta/receipts" +} + +export type HarnessMetaRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessMetaRecordError = HarnessMetaRecordErrors[keyof HarnessMetaRecordErrors] + +export type HarnessMetaRecordResponses = { + /** + * Immutable meta-harness qualification receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "meta-harness-receipt-v1" + receiptID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + selection: { + schemaVersion: 1 + protocolVersion: "meta-harness-selection-v1" + selectionID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + searchRevision: number + stopReason: "budget_exhausted" | "objective_met" | "no_improvement" | "user_cancelled" | "runtime_error" + candidateID: string + candidateArtifact: { + uri: string + sha256: string + } + optimizationResultSHA256: string + optimizationEvaluationSHA256: string + selectedAt: number + } + candidateManifestSHA256: string + protectedManifestSHA256: string + validatorSHA256: string + archive: { + uri: string + sha256: string + schemaSHA256: string + indexSHA256: string + contents: "full-source-scores-traces" + query: "filesystem" + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + entries: Array<{ + candidateID: string + artifactSHA256: string + sourceSHA256: string + state: "evaluated" | "unevaluated" + scoresSHA256?: string + resultSHA256?: string + evaluationSHA256?: string + trace?: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + }> + } + refinements: Array<{ + revision: number + scope: "session" + parentSnapshotSHA256: string + snapshotSHA256: string + trigger: string + diagnosis: { + kind: "implementation" | "fundamental" | "inconclusive" + rationale: string + } + rootCause: string + expectedOutcome: string + changes: Array<{ + action: "create" | "update" | "delete" | "rollback" + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + path: string + beforeSHA256?: string + afterSHA256?: string + reason: string + }> + evidence: Array<{ + candidateID: string + traceSHA256: string + messageIndex: number + excerptSHA256: string + }> + predictions: Array<{ + modelID: string + taskID: string + expected: "fail_to_pass" | "remain_pass" + }> + }> + cells: Array< + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "baseline" + } + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "candidate" + loaded: boolean + phases: Array<{ + followed: number + violatedCommission: number + violatedOmission: number + requiredUnobserved: number + notApplicable: number + insufficientEvidence: number + phase: "loaded" | "midpoint" | "pre_final" | "final_validation" + }> + } + > + diagnostics: { + updaterGain?: number + beneficiaryGain?: number + worstHeldoutModelGain?: number + activationRate: number + requiredAdherence?: number + finalAdherence?: number + maxPhaseDrift?: number + predictionPrecision: number + riskRegressions: number + maxContextTokens: number + meanContextIncrease: number + loadedBenefit?: number + searchPairs: number + heldoutPairs: number + } + status: "passed" | "failed" | "inconclusive" + failures: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessMetaRecordResponse = HarnessMetaRecordResponses[keyof HarnessMetaRecordResponses] + +export type HarnessMetaReceiptData = { + body?: { + sessionID: string + metaToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/meta/receipts/{receiptID}" +} + +export type HarnessMetaReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessMetaReceiptError = HarnessMetaReceiptErrors[keyof HarnessMetaReceiptErrors] + +export type HarnessMetaReceiptResponses = { + /** + * Canonical meta-harness qualification receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "meta-harness-receipt-v1" + receiptID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + selection: { + schemaVersion: 1 + protocolVersion: "meta-harness-selection-v1" + selectionID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + searchRevision: number + stopReason: "budget_exhausted" | "objective_met" | "no_improvement" | "user_cancelled" | "runtime_error" + candidateID: string + candidateArtifact: { + uri: string + sha256: string + } + optimizationResultSHA256: string + optimizationEvaluationSHA256: string + selectedAt: number + } + candidateManifestSHA256: string + protectedManifestSHA256: string + validatorSHA256: string + archive: { + uri: string + sha256: string + schemaSHA256: string + indexSHA256: string + contents: "full-source-scores-traces" + query: "filesystem" + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + entries: Array<{ + candidateID: string + artifactSHA256: string + sourceSHA256: string + state: "evaluated" | "unevaluated" + scoresSHA256?: string + resultSHA256?: string + evaluationSHA256?: string + trace?: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + }> + } + refinements: Array<{ + revision: number + scope: "session" + parentSnapshotSHA256: string + snapshotSHA256: string + trigger: string + diagnosis: { + kind: "implementation" | "fundamental" | "inconclusive" + rationale: string + } + rootCause: string + expectedOutcome: string + changes: Array<{ + action: "create" | "update" | "delete" | "rollback" + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + path: string + beforeSHA256?: string + afterSHA256?: string + reason: string + }> + evidence: Array<{ + candidateID: string + traceSHA256: string + messageIndex: number + excerptSHA256: string + }> + predictions: Array<{ + modelID: string + taskID: string + expected: "fail_to_pass" | "remain_pass" + }> + }> + cells: Array< + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "baseline" + } + | { + split: "search" | "held_out" + modelID: string + modelCommitment: string + taskID: string + taskCommitment: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + passed?: boolean + contextTokens: number + outputSHA256: string + trace: { + uri: string + sha256: string + schemaSHA256: string + complete: true + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + evidence: Array + role: "candidate" + loaded: boolean + phases: Array<{ + followed: number + violatedCommission: number + violatedOmission: number + requiredUnobserved: number + notApplicable: number + insufficientEvidence: number + phase: "loaded" | "midpoint" | "pre_final" | "final_validation" + }> + } + > + diagnostics: { + updaterGain?: number + beneficiaryGain?: number + worstHeldoutModelGain?: number + activationRate: number + requiredAdherence?: number + finalAdherence?: number + maxPhaseDrift?: number + predictionPrecision: number + riskRegressions: number + maxContextTokens: number + meanContextIncrease: number + loadedBenefit?: number + searchPairs: number + heldoutPairs: number + } + status: "passed" | "failed" | "inconclusive" + failures: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessMetaReceiptResponse = HarnessMetaReceiptResponses[keyof HarnessMetaReceiptResponses] + +export type HarnessConfirmationSelectionData = { + body?: { + sessionID: string + confirmationToken: string + } + path?: never + query?: { + directory?: string + } + url: "/harness/confirmations/selection" +} + +export type HarnessConfirmationSelectionErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessConfirmationSelectionError = + HarnessConfirmationSelectionErrors[keyof HarnessConfirmationSelectionErrors] + +export type HarnessConfirmationSelectionResponses = { + /** + * Immutable terminal winner selection + */ + 200: { + schemaVersion: 1 + protocolVersion: "terminal-verified-best-selection-v1" + selectionID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + searchRevision: number + stopReason: "budget_exhausted" | "objective_met" | "no_improvement" | "user_cancelled" | "runtime_error" + candidateID: string + candidateArtifact: { + uri: string + sha256: string + } + candidateCreatedAt: number + optimizationResultSHA256: string + optimizationEvaluationSHA256: string + selectedAt: number + } +} + +export type HarnessConfirmationSelectionResponse = + HarnessConfirmationSelectionResponses[keyof HarnessConfirmationSelectionResponses] + +export type HarnessConfirmationRecordData = { + body?: { + schemaVersion: 1 + sessionID: string + confirmationToken: string + candidateSHA256: string + manifestSHA256: string + validatorSHA256: string + environmentSHA256: string + outcome: "completed" | "failed" | "inconclusive" + score?: number + metrics?: { + [key: string]: number + } + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + score?: number + evidence?: Array + note?: string + }> + evidence: Array + usage?: { + wallTimeMs?: number + costUSD?: number + } + outputSHA256: string + evaluatedAt: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/confirmations/receipts" +} + +export type HarnessConfirmationRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessConfirmationRecordError = HarnessConfirmationRecordErrors[keyof HarnessConfirmationRecordErrors] + +export type HarnessConfirmationRecordResponses = { + /** + * Immutable sealed confirmation receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "sealed-confirmation-receipt-v1" + receiptID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + selection: { + schemaVersion: 1 + protocolVersion: "terminal-verified-best-selection-v1" + selectionID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + searchRevision: number + stopReason: "budget_exhausted" | "objective_met" | "no_improvement" | "user_cancelled" | "runtime_error" + candidateID: string + candidateArtifact: { + uri: string + sha256: string + } + candidateCreatedAt: number + optimizationResultSHA256: string + optimizationEvaluationSHA256: string + selectedAt: number + } + claim: { + taskID: string + split: "held_out" | "release" + manifestSHA256: string + validatorSHA256: string + environmentSHA256: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + source?: { + repository: string + revision: string + } + metric: string + direction: "maximize" | "minimize" + target: number + } + outcome: "completed" | "failed" | "inconclusive" + status: "passed" | "failed" | "inconclusive" + score?: number + metrics: { + [key: string]: number + } + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + score?: number + evidence?: Array + note?: string + }> + failures: Array + evidence: Array + usage?: { + wallTimeMs?: number + costUSD?: number + } + outputSHA256: string + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessConfirmationRecordResponse = + HarnessConfirmationRecordResponses[keyof HarnessConfirmationRecordResponses] + +export type HarnessConfirmationReceiptData = { + body?: { + sessionID: string + confirmationToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/confirmations/receipts/{receiptID}" +} + +export type HarnessConfirmationReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessConfirmationReceiptError = HarnessConfirmationReceiptErrors[keyof HarnessConfirmationReceiptErrors] + +export type HarnessConfirmationReceiptResponses = { + /** + * Canonical sealed confirmation receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "sealed-confirmation-receipt-v1" + receiptID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + selection: { + schemaVersion: 1 + protocolVersion: "terminal-verified-best-selection-v1" + selectionID: string + contractSHA256: string + protocolSHA256: string + sourceSessionID: string + runID: string + searchRevision: number + stopReason: "budget_exhausted" | "objective_met" | "no_improvement" | "user_cancelled" | "runtime_error" + candidateID: string + candidateArtifact: { + uri: string + sha256: string + } + candidateCreatedAt: number + optimizationResultSHA256: string + optimizationEvaluationSHA256: string + selectedAt: number + } + claim: { + taskID: string + split: "held_out" | "release" + manifestSHA256: string + validatorSHA256: string + environmentSHA256: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + source?: { + repository: string + revision: string + } + metric: string + direction: "maximize" | "minimize" + target: number + } + outcome: "completed" | "failed" | "inconclusive" + status: "passed" | "failed" | "inconclusive" + score?: number + metrics: { + [key: string]: number + } + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + score?: number + evidence?: Array + note?: string + }> + failures: Array + evidence: Array + usage?: { + wallTimeMs?: number + costUSD?: number + } + outputSHA256: string + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessConfirmationReceiptResponse = + HarnessConfirmationReceiptResponses[keyof HarnessConfirmationReceiptResponses] + +export type HarnessSemanticRecordData = { + body?: { + sessionID: string + reviewerToken: string + subject: { + type: "run" | "candidate" + id: string + } + reviews: Array<{ + actor: string + sessionID: string + correctness: "passed" | "failed" | "inconclusive" + alignment: "intended" | "reasonable_alternative" | "misinterpreted" | "ambiguous" + novelty: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + vacuous: boolean + confidence: number + criteria: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + shortcuts: Array<{ + id: string + observed: boolean + evidence: Array + }> + literatureRefs?: Array + evidence: Array + summary: string + reviewedAt: number + }> + } + path?: never + query?: { + directory?: string + } + url: "/harness/semantics/receipts" +} + +export type HarnessSemanticRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessSemanticRecordError = HarnessSemanticRecordErrors[keyof HarnessSemanticRecordErrors] + +export type HarnessSemanticRecordResponses = { + /** + * Immutable semantic audit receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "semantic-audit-receipt-v1" + receiptID: string + protocolSHA256: string + sourceSessionID: string + subject: { + type: "run" | "candidate" + id: string + } + reviewer: { + name: string + version: string + source: "gate" | "human" | "external" + } + scope: { + objectiveSHA256: string + criteria: Array<{ + id: string + requirement: string + }> + forbiddenShortcuts: Array<{ + id: string + description: string + }> + literature: { + cutoff: string + corpusSHA256: string + } + noveltyFloor: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + } + reviews: Array<{ + actor: string + sessionID: string + correctness: "passed" | "failed" | "inconclusive" + alignment: "intended" | "reasonable_alternative" | "misinterpreted" | "ambiguous" + novelty: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + vacuous: boolean + confidence: number + criteria: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + shortcuts: Array<{ + id: string + observed: boolean + evidence: Array + }> + literatureRefs?: Array + evidence: Array + summary: string + reviewedAt: number + }> + status: "meaningful" | "technical_only" | "ambiguous" | "failed" + failures: Array + evidence: Array + reviewedAt: number + recordedAt: number + } +} + +export type HarnessSemanticRecordResponse = HarnessSemanticRecordResponses[keyof HarnessSemanticRecordResponses] + +export type HarnessSemanticReceiptData = { + body?: { + sessionID: string + reviewerToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/semantics/receipts/{receiptID}" +} + +export type HarnessSemanticReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessSemanticReceiptError = HarnessSemanticReceiptErrors[keyof HarnessSemanticReceiptErrors] + +export type HarnessSemanticReceiptResponses = { + /** + * Semantic audit receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "semantic-audit-receipt-v1" + receiptID: string + protocolSHA256: string + sourceSessionID: string + subject: { + type: "run" | "candidate" + id: string + } + reviewer: { + name: string + version: string + source: "gate" | "human" | "external" + } + scope: { + objectiveSHA256: string + criteria: Array<{ + id: string + requirement: string + }> + forbiddenShortcuts: Array<{ + id: string + description: string + }> + literature: { + cutoff: string + corpusSHA256: string + } + noveltyFloor: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + } + reviews: Array<{ + actor: string + sessionID: string + correctness: "passed" | "failed" | "inconclusive" + alignment: "intended" | "reasonable_alternative" | "misinterpreted" | "ambiguous" + novelty: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + vacuous: boolean + confidence: number + criteria: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + shortcuts: Array<{ + id: string + observed: boolean + evidence: Array + }> + literatureRefs?: Array + evidence: Array + summary: string + reviewedAt: number + }> + status: "meaningful" | "technical_only" | "ambiguous" | "failed" + failures: Array + evidence: Array + reviewedAt: number + recordedAt: number + } +} + +export type HarnessSemanticReceiptResponse = HarnessSemanticReceiptResponses[keyof HarnessSemanticReceiptResponses] + +export type HarnessSynthesisRecordData = { + body?: { + sessionID: string + evaluatorToken: string + subject: { + type: "run" | "candidate" + id: string + } + conclusionSHA256: string + evaluatorAuditReceiptID: string + trace: { + owner: "evaluator_runtime" + complete: true + schemaSHA256: string + filterPolicySHA256: string + events: Array<{ + sequence: number + tool: "google_search" | "paper_search" | "web_browse" + requestSHA256: string + responseSHA256: string + sourceSHA256: string + publishedAt?: string + matches: { + forbiddenDomain: boolean + referenceTitle: boolean + } + decision: "allowed" | "blocked" + evidence: Array + }> + } + decomposition: { + status: "passed" | "failed" + outputSHA256?: string + evidence: Array + } + generatedFacts: Array<{ + id: string + commitment: string + verdict: "supported" | "contradicted" | "unsupported" | "judge_error" + evidence: Array + }> + referenceFacts: Array<{ + id: string + commitment: string + coverage: "covered" | "missed" | "judge_error" + evidence: Array + }> + evaluatedAt: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/syntheses/receipts" +} + +export type HarnessSynthesisRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessSynthesisRecordError = HarnessSynthesisRecordErrors[keyof HarnessSynthesisRecordErrors] + +export type HarnessSynthesisRecordResponses = { + /** + * Immutable scientific synthesis receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "scientific-synthesis-receipt-v1" + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + protocolSHA256: string + subject: { + type: "run" | "candidate" + id: string + } + conclusionSHA256: string + evaluatorAuditReceiptID: string + traceSHA256: string + events: Array<{ + sequence: number + tool: "google_search" | "paper_search" | "web_browse" + requestSHA256: string + responseSHA256: string + sourceSHA256: string + publishedAt?: string + matches: { + forbiddenDomain: boolean + referenceTitle: boolean + } + decision: "allowed" | "blocked" + evidence: Array + eventID: string + violations: Array<"forbidden_domain" | "reference_title" | "post_cutoff" | "unknown_date" | "duplicate_output"> + }> + decomposition: { + status: "passed" | "failed" + outputSHA256?: string + evidence: Array + } + generatedFacts: Array<{ + id: string + commitment: string + verdict: "supported" | "contradicted" | "unsupported" | "judge_error" + evidence: Array + }> + referenceFacts: Array<{ + id: string + commitment: string + coverage: "covered" | "missed" | "judge_error" + evidence: Array + }> + metrics: { + toolEvents: number + allowedSources: number + blockedSources: number + violations: { + [key: string]: number + } + generatedFacts: number + supported: number + contradicted: number + unsupported: number + precisionJudgeErrors: number + referenceFacts: number + covered: number + missed: number + recallJudgeErrors: number + precision?: number + recall?: number + f1?: number + } + status: "passed" | "failed" | "inconclusive" + failures: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessSynthesisRecordResponse = HarnessSynthesisRecordResponses[keyof HarnessSynthesisRecordResponses] + +export type HarnessSynthesisReceiptData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/syntheses/receipts/{receiptID}" +} + +export type HarnessSynthesisReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessSynthesisReceiptError = HarnessSynthesisReceiptErrors[keyof HarnessSynthesisReceiptErrors] + +export type HarnessSynthesisReceiptResponses = { + /** + * Canonical scientific synthesis receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "scientific-synthesis-receipt-v1" + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + protocolSHA256: string + subject: { + type: "run" | "candidate" + id: string + } + conclusionSHA256: string + evaluatorAuditReceiptID: string + traceSHA256: string + events: Array<{ + sequence: number + tool: "google_search" | "paper_search" | "web_browse" + requestSHA256: string + responseSHA256: string + sourceSHA256: string + publishedAt?: string + matches: { + forbiddenDomain: boolean + referenceTitle: boolean + } + decision: "allowed" | "blocked" + evidence: Array + eventID: string + violations: Array<"forbidden_domain" | "reference_title" | "post_cutoff" | "unknown_date" | "duplicate_output"> + }> + decomposition: { + status: "passed" | "failed" + outputSHA256?: string + evidence: Array + } + generatedFacts: Array<{ + id: string + commitment: string + verdict: "supported" | "contradicted" | "unsupported" | "judge_error" + evidence: Array + }> + referenceFacts: Array<{ + id: string + commitment: string + coverage: "covered" | "missed" | "judge_error" + evidence: Array + }> + metrics: { + toolEvents: number + allowedSources: number + blockedSources: number + violations: { + [key: string]: number + } + generatedFacts: number + supported: number + contradicted: number + unsupported: number + precisionJudgeErrors: number + referenceFacts: number + covered: number + missed: number + recallJudgeErrors: number + precision?: number + recall?: number + f1?: number + } + status: "passed" | "failed" | "inconclusive" + failures: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessSynthesisReceiptResponse = HarnessSynthesisReceiptResponses[keyof HarnessSynthesisReceiptResponses] + +export type HarnessAutonomyRecordData = { + body?: { + sessionID: string + evaluatorToken: string + subject: { + type: "run" | "candidate" + id: string + } + artifactSHA256: string + trace: { + owner: "evaluator_runtime" + complete: true + recorderArtifactSHA256: string + schemaSHA256: string + classificationPolicySHA256: string + rawLogSHA256: string + startedAt: number + endedAt: number + events: Array<{ + sequence: number + at: number + actor: "benchmark" | "human" | "agent" + kind: + | "problem_statement" + | "clarification" + | "resource_provision" + | "strategy" + | "technical_correction" + | "artifact_edit" + | "candidate_selection" + | "evaluation_feedback" + | "exposition" + | "other" + contribution: "problem" | "auxiliary" | "essential" | "core" | "unclear" + contentSHA256: string + artifactBeforeSHA256?: string + artifactAfterSHA256?: string + evidence: Array + }> + } + } + path?: never + query?: { + directory?: string + } + url: "/harness/autonomy/receipts" +} + +export type HarnessAutonomyRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessAutonomyRecordError = HarnessAutonomyRecordErrors[keyof HarnessAutonomyRecordErrors] + +export type HarnessAutonomyRecordResponses = { + /** + * Immutable backend-derived human-AI autonomy receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "human-ai-autonomy-receipt-v1" + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + protocolSHA256: string + subject: { + type: "run" | "candidate" + id: string + } + artifactSHA256: string + traceSHA256: string + recorderArtifactSHA256: string + rawLogSHA256: string + startedAt: number + endedAt: number + events: Array<{ + sequence: number + at: number + actor: "benchmark" | "human" | "agent" + kind: + | "problem_statement" + | "clarification" + | "resource_provision" + | "strategy" + | "technical_correction" + | "artifact_edit" + | "candidate_selection" + | "evaluation_feedback" + | "exposition" + | "other" + contribution: "problem" | "auxiliary" | "essential" | "core" | "unclear" + contentSHA256: string + artifactBeforeSHA256?: string + artifactAfterSHA256?: string + evidence: Array + priorEventID: string | null + eventID: string + }> + claimedLevel: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + derivedLevel?: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + metrics: { + events: number + counts: { + [key: string]: { + [key: string]: number + } + } + problemEvents: number + humanSubstantiveEvents: number + agentSubstantiveEvents: number + unclearEvents: number + linkedArtifactEvents: number + artifactTransitions: number + finalArtifactLinked: boolean + } + status: "passed" | "failed" | "inconclusive" + failures: Array + recordedAt: number + } +} + +export type HarnessAutonomyRecordResponse = HarnessAutonomyRecordResponses[keyof HarnessAutonomyRecordResponses] + +export type HarnessAutonomyReceiptData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/autonomy/receipts/{receiptID}" +} + +export type HarnessAutonomyReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessAutonomyReceiptError = HarnessAutonomyReceiptErrors[keyof HarnessAutonomyReceiptErrors] + +export type HarnessAutonomyReceiptResponses = { + /** + * Canonical human-AI autonomy receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "human-ai-autonomy-receipt-v1" + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + protocolSHA256: string + subject: { + type: "run" | "candidate" + id: string + } + artifactSHA256: string + traceSHA256: string + recorderArtifactSHA256: string + rawLogSHA256: string + startedAt: number + endedAt: number + events: Array<{ + sequence: number + at: number + actor: "benchmark" | "human" | "agent" + kind: + | "problem_statement" + | "clarification" + | "resource_provision" + | "strategy" + | "technical_correction" + | "artifact_edit" + | "candidate_selection" + | "evaluation_feedback" + | "exposition" + | "other" + contribution: "problem" | "auxiliary" | "essential" | "core" | "unclear" + contentSHA256: string + artifactBeforeSHA256?: string + artifactAfterSHA256?: string + evidence: Array + priorEventID: string | null + eventID: string + }> + claimedLevel: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + derivedLevel?: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + metrics: { + events: number + counts: { + [key: string]: { + [key: string]: number + } + } + problemEvents: number + humanSubstantiveEvents: number + agentSubstantiveEvents: number + unclearEvents: number + linkedArtifactEvents: number + artifactTransitions: number + finalArtifactLinked: boolean + } + status: "passed" | "failed" | "inconclusive" + failures: Array + recordedAt: number + } +} + +export type HarnessAutonomyReceiptResponse = HarnessAutonomyReceiptResponses[keyof HarnessAutonomyReceiptResponses] + +export type HarnessBlueprintInitializeData = { + body?: { + sessionID: string + evaluatorToken: string + } + path?: never + query?: { + directory?: string + } + url: "/harness/proofs/blueprints" +} + +export type HarnessBlueprintInitializeErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessBlueprintInitializeError = HarnessBlueprintInitializeErrors[keyof HarnessBlueprintInitializeErrors] + +export type HarnessBlueprintInitializeResponses = { + /** + * Canonical proof blueprint view + */ + 200: { + schemaVersion: 1 + protocolVersion: "proof-blueprint-view-v1" + runID: string + sessionID: string + rootGoalID: string + summary: { + blueprintID: string + status: "open" | "proved" | "refuted" | "exhausted" + goals: number + proved: number + refuted: number + exhausted: number + decompositions: number + attempts: number + rejected: number + refinements: number + openLeases: number + revision: number + } + goals: Array<{ + statementSHA256: string + declaration: string + module: string + id: string + createdAt: number + status: "open" | "proved" | "refuted" | "exhausted" + ready: boolean + }> + decompositions: Array<{ + id: string + parentID: string + childIDs: Array + informalPlanSHA256: string + sketchArtifactSHA256: string + sketchTranscriptSHA256: string + reviewerTranscriptSHA256: string + attemptID: string + createdAt: number + status: "open" | "closed" | "blocked" + }> + attempts: Array<{ + id: string + ordinal: number + goalID: string + leaseID: string + kind: "direct" | "decomposition" + artifactSHA256: string + result: "proved" | "refuted" | "failed" | "accepted" | "rejected" + claim?: "proof" | "refutation" | "failure" + decompositionID?: string + transcriptSHA256: string + feedbackSHA256: string + failures: Array + startedAt: number + endedAt: number + recordedAt: number + }> + leases: Array<{ + id: string + goalID: string + revision: number + ordinal: number + status: "open" | "consumed" | "expired" + issuedAt: number + expiresAt: number + consumedAt?: number + }> + } +} + +export type HarnessBlueprintInitializeResponse = + HarnessBlueprintInitializeResponses[keyof HarnessBlueprintInitializeResponses] + +export type HarnessBlueprintStatusData = { + body?: { + sessionID: string + evaluatorToken: string + } + path?: never + query?: { + directory?: string + } + url: "/harness/proofs/blueprints/status" +} + +export type HarnessBlueprintStatusErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessBlueprintStatusError = HarnessBlueprintStatusErrors[keyof HarnessBlueprintStatusErrors] + +export type HarnessBlueprintStatusResponses = { + /** + * Backend-derived goal, decomposition, attempt, and lease state + */ + 200: { + schemaVersion: 1 + protocolVersion: "proof-blueprint-view-v1" + runID: string + sessionID: string + rootGoalID: string + summary: { + blueprintID: string + status: "open" | "proved" | "refuted" | "exhausted" + goals: number + proved: number + refuted: number + exhausted: number + decompositions: number + attempts: number + rejected: number + refinements: number + openLeases: number + revision: number + } + goals: Array<{ + statementSHA256: string + declaration: string + module: string + id: string + createdAt: number + status: "open" | "proved" | "refuted" | "exhausted" + ready: boolean + }> + decompositions: Array<{ + id: string + parentID: string + childIDs: Array + informalPlanSHA256: string + sketchArtifactSHA256: string + sketchTranscriptSHA256: string + reviewerTranscriptSHA256: string + attemptID: string + createdAt: number + status: "open" | "closed" | "blocked" + }> + attempts: Array<{ + id: string + ordinal: number + goalID: string + leaseID: string + kind: "direct" | "decomposition" + artifactSHA256: string + result: "proved" | "refuted" | "failed" | "accepted" | "rejected" + claim?: "proof" | "refutation" | "failure" + decompositionID?: string + transcriptSHA256: string + feedbackSHA256: string + failures: Array + startedAt: number + endedAt: number + recordedAt: number + }> + leases: Array<{ + id: string + goalID: string + revision: number + ordinal: number + status: "open" | "consumed" | "expired" + issuedAt: number + expiresAt: number + consumedAt?: number + }> + } +} + +export type HarnessBlueprintStatusResponse = HarnessBlueprintStatusResponses[keyof HarnessBlueprintStatusResponses] + +export type HarnessBlueprintLeaseData = { + body?: { + sessionID: string + evaluatorToken: string + count: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/proofs/blueprints/leases" +} + +export type HarnessBlueprintLeaseErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessBlueprintLeaseError = HarnessBlueprintLeaseErrors[keyof HarnessBlueprintLeaseErrors] + +export type HarnessBlueprintLeaseResponses = { + /** + * Issued leases and updated proof blueprint + */ + 200: { + leases: Array<{ + id: string + goalID: string + revision: number + ordinal: number + status: "open" | "consumed" | "expired" + issuedAt: number + expiresAt: number + consumedAt?: number + }> + state: { + schemaVersion: 1 + protocolVersion: "proof-blueprint-view-v1" + runID: string + sessionID: string + rootGoalID: string + summary: { + blueprintID: string + status: "open" | "proved" | "refuted" | "exhausted" + goals: number + proved: number + refuted: number + exhausted: number + decompositions: number + attempts: number + rejected: number + refinements: number + openLeases: number + revision: number + } + goals: Array<{ + statementSHA256: string + declaration: string + module: string + id: string + createdAt: number + status: "open" | "proved" | "refuted" | "exhausted" + ready: boolean + }> + decompositions: Array<{ + id: string + parentID: string + childIDs: Array + informalPlanSHA256: string + sketchArtifactSHA256: string + sketchTranscriptSHA256: string + reviewerTranscriptSHA256: string + attemptID: string + createdAt: number + status: "open" | "closed" | "blocked" + }> + attempts: Array<{ + id: string + ordinal: number + goalID: string + leaseID: string + kind: "direct" | "decomposition" + artifactSHA256: string + result: "proved" | "refuted" | "failed" | "accepted" | "rejected" + claim?: "proof" | "refutation" | "failure" + decompositionID?: string + transcriptSHA256: string + feedbackSHA256: string + failures: Array + startedAt: number + endedAt: number + recordedAt: number + }> + leases: Array<{ + id: string + goalID: string + revision: number + ordinal: number + status: "open" | "consumed" | "expired" + issuedAt: number + expiresAt: number + consumedAt?: number + }> + } + } +} + +export type HarnessBlueprintLeaseResponse = HarnessBlueprintLeaseResponses[keyof HarnessBlueprintLeaseResponses] + +export type HarnessBlueprintRecordData = { + body?: + | { + sessionID: string + evaluatorToken: string + kind: "direct" + leaseID: string + artifactSHA256: string + claim: "proof" | "refutation" | "failure" + verification: { + compilerArtifactSHA256: string + statementMatched: boolean + exitCode: number + warnings: number + transcriptSHA256: string + feedbackSHA256: string + startedAt: number + endedAt: number + } + } + | { + sessionID: string + evaluatorToken: string + kind: "decomposition" + leaseID: string + informalPlanSHA256: string + artifactSHA256: string + children: Array<{ + statementSHA256: string + declaration: string + module: string + }> + verification: { + compilerArtifactSHA256: string + statementMatched: boolean + exitCode: number + warnings: number + transcriptSHA256: string + feedbackSHA256: string + startedAt: number + endedAt: number + validatorArtifactSHA256: string + placeholderDeclarations: Array + validatorTranscriptSHA256: string + } + review: { + reviewerArtifactSHA256: string + promptSHA256: string + relevant: boolean + easier: boolean + plausible: boolean + transcriptSHA256: string + } + } + path?: never + query?: { + directory?: string + } + url: "/harness/proofs/blueprints/attempts" +} + +export type HarnessBlueprintRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessBlueprintRecordError = HarnessBlueprintRecordErrors[keyof HarnessBlueprintRecordErrors] + +export type HarnessBlueprintRecordResponses = { + /** + * Recorded attempt and updated proof blueprint + */ + 200: { + attemptID?: string + decompositionID?: string + state: { + schemaVersion: 1 + protocolVersion: "proof-blueprint-view-v1" + runID: string + sessionID: string + rootGoalID: string + summary: { + blueprintID: string + status: "open" | "proved" | "refuted" | "exhausted" + goals: number + proved: number + refuted: number + exhausted: number + decompositions: number + attempts: number + rejected: number + refinements: number + openLeases: number + revision: number + } + goals: Array<{ + statementSHA256: string + declaration: string + module: string + id: string + createdAt: number + status: "open" | "proved" | "refuted" | "exhausted" + ready: boolean + }> + decompositions: Array<{ + id: string + parentID: string + childIDs: Array + informalPlanSHA256: string + sketchArtifactSHA256: string + sketchTranscriptSHA256: string + reviewerTranscriptSHA256: string + attemptID: string + createdAt: number + status: "open" | "closed" | "blocked" + }> + attempts: Array<{ + id: string + ordinal: number + goalID: string + leaseID: string + kind: "direct" | "decomposition" + artifactSHA256: string + result: "proved" | "refuted" | "failed" | "accepted" | "rejected" + claim?: "proof" | "refutation" | "failure" + decompositionID?: string + transcriptSHA256: string + feedbackSHA256: string + failures: Array + startedAt: number + endedAt: number + recordedAt: number + }> + leases: Array<{ + id: string + goalID: string + revision: number + ordinal: number + status: "open" | "consumed" | "expired" + issuedAt: number + expiresAt: number + consumedAt?: number + }> + } + } +} + +export type HarnessBlueprintRecordResponse = HarnessBlueprintRecordResponses[keyof HarnessBlueprintRecordResponses] + +export type HarnessFormalRecordData = { + body?: { + sessionID: string + evaluatorToken: string + subject: { + type: "run" | "candidate" + id: string + } + artifactSHA256: string + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256: string + statementSHA256: string + declaration: string + module: string + environment: { + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + } + manifest: { + complete: boolean + files: Array<{ + path: string + role: + | "challenge" + | "statement" + | "proof" + | "lean_toolchain" + | "lake_manifest" + | "dependency_tree" + | "config" + | "support" + sha256: string + }> + } + verification: { + startedAt: number + endedAt: number + build: { + verifierArtifactSHA256: string + exitCode: number + warnings: number + transcriptSHA256: string + } + source: { + verifierArtifactSHA256: string + complete: boolean + findings: Array<{ + construct: "sorry" | "admit" | "debug.skipKernelTC" | "native_decide" + path: string + line: number + }> + transcriptSHA256: string + } + axioms: { + verifierArtifactSHA256: string + complete: boolean + typesTraversed: boolean + observed: Array + transcriptSHA256: string + } + fresh?: { + verifierArtifactSHA256: string + fresh: boolean + exitCode: number + transcriptSHA256: string + } + external?: { + comparatorArtifactSHA256: string + sandboxImageSHA256: string + sandboxed: boolean + challengeMatched: boolean + proofTermSHA256: string + transcriptSHA256: string + checks: [ + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + ] + } + } + } + path?: never + query?: { + directory?: string + } + url: "/harness/proofs/receipts" +} + +export type HarnessFormalRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessFormalRecordError = HarnessFormalRecordErrors[keyof HarnessFormalRecordErrors] + +export type HarnessFormalRecordResponses = { + /** + * Immutable backend-derived formal proof receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "formal-proof-receipt-v1" + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + protocolSHA256: string + subject: { + type: "run" | "candidate" + id: string + } + artifactSHA256: string + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256: string + statementSHA256: string + declaration: string + module: string + environment: { + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + } + manifestSHA256: string + files: Array<{ + path: string + role: + | "challenge" + | "statement" + | "proof" + | "lean_toolchain" + | "lake_manifest" + | "dependency_tree" + | "config" + | "support" + sha256: string + }> + verification: { + startedAt: number + endedAt: number + build: { + verifierArtifactSHA256: string + exitCode: number + warnings: number + transcriptSHA256: string + } + source: { + verifierArtifactSHA256: string + complete: boolean + findings: Array<{ + construct: "sorry" | "admit" | "debug.skipKernelTC" | "native_decide" + path: string + line: number + }> + transcriptSHA256: string + } + axioms: { + verifierArtifactSHA256: string + complete: boolean + typesTraversed: boolean + observed: Array + transcriptSHA256: string + } + fresh?: { + verifierArtifactSHA256: string + fresh: boolean + exitCode: number + transcriptSHA256: string + } + external?: { + comparatorArtifactSHA256: string + sandboxImageSHA256: string + sandboxed: boolean + challengeMatched: boolean + proofTermSHA256: string + transcriptSHA256: string + checks: [ + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + ] + } + } + tier: "kernel" | "fresh_recheck" | "external_crosscheck" + metrics: { + files: number + warnings: number + observedAxioms: number + disallowedAxioms: Array + manifestComplete: boolean + buildAccepted: boolean + sourceAuditAccepted: boolean + forbiddenFindings: Array<{ + construct: "sorry" | "admit" | "debug.skipKernelTC" | "native_decide" + path: string + line: number + }> + axiomAuditAccepted: boolean + freshRecheckAccepted: boolean + externalCrosscheckAccepted: boolean + statementMatched: boolean + } + status: "passed" | "failed" + failures: Array + recordedAt: number + } +} + +export type HarnessFormalRecordResponse = HarnessFormalRecordResponses[keyof HarnessFormalRecordResponses] + +export type HarnessFormalReceiptData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/proofs/receipts/{receiptID}" +} + +export type HarnessFormalReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessFormalReceiptError = HarnessFormalReceiptErrors[keyof HarnessFormalReceiptErrors] + +export type HarnessFormalReceiptResponses = { + /** + * Canonical formal proof receipt + */ + 200: { + schemaVersion: 1 + protocolVersion: "formal-proof-receipt-v1" + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + protocolSHA256: string + subject: { + type: "run" | "candidate" + id: string + } + artifactSHA256: string + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256: string + statementSHA256: string + declaration: string + module: string + environment: { + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + } + manifestSHA256: string + files: Array<{ + path: string + role: + | "challenge" + | "statement" + | "proof" + | "lean_toolchain" + | "lake_manifest" + | "dependency_tree" + | "config" + | "support" + sha256: string + }> + verification: { + startedAt: number + endedAt: number + build: { + verifierArtifactSHA256: string + exitCode: number + warnings: number + transcriptSHA256: string + } + source: { + verifierArtifactSHA256: string + complete: boolean + findings: Array<{ + construct: "sorry" | "admit" | "debug.skipKernelTC" | "native_decide" + path: string + line: number + }> + transcriptSHA256: string + } + axioms: { + verifierArtifactSHA256: string + complete: boolean + typesTraversed: boolean + observed: Array + transcriptSHA256: string + } + fresh?: { + verifierArtifactSHA256: string + fresh: boolean + exitCode: number + transcriptSHA256: string + } + external?: { + comparatorArtifactSHA256: string + sandboxImageSHA256: string + sandboxed: boolean + challengeMatched: boolean + proofTermSHA256: string + transcriptSHA256: string + checks: [ + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + { + role: "lean_kernel" | "external_checker" + verifierArtifactSHA256: string + accepted: boolean + transcriptSHA256: string + }, + ] + } + } + tier: "kernel" | "fresh_recheck" | "external_crosscheck" + metrics: { + files: number + warnings: number + observedAxioms: number + disallowedAxioms: Array + manifestComplete: boolean + buildAccepted: boolean + sourceAuditAccepted: boolean + forbiddenFindings: Array<{ + construct: "sorry" | "admit" | "debug.skipKernelTC" | "native_decide" + path: string + line: number + }> + axiomAuditAccepted: boolean + freshRecheckAccepted: boolean + externalCrosscheckAccepted: boolean + statementMatched: boolean + } + status: "passed" | "failed" + failures: Array + recordedAt: number + } +} + +export type HarnessFormalReceiptResponse = HarnessFormalReceiptResponses[keyof HarnessFormalReceiptResponses] + +export type HarnessIntegrityRecordData = { + body?: { + schemaVersion: 1 + runID: string + sessionID: string + evaluatorToken: string + protocol: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + subject: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + trace: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + events: number + dropped: number + startedAt: number + endedAt: number + } + model: { + name: string + baseArtifactSHA256: string + configSHA256: string + outputArtifactSHA256: string + lineageVerified: boolean + } + audits: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + ] + activity: { + unapprovedExternalModelCalls: number + benchmarkLookupEvents: number + hiddenCanaryManifestSHA256: string + hiddenCanariesTested: number + hiddenCanaryViolations: number + } + validator: { + name: "verify-benchmark-integrity" + version: 1 + scriptSHA256: string + } + evidence: Array + evaluatedAt: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/integrity/receipts" +} + +export type HarnessIntegrityRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessIntegrityRecordError = HarnessIntegrityRecordErrors[keyof HarnessIntegrityRecordErrors] + +export type HarnessIntegrityRecordResponses = { + /** + * Immutable runtime integrity receipt + */ + 200: { + schemaVersion: 1 + receiptID: string + submissionID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + protocol: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + trace: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + events: number + dropped: number + startedAt: number + endedAt: number + } + traceCoverage: number + model: { + name: string + baseArtifactSHA256: string + configSHA256: string + outputArtifactSHA256: string + lineageVerified: boolean + } + audits: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + ] + activity: { + unapprovedExternalModelCalls: number + benchmarkLookupEvents: number + hiddenCanaryManifestSHA256: string + hiddenCanariesTested: number + hiddenCanaryViolations: number + } + validator: { + name: "verify-benchmark-integrity" + version: 1 + scriptSHA256: string + } + checks: { + traceCompleteness: boolean + modelIdentity: boolean + testItemContamination: boolean + externalModelUse: boolean + benchmarkLookup: boolean + hiddenCanary: boolean + } + status: "passed" | "failed" + failures: Array< + | "trace_schema" + | "trace_event_floor" + | "trace_coverage" + | "model_name" + | "model_base_artifact" + | "model_config" + | "model_lineage" + | "forbidden_model_artifact" + | "test_item_contamination" + | "external_model_use" + | "benchmark_lookup" + | "hidden_canary_manifest" + | "hidden_canary_coverage" + | "hidden_canary_violation" + > + evidence: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessIntegrityRecordResponse = HarnessIntegrityRecordResponses[keyof HarnessIntegrityRecordResponses] + +export type HarnessIntegrityReceiptData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/integrity/receipts/{receiptID}" +} + +export type HarnessIntegrityReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessIntegrityReceiptError = HarnessIntegrityReceiptErrors[keyof HarnessIntegrityReceiptErrors] + +export type HarnessIntegrityReceiptResponses = { + /** + * Runtime integrity receipt + */ + 200: { + schemaVersion: 1 + receiptID: string + submissionID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + protocol: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + trace: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + events: number + dropped: number + startedAt: number + endedAt: number + } + traceCoverage: number + model: { + name: string + baseArtifactSHA256: string + configSHA256: string + outputArtifactSHA256: string + lineageVerified: boolean + } + audits: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + decision: "clean" | "flagged" | "abstain" + confidence: number + evidence: Array + }, + ] + activity: { + unapprovedExternalModelCalls: number + benchmarkLookupEvents: number + hiddenCanaryManifestSHA256: string + hiddenCanariesTested: number + hiddenCanaryViolations: number + } + validator: { + name: "verify-benchmark-integrity" + version: 1 + scriptSHA256: string + } + checks: { + traceCompleteness: boolean + modelIdentity: boolean + testItemContamination: boolean + externalModelUse: boolean + benchmarkLookup: boolean + hiddenCanary: boolean + } + status: "passed" | "failed" + failures: Array< + | "trace_schema" + | "trace_event_floor" + | "trace_coverage" + | "model_name" + | "model_base_artifact" + | "model_config" + | "model_lineage" + | "forbidden_model_artifact" + | "test_item_contamination" + | "external_model_use" + | "benchmark_lookup" + | "hidden_canary_manifest" + | "hidden_canary_coverage" + | "hidden_canary_violation" + > + evidence: Array + evaluatedAt: number + recordedAt: number + } | null +} + +export type HarnessIntegrityReceiptResponse = HarnessIntegrityReceiptResponses[keyof HarnessIntegrityReceiptResponses] + +export type HarnessEvolutionRecordData = { + body?: { + schemaVersion: 1 + runID: string + sessionID: string + evaluatorToken: string + protocol: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + snapshot: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + files: Array<{ + path: string + sha256: string + bytes: number + lineHashes: Array + }> + } + parents: Array<{ + id: string + artifact: { + uri: string + sha256: string + } + receiptID: string + snapshotSHA256: string + delta: { + uri: string + sha256: string + } + }> + validator: { + name: "trace-evolutionary-candidate" + version: 1 + scriptSHA256: string + } + evidence: Array + evaluatedAt: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/evolution/receipts" +} + +export type HarnessEvolutionRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessEvolutionRecordError = HarnessEvolutionRecordErrors[keyof HarnessEvolutionRecordErrors] + +export type HarnessEvolutionRecordResponses = { + /** + * Immutable evolution trace receipt + */ + 200: { + schemaVersion: 1 + receiptID: string + submissionID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + protocol: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + snapshot: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + files: Array<{ + path: string + sha256: string + bytes: number + lineHashes: Array + }> + } + parents: Array<{ + id: string + artifact: { + uri: string + sha256: string + } + receiptID: string + snapshotSHA256: string + delta: { + uri: string + sha256: string + } + }> + validator: { + name: "trace-evolutionary-candidate" + version: 1 + scriptSHA256: string + } + diagnostics: { + files: number + bytes: number + sourceLines: number + depth: number + ancestors: number + addedLines: number + deletedLines: number + ancestralDeletedLines: number + reintroducedLines: number + reintroducedHashes: number + reintroducedFraction: number + novelLines: number + sourceChanged: boolean + cycleDetected: boolean + parents: Array<{ + id: string + receiptID: string + filesChanged: number + addedLines: number + deletedLines: number + }> + } + evidence: Array + evaluatedAt: number + recordedAt: number + } +} + +export type HarnessEvolutionRecordResponse = HarnessEvolutionRecordResponses[keyof HarnessEvolutionRecordResponses] + +export type HarnessEvolutionReceiptData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/evolution/receipts/{receiptID}" +} + +export type HarnessEvolutionReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessEvolutionReceiptError = HarnessEvolutionReceiptErrors[keyof HarnessEvolutionReceiptErrors] + +export type HarnessEvolutionReceiptResponses = { + /** + * Evolution trace receipt + */ + 200: { + schemaVersion: 1 + receiptID: string + submissionID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + protocol: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + snapshot: { + artifact: { + uri: string + sha256: string + } + schemaSHA256: string + files: Array<{ + path: string + sha256: string + bytes: number + lineHashes: Array + }> + } + parents: Array<{ + id: string + artifact: { + uri: string + sha256: string + } + receiptID: string + snapshotSHA256: string + delta: { + uri: string + sha256: string + } + }> + validator: { + name: "trace-evolutionary-candidate" + version: 1 + scriptSHA256: string + } + diagnostics: { + files: number + bytes: number + sourceLines: number + depth: number + ancestors: number + addedLines: number + deletedLines: number + ancestralDeletedLines: number + reintroducedLines: number + reintroducedHashes: number + reintroducedFraction: number + novelLines: number + sourceChanged: boolean + cycleDetected: boolean + parents: Array<{ + id: string + receiptID: string + filesChanged: number + addedLines: number + deletedLines: number + }> + } + evidence: Array + evaluatedAt: number + recordedAt: number + } | null +} + +export type HarnessEvolutionReceiptResponse = HarnessEvolutionReceiptResponses[keyof HarnessEvolutionReceiptResponses] + +export type HarnessSimulationRecordData = { + body?: { + schemaVersion: 1 + runID: string + sessionID: string + evaluatorToken: string + subject: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + engine: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256: string + reference: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validationInputSHA256: string + levels: Array<{ + label: string + h: number + error: number + residual: number + invariants: { + [key: string]: number + } + }> + stressTests: Array<{ + id: + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + evidence: Array + evaluatedAt: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/simulations/receipts" +} + +export type HarnessSimulationRecordErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessSimulationRecordError = HarnessSimulationRecordErrors[keyof HarnessSimulationRecordErrors] + +export type HarnessSimulationRecordResponses = { + /** + * Immutable simulator validation receipt + */ + 200: { + schemaVersion: 1 + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + engine: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256: string + reference: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validationInputSHA256: string + levels: Array<{ + label: string + h: number + error: number + residual: number + invariants: { + [key: string]: number + } + }> + observedOrders: Array + medianObservedOrder: number + stressTests: Array<{ + id: + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + checks: { + enoughLevels: boolean + resolutionDecreases: boolean + errorDecreases: boolean + observedOrder: boolean + residualBound: boolean + invariants: { + [key: string]: boolean + } + stressTests: { + [key: string]: boolean + } + } + status: "passed" | "failed" + evidence: Array + evaluatedAt: number + } +} + +export type HarnessSimulationRecordResponse = HarnessSimulationRecordResponses[keyof HarnessSimulationRecordResponses] + +export type HarnessSimulationReceiptData = { + body?: { + sessionID: string + evaluatorToken: string + } + path: { + receiptID: string + } + query?: { + directory?: string + } + url: "/harness/simulations/receipts/{receiptID}" +} + +export type HarnessSimulationReceiptErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessSimulationReceiptError = HarnessSimulationReceiptErrors[keyof HarnessSimulationReceiptErrors] + +export type HarnessSimulationReceiptResponses = { + /** + * Simulator validation receipt + */ + 200: { + schemaVersion: 1 + receiptID: string + runID: string + sessionID: string + contractFingerprint: string + subject: { + type: "run" | "candidate" + id: string + artifact: { + uri: string + sha256: string + } + } + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + engine: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256: string + reference: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validationInputSHA256: string + levels: Array<{ + label: string + h: number + error: number + residual: number + invariants: { + [key: string]: number + } + }> + observedOrders: Array + medianObservedOrder: number + stressTests: Array<{ + id: + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + status: "passed" | "failed" | "inconclusive" + evidence: Array + }> + checks: { + enoughLevels: boolean + resolutionDecreases: boolean + errorDecreases: boolean + observedOrder: boolean + residualBound: boolean + invariants: { + [key: string]: boolean + } + stressTests: { + [key: string]: boolean + } + } + status: "passed" | "failed" + evidence: Array + evaluatedAt: number + } | null +} + +export type HarnessSimulationReceiptResponse = + HarnessSimulationReceiptResponses[keyof HarnessSimulationReceiptResponses] + +export type HarnessOrchestrationStatusData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/orchestration" +} + +export type HarnessOrchestrationStatusErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessOrchestrationStatusError = HarnessOrchestrationStatusErrors[keyof HarnessOrchestrationStatusErrors] + +export type HarnessOrchestrationStatusResponses = { + /** + * Scientific orchestration state + */ + 200: { + schemaVersion: 3 + protocolVersion: "coalition-v1" | "coalition-v2" | "coalition-v3" + sessionPolicy: "legacy-v1" | "fresh-v1" | "producer-lanes-v1" + workerPolicy: "claimed-v1" | "task-attested-v1" + runID: string + sessionID: string + contractFingerprint: string + objective: string + selection: { + topology: "solo" | "centralized" | "fork_join" | "tournament" | "evolution" | "verifier_loop" + source: "contract" | "policy" + reasons: Array + traits: { + decomposability: number + sequentiality: number + toolIntensity: number + uncertainty: number + verificationRisk: number + novelty: number + crossDomain: number + } + } + maxWorkers: number + maxRounds: number + minIndependentVerifiers: number + status: "active" | "awaiting_checkpoint" | "completed" + adaptive?: { + protocolVersion: "marginal-utility-v1" + minRounds: number + patience: number + minUtilityGain: number + maxUncertainty: number + targetUtility?: number + checkpoints: Array<{ + round: number + utility: number + uncertainty: number + evidenceRefs: Array + evaluatedAt: number + id: string + gain: number | null + qualified: boolean + recordedAt: number + }> + stalled: number + phase: "searching" | "finalizing" + stopReason?: "target_reached" | "marginal_utility_exhausted" | "max_rounds" + } + repair?: { + protocolVersion: "verifier-routed-v1" + minConfidence: number + phase: "producing" | "verifying" | "investigating" | "completed" + candidateID: string + verifierIDs: Array + evidenceID?: string + routes: Array<{ + id: string + attempt: number + candidateID: string + actionID?: string + verifierIDs: Array + decision: "accept" | "revise" | "restart" | "investigate" + confidence: number + evidenceRefs: Array + recordedAt: number + }> + stopReason?: "accepted" | "attempt_limit" | "work_failed" + } + consensus?: { + status: "supported" | "rejected" | "disputed" | "insufficient" + verifierCount: number + support: number + reject: number + abstain: number + confidence: number + evidenceRefs: Array + provisional: true + derivedAt: number + } + work: { + [key: string]: { + id: string + role: + | "generation" + | "proximity" + | "reflection" + | "ranking" + | "evolution" + | "revision" + | "verification" + | "investigation" + | "simulation" + | "synthesis" + label: string + round: number + agent: "task" | "biology" | "physics" | "ml" | "critique" | "physics-critique" | "reviewer" + dependencies: Array + prompt: string + allocation: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + lane?: "producer-a" | "producer-b" + status: "pending" | "executed" | "completed" | "failed" | "cancelled" + workerSessionID?: string + workerReceipt?: { + id: string + workID: string + workerSessionID: string + turnID: string + agent: "task" | "biology" | "physics" | "ml" | "critique" | "physics-critique" | "reviewer" + workPromptSHA256: string + taskPromptSHA256: string + outcome: "completed" | "failed" + usage: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + toolCalls: number + failedToolCalls: number + startedAt: number + completedAt: number + provisional: true + } + result?: { + summary: string + artifactRefs?: Array + evidenceRefs?: Array + usage?: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + verdict?: { + decision: "support" | "reject" | "abstain" + severity?: "none" | "minor" | "critical" | "unknown" + confidence: number + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + evidenceRefs: Array + }> + } + completedAt: number + } + failure?: string + } + } + order: Array + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessOrchestrationStatusResponse = + HarnessOrchestrationStatusResponses[keyof HarnessOrchestrationStatusResponses] + +export type HarnessOrchestrationStartData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/orchestration" +} + +export type HarnessOrchestrationStartErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessOrchestrationStartError = HarnessOrchestrationStartErrors[keyof HarnessOrchestrationStartErrors] + +export type HarnessOrchestrationStartResponses = { + /** + * Scientific orchestration state + */ + 200: { + schemaVersion: 3 + protocolVersion: "coalition-v1" | "coalition-v2" | "coalition-v3" + sessionPolicy: "legacy-v1" | "fresh-v1" | "producer-lanes-v1" + workerPolicy: "claimed-v1" | "task-attested-v1" + runID: string + sessionID: string + contractFingerprint: string + objective: string + selection: { + topology: "solo" | "centralized" | "fork_join" | "tournament" | "evolution" | "verifier_loop" + source: "contract" | "policy" + reasons: Array + traits: { + decomposability: number + sequentiality: number + toolIntensity: number + uncertainty: number + verificationRisk: number + novelty: number + crossDomain: number + } + } + maxWorkers: number + maxRounds: number + minIndependentVerifiers: number + status: "active" | "awaiting_checkpoint" | "completed" + adaptive?: { + protocolVersion: "marginal-utility-v1" + minRounds: number + patience: number + minUtilityGain: number + maxUncertainty: number + targetUtility?: number + checkpoints: Array<{ + round: number + utility: number + uncertainty: number + evidenceRefs: Array + evaluatedAt: number + id: string + gain: number | null + qualified: boolean + recordedAt: number + }> + stalled: number + phase: "searching" | "finalizing" + stopReason?: "target_reached" | "marginal_utility_exhausted" | "max_rounds" + } + repair?: { + protocolVersion: "verifier-routed-v1" + minConfidence: number + phase: "producing" | "verifying" | "investigating" | "completed" + candidateID: string + verifierIDs: Array + evidenceID?: string + routes: Array<{ + id: string + attempt: number + candidateID: string + actionID?: string + verifierIDs: Array + decision: "accept" | "revise" | "restart" | "investigate" + confidence: number + evidenceRefs: Array + recordedAt: number + }> + stopReason?: "accepted" | "attempt_limit" | "work_failed" + } + consensus?: { + status: "supported" | "rejected" | "disputed" | "insufficient" + verifierCount: number + support: number + reject: number + abstain: number + confidence: number + evidenceRefs: Array + provisional: true + derivedAt: number + } + work: { + [key: string]: { + id: string + role: + | "generation" + | "proximity" + | "reflection" + | "ranking" + | "evolution" + | "revision" + | "verification" + | "investigation" + | "simulation" + | "synthesis" + label: string + round: number + agent: "task" | "biology" | "physics" | "ml" | "critique" | "physics-critique" | "reviewer" + dependencies: Array + prompt: string + allocation: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + lane?: "producer-a" | "producer-b" + status: "pending" | "executed" | "completed" | "failed" | "cancelled" + workerSessionID?: string + workerReceipt?: { + id: string + workID: string + workerSessionID: string + turnID: string + agent: "task" | "biology" | "physics" | "ml" | "critique" | "physics-critique" | "reviewer" + workPromptSHA256: string + taskPromptSHA256: string + outcome: "completed" | "failed" + usage: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + toolCalls: number + failedToolCalls: number + startedAt: number + completedAt: number + provisional: true + } + result?: { + summary: string + artifactRefs?: Array + evidenceRefs?: Array + usage?: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + verdict?: { + decision: "support" | "reject" | "abstain" + severity?: "none" | "minor" | "critical" | "unknown" + confidence: number + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + evidenceRefs: Array + }> + } + completedAt: number + } + failure?: string + } + } + order: Array + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessOrchestrationStartResponse = + HarnessOrchestrationStartResponses[keyof HarnessOrchestrationStartResponses] + +export type HarnessOrchestrationCheckpointData = { + body?: { + evaluatorToken: string + round: number + utility: number + uncertainty: number + evidenceRefs: Array + evaluatedAt: number + } + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/orchestration/checkpoints" +} + +export type HarnessOrchestrationCheckpointErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessOrchestrationCheckpointError = + HarnessOrchestrationCheckpointErrors[keyof HarnessOrchestrationCheckpointErrors] + +export type HarnessOrchestrationCheckpointResponses = { + /** + * Scientific orchestration state + */ + 200: { + schemaVersion: 3 + protocolVersion: "coalition-v1" | "coalition-v2" | "coalition-v3" + sessionPolicy: "legacy-v1" | "fresh-v1" | "producer-lanes-v1" + workerPolicy: "claimed-v1" | "task-attested-v1" + runID: string + sessionID: string + contractFingerprint: string + objective: string + selection: { + topology: "solo" | "centralized" | "fork_join" | "tournament" | "evolution" | "verifier_loop" + source: "contract" | "policy" + reasons: Array + traits: { + decomposability: number + sequentiality: number + toolIntensity: number + uncertainty: number + verificationRisk: number + novelty: number + crossDomain: number + } + } + maxWorkers: number + maxRounds: number + minIndependentVerifiers: number + status: "active" | "awaiting_checkpoint" | "completed" + adaptive?: { + protocolVersion: "marginal-utility-v1" + minRounds: number + patience: number + minUtilityGain: number + maxUncertainty: number + targetUtility?: number + checkpoints: Array<{ + round: number + utility: number + uncertainty: number + evidenceRefs: Array + evaluatedAt: number + id: string + gain: number | null + qualified: boolean + recordedAt: number + }> + stalled: number + phase: "searching" | "finalizing" + stopReason?: "target_reached" | "marginal_utility_exhausted" | "max_rounds" + } + repair?: { + protocolVersion: "verifier-routed-v1" + minConfidence: number + phase: "producing" | "verifying" | "investigating" | "completed" + candidateID: string + verifierIDs: Array + evidenceID?: string + routes: Array<{ + id: string + attempt: number + candidateID: string + actionID?: string + verifierIDs: Array + decision: "accept" | "revise" | "restart" | "investigate" + confidence: number + evidenceRefs: Array + recordedAt: number + }> + stopReason?: "accepted" | "attempt_limit" | "work_failed" + } + consensus?: { + status: "supported" | "rejected" | "disputed" | "insufficient" + verifierCount: number + support: number + reject: number + abstain: number + confidence: number + evidenceRefs: Array + provisional: true + derivedAt: number + } + work: { + [key: string]: { + id: string + role: + | "generation" + | "proximity" + | "reflection" + | "ranking" + | "evolution" + | "revision" + | "verification" + | "investigation" + | "simulation" + | "synthesis" + label: string + round: number + agent: "task" | "biology" | "physics" | "ml" | "critique" | "physics-critique" | "reviewer" + dependencies: Array + prompt: string + allocation: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + lane?: "producer-a" | "producer-b" + status: "pending" | "executed" | "completed" | "failed" | "cancelled" + workerSessionID?: string + workerReceipt?: { + id: string + workID: string + workerSessionID: string + turnID: string + agent: "task" | "biology" | "physics" | "ml" | "critique" | "physics-critique" | "reviewer" + workPromptSHA256: string + taskPromptSHA256: string + outcome: "completed" | "failed" + usage: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + toolCalls: number + failedToolCalls: number + startedAt: number + completedAt: number + provisional: true + } + result?: { + summary: string + artifactRefs?: Array + evidenceRefs?: Array + usage?: { + steps?: number + tokens?: number + costUSD?: number + wallTimeMs?: number + } + verdict?: { + decision: "support" | "reject" | "abstain" + severity?: "none" | "minor" | "critical" | "unknown" + confidence: number + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + evidenceRefs: Array + }> + } + completedAt: number + } + failure?: string + } + } + order: Array + revision: number + createdAt: number + updatedAt: number + } +} + +export type HarnessOrchestrationCheckpointResponse = + HarnessOrchestrationCheckpointResponses[keyof HarnessOrchestrationCheckpointResponses] + +export type HarnessWorldStatusData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/world" +} + +export type HarnessWorldStatusErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessWorldStatusError = HarnessWorldStatusErrors[keyof HarnessWorldStatusErrors] + +export type HarnessWorldStatusResponses = { + /** + * Continual world-model state + */ + 200: { + schemaVersion: 1 + sessionID: string + runID: string + basePromptSHA256: string + entries: { + [key: string]: { + id: string + key: string + kind: "hypothesis" | "observation" | "strategy" | "memory" | "skill" | "subagent" + content: string + confidence: number + evidence: Array<{ + ref: string + authority: "self" | "tool" | "evaluator" | "human" + }> + updatedAt: number + revision: number + } + } + events: Array<{ + id: string + type: "analysis" | "tool" | "evaluation" | "failure" | "milestone" | "stagnation" | "manual" + summary: string + evidenceRefs: Array + changed: boolean + createdAt: number + }> + snapshots: Array<{ + revision: number + entries: { + [key: string]: { + id: string + key: string + kind: "hypothesis" | "observation" | "strategy" | "memory" | "skill" | "subagent" + content: string + confidence: number + evidence: Array<{ + ref: string + authority: "self" | "tool" | "evaluator" | "human" + }> + updatedAt: number + revision: number + } + } + sha256: string + createdAt: number + }> + revision: number + contextEpoch: number + eventsSinceRefine: number + refinement: { + recommended: boolean + trigger?: "manual" | "failure" | "stagnation" | "milestone" | "periodic" + } + createdAt: number + updatedAt: number + } +} + +export type HarnessWorldStatusResponse = HarnessWorldStatusResponses[keyof HarnessWorldStatusResponses] + +export type HarnessWorldRefineData = { + body?: { + evaluatorToken: string + expectedRevision: number + reason: "manual" | "failure" | "stagnation" | "milestone" | "periodic" + patches: Array< + | { + op: "upsert" + key: string + kind: "hypothesis" | "observation" | "strategy" | "memory" | "skill" | "subagent" + content: string + confidence: number + evidenceRefs?: Array + } + | { + op: "remove" + key: string + } + > + } + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/world/refinements" +} + +export type HarnessWorldRefineErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessWorldRefineError = HarnessWorldRefineErrors[keyof HarnessWorldRefineErrors] + +export type HarnessWorldRefineResponses = { + /** + * Refined continual world-model state + */ + 200: { + schemaVersion: 1 + sessionID: string + runID: string + basePromptSHA256: string + entries: { + [key: string]: { + id: string + key: string + kind: "hypothesis" | "observation" | "strategy" | "memory" | "skill" | "subagent" + content: string + confidence: number + evidence: Array<{ + ref: string + authority: "self" | "tool" | "evaluator" | "human" + }> + updatedAt: number + revision: number + } + } + events: Array<{ + id: string + type: "analysis" | "tool" | "evaluation" | "failure" | "milestone" | "stagnation" | "manual" + summary: string + evidenceRefs: Array + changed: boolean + createdAt: number + }> + snapshots: Array<{ + revision: number + entries: { + [key: string]: { + id: string + key: string + kind: "hypothesis" | "observation" | "strategy" | "memory" | "skill" | "subagent" + content: string + confidence: number + evidence: Array<{ + ref: string + authority: "self" | "tool" | "evaluator" | "human" + }> + updatedAt: number + revision: number + } + } + sha256: string + createdAt: number + }> + revision: number + contextEpoch: number + eventsSinceRefine: number + refinement: { + recommended: boolean + trigger?: "manual" | "failure" | "stagnation" | "milestone" | "periodic" + } + createdAt: number + updatedAt: number + } +} + +export type HarnessWorldRefineResponse = HarnessWorldRefineResponses[keyof HarnessWorldRefineResponses] + +export type HarnessBindData = { + body?: { + schemaVersion: 1 + runID: string + sessionID: string + benchmark: string + title?: string + family?: "data" | "biology" | "physics" | "chemistry" | "ml" | "generalist" | "custom" + task?: string + version: string + taskID: string + split: "development" | "validation" | "held_out" | "release" + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + token: string + } + objective: string + profile?: "react" | "optimize" | "reproduce" | "theory" | "numerical" | "training" | "forecast" + search?: "adaptive" | "static" + orchestration?: { + topology: "auto" | "solo" | "centralized" | "fork_join" | "tournament" | "evolution" | "verifier_loop" + traits?: { + decomposability: number + sequentiality: number + toolIntensity: number + uncertainty: number + verificationRisk: number + novelty: number + crossDomain: number + } + maxWorkers: number + maxRounds: number + roles?: Array< + | "generation" + | "proximity" + | "reflection" + | "ranking" + | "evolution" + | "revision" + | "verification" + | "investigation" + | "simulation" + | "synthesis" + > + minIndependentVerifiers: number + adaptive?: { + protocolVersion: "marginal-utility-v1" + minRounds: number + patience: number + minUtilityGain: number + maxUncertainty: number + targetUtility?: number + } + repair?: { + protocolVersion: "verifier-routed-v1" + minConfidence: number + } + } + audit?: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + failureDiscovery?: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + integrity?: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + evolution?: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + metaHarness?: { + protocol: { + protocolVersion: "meta-harness-v1" + validatorSHA256: string + archiveSchemaSHA256: string + traceSchemaSHA256: string + baseline: { + artifactSHA256: string + manifestSHA256: string + } + mutable: Array<{ + root: string + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + }> + protected: { + manifestSHA256: string + roots: Array + } + archive: { + contents: "full-source-scores-traces" + query: "filesystem" + summariesOnly: false + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + updater: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judge: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + search: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + heldout: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + thresholds: { + minSearchGain: number + minHeldoutGain: number + maxModelRegression: number + minActivationRate: number + minRequiredAdherence: number + minFinalAdherence: number + maxPhaseDrift: number + minPredictionPrecision: number + maxRiskRegressions: number + maxContextTokens: number + maxMeanContextIncrease: number + } + promotionRequired: true + } + token: string + } + interventions?: { + protocolVersion: "intervention-study-v1" + validatorSHA256: string + requiredForPromotion: boolean + minPairs: number + maxPairs: number + maxTotalPairs: number + confidence: 0.95 + required: Array< + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + > + rules: Array< + | { + family: "replay" + mode: "max_absolute_effect" + threshold: number + } + | { + family: "retune" | "ablation" | "repair" + mode: "min_effect" + threshold: number + } + | { + family: "model_transfer" | "context_transfer" | "evaluator_transfer" | "split_transfer" + mode: "max_regression" + threshold: number + } + > + } + simulation?: { + kind: "ode" | "pde" | "cfd" | "materials" | "molecular" | "agentic" + engine: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256: string + reference: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validation: { + errorNorm: string + minLevels: number + maxLevels?: number + expectedOrder: number + orderTolerance: number + maxResidual: number + invariantTolerances: { + [key: string]: number + } + requiredStressTests: Array< + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + > + } + } + evaluatorAudit?: { + protocol: { + protocolVersion: "evaluator-audit-v1" + auditor: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + suite: { + name: string + version: string + commitmentSHA256: string + } + minCleanCases: number + minCasesPerFault: number + requiredFaults: Array< + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + > + minSensitivity: number + minSpecificity: number + minBalancedAccuracy: number + minFaultRecall: number + maxBrierScore: number + } + token: string + } + semanticAudit?: { + protocol: { + protocolVersion: "semantic-audit-v1" + reviewer: { + name: string + version: string + source: "gate" | "human" | "external" + } + scope: { + objectiveSHA256: string + criteria: Array<{ + id: string + requirement: string + }> + forbiddenShortcuts: Array<{ + id: string + description: string + }> + literature: { + cutoff: string + corpusSHA256: string + } + noveltyFloor: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + } + minReviewers: number + minConfidence: number + } + token: string + } + synthesis?: { + protocolVersion: "scientific-synthesis-v1" + querySHA256: string + referenceSHA256: string + referenceFactsSHA256: string + referenceFactCount: number + cutoff: string + tools: Array<"google_search" | "paper_search" | "web_browse"> + traceSchemaSHA256: string + filterPolicySHA256: string + maxToolEvents: number + decomposer: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judges: { + precision: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + recall: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + minGeneratedFacts: number + minPrecision: number + minRecall: number + minF1: number + cleanRoomRequired: true + judgeFailurePolicy: "inconclusive" + } + autonomy?: { + protocolVersion: "human-ai-autonomy-v1" + claimedLevel: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + recorder: { + name: string + version: string + artifactSHA256: string + source: "evaluator_runtime" + } + traceSchemaSHA256: string + classificationPolicySHA256: string + maxEvents: number + rawRetention: "required" + disclosure: "evaluator_retained" | "public_essential_after_release" + completeTraceRequired: true + uncertaintyPolicy: "inconclusive" + } + formalProof?: { + protocolVersion: "formal-proof-v1" + language: "lean4" + tier: "kernel" | "fresh_recheck" | "external_crosscheck" + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256: string + statementSHA256: string + declaration: string + module: string + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + verifiers: Array<{ + role: + | "lean_kernel" + | "source_auditor" + | "axiom_auditor" + | "fresh_rechecker" + | "sandbox_comparator" + | "external_checker" + name: string + version: string + artifactSHA256: string + }> + sandboxImageSHA256?: string + forbiddenConstructs: [ + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + ] + allowedAxioms: Array + maxFiles: number + completeManifestRequired: true + warningPolicy: "fail" + semanticPolicy: "formal_statement_only" + blueprint?: { + protocolVersion: "proof-blueprint-v1" + graphSchemaSHA256: string + compilerArtifactSHA256: string + sketchValidatorArtifactSHA256: string + reviewerArtifactSHA256: string + reviewerPromptSHA256: string + nodePolicy: "and-or-monotone-v1" + failurePolicy: "preserve-and-refine" + memoization: "goal-sha256" + finalAuthority: "formal-proof-v1" + directAttemptFirst: true + verifiedSketchRequired: true + completeFailureHistoryRequired: true + maxNodes: number + maxDepth: number + maxParallel: number + maxAttemptsPerGoal: number + maxRefinementsPerGoal: number + leaseDurationMs: number + } + } + replication?: { + protocolVersion: "replicated-evaluation-v1" + validatorSHA256: string + environmentSHA256: string + sampling: { + design: "crossed-stratified-cluster-v1" + stratumKind: string + clusterKind: string + strata: Array<{ + id: string + commitmentSHA256: string + }> + clusters: Array<{ + id: string + commitmentSHA256: string + }> + } + estimator: "mean" | "median" | "iqm" | "pass_rate" + interval: + | { + method: "stratified-bootstrap-percentile-v1" + confidence: 0.95 + resamples: number + seed: number + } + | { + method: "wilson-score-v1" + confidence: 0.95 + } + decision: { + rule: "conservative-bound-v1" + direction: "maximize" | "minimize" | "pass" + target: number + maxIntervalWidth?: number + } + failurePolicy: "fail-closed" + } + confirmation?: { + protocol: { + protocolVersion: "sealed-confirmation-v1" + optimization: { + split: "development" | "validation" + manifestSHA256: string + } + claim: { + taskID: string + split: "held_out" | "release" + manifestSHA256: string + validatorSHA256: string + environmentSHA256: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + source?: { + repository: string + revision: string + } + metric: string + direction: "maximize" | "minimize" + target: number + } + selection: { + rule: "terminal-verified-best-v1" + subjects: 1 + } + exposure: { + policy: "terminal-receipt-only" + searchFeedback: false + memoryCapture: false + } + failurePolicy: "fail-closed" + } + token: string + } + packs?: Array<"statistics" | "biology" | "physics" | "pde" | "chemistry" | "ml" | "forecast" | "formal"> + metric?: { + name?: string + direction: "maximize" | "minimize" | "pass" + target?: number + } + objectives?: Array<{ + metric: string + direction: "maximize" | "minimize" + }> + objectiveAudit?: { + schemaVersion: 1 + planSHA256: string + validatorSHA256: string + contractSHA256: string + guardIDs: Array + } + fidelities?: Array<{ + id: string + final: boolean + maxWallTimeMs?: number + maxCostUSD?: number + }> + model: { + provider: string + name: string + effort?: string + } + tools?: Array + skills?: Array<{ + name: string + version?: string + sha256?: string + }> + budget: { + wallTimeMs?: number + steps?: number + candidates?: number + tokens?: number + costUSD?: number + cpuHours?: number + gpuHours?: number + } + seed: number + intervention: "autonomous" | "human_reprompted" + contamination: { + policy: string + hiddenTestsAccessible: false + publicDataCutoff?: string + } + createdAt: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/runs" +} + +export type HarnessBindErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessBindError = HarnessBindErrors[keyof HarnessBindErrors] + +export type HarnessBindResponses = { + /** + * Bound harness contract + */ + 200: { + schemaVersion: 1 + runID: string + sessionID: string + objective: string + benchmark: { + name: string + title?: string + family?: "data" | "biology" | "physics" | "chemistry" | "ml" | "generalist" | "custom" + task?: string + version: string + taskID: string + split: "development" | "validation" | "held_out" | "release" + evaluator: string + evaluatorVersion?: string + evaluatorSource?: "benchmark" | "gate" | "human" | "external" + fidelities?: Array<{ + id: string + final: boolean + maxWallTimeMs?: number + maxCostUSD?: number + }> + metric?: string + direction?: "maximize" | "minimize" | "pass" + target?: number + objectives?: Array<{ + metric: string + direction: "maximize" | "minimize" + }> + objectiveAudit?: { + schemaVersion: 1 + planSHA256: string + validatorSHA256: string + contractSHA256: string + guardIDs: Array + } + } + profile: "react" | "optimize" | "reproduce" | "theory" | "numerical" | "training" | "forecast" + orchestration?: { + topology: "auto" | "solo" | "centralized" | "fork_join" | "tournament" | "evolution" | "verifier_loop" + traits?: { + decomposability: number + sequentiality: number + toolIntensity: number + uncertainty: number + verificationRisk: number + novelty: number + crossDomain: number + } + maxWorkers: number + maxRounds: number + roles?: Array< + | "generation" + | "proximity" + | "reflection" + | "ranking" + | "evolution" + | "revision" + | "verification" + | "investigation" + | "simulation" + | "synthesis" + > + minIndependentVerifiers: number + adaptive?: { + protocolVersion: "marginal-utility-v1" + minRounds: number + patience: number + minUtilityGain: number + maxUncertainty: number + targetUtility?: number + } + repair?: { + protocolVersion: "verifier-routed-v1" + minConfidence: number + } + } + search?: { + protocolVersion: "adaptive-search-v1" + signal: { + source: "verified-final-evaluations" + decay: 0.9 + epsilon: 1e-8 + } + local: { + minIntensity: 0.15 + maxIntensity: 0.5 + } + global: { + exploration: 1.4142135623730951 + minVisits: 2 + } + stagnation: { + patience: 5 + maxSignal: 0.02 + } + } + audit?: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + failureDiscovery?: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + integrity?: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + evolution?: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + metaHarness?: { + protocolVersion: "meta-harness-v1" + validatorSHA256: string + archiveSchemaSHA256: string + traceSchemaSHA256: string + baseline: { + artifactSHA256: string + manifestSHA256: string + } + mutable: Array<{ + root: string + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + }> + protected: { + manifestSHA256: string + roots: Array + } + archive: { + contents: "full-source-scores-traces" + query: "filesystem" + summariesOnly: false + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + updater: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judge: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + search: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + heldout: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + thresholds: { + minSearchGain: number + minHeldoutGain: number + maxModelRegression: number + minActivationRate: number + minRequiredAdherence: number + minFinalAdherence: number + maxPhaseDrift: number + minPredictionPrecision: number + maxRiskRegressions: number + maxContextTokens: number + maxMeanContextIncrease: number + } + promotionRequired: true + } + interventions?: { + protocolVersion: "intervention-study-v1" + validatorSHA256: string + requiredForPromotion: boolean + minPairs: number + maxPairs: number + maxTotalPairs: number + confidence: 0.95 + required: Array< + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + > + rules: Array< + | { + family: "replay" + mode: "max_absolute_effect" + threshold: number + } + | { + family: "retune" | "ablation" | "repair" + mode: "min_effect" + threshold: number + } + | { + family: "model_transfer" | "context_transfer" | "evaluator_transfer" | "split_transfer" + mode: "max_regression" + threshold: number + } + > + } + simulation?: { + kind: "ode" | "pde" | "cfd" | "materials" | "molecular" | "agentic" + engine: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256: string + reference: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validation: { + errorNorm: string + minLevels: number + maxLevels?: number + expectedOrder: number + orderTolerance: number + maxResidual: number + invariantTolerances: { + [key: string]: number + } + requiredStressTests: Array< + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + > + } + } + evaluatorAudit?: { + protocolVersion: "evaluator-audit-v1" + auditor: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + suite: { + name: string + version: string + commitmentSHA256: string + } + minCleanCases: number + minCasesPerFault: number + requiredFaults: Array< + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + > + minSensitivity: number + minSpecificity: number + minBalancedAccuracy: number + minFaultRecall: number + maxBrierScore: number + } + semanticAudit?: { + protocolVersion: "semantic-audit-v1" + reviewer: { + name: string + version: string + source: "gate" | "human" | "external" + } + scope: { + objectiveSHA256: string + criteria: Array<{ + id: string + requirement: string + }> + forbiddenShortcuts: Array<{ + id: string + description: string + }> + literature: { + cutoff: string + corpusSHA256: string + } + noveltyFloor: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + } + minReviewers: number + minConfidence: number + } + synthesis?: { + protocolVersion: "scientific-synthesis-v1" + querySHA256: string + referenceSHA256: string + referenceFactsSHA256: string + referenceFactCount: number + cutoff: string + tools: Array<"google_search" | "paper_search" | "web_browse"> + traceSchemaSHA256: string + filterPolicySHA256: string + maxToolEvents: number + decomposer: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judges: { + precision: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + recall: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + minGeneratedFacts: number + minPrecision: number + minRecall: number + minF1: number + cleanRoomRequired: true + judgeFailurePolicy: "inconclusive" + } + autonomy?: { + protocolVersion: "human-ai-autonomy-v1" + claimedLevel: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + recorder: { + name: string + version: string + artifactSHA256: string + source: "evaluator_runtime" + } + traceSchemaSHA256: string + classificationPolicySHA256: string + maxEvents: number + rawRetention: "required" + disclosure: "evaluator_retained" | "public_essential_after_release" + completeTraceRequired: true + uncertaintyPolicy: "inconclusive" + } + formalProof?: { + protocolVersion: "formal-proof-v1" + language: "lean4" + tier: "kernel" | "fresh_recheck" | "external_crosscheck" + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256: string + statementSHA256: string + declaration: string + module: string + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + verifiers: Array<{ + role: + | "lean_kernel" + | "source_auditor" + | "axiom_auditor" + | "fresh_rechecker" + | "sandbox_comparator" + | "external_checker" + name: string + version: string + artifactSHA256: string + }> + sandboxImageSHA256?: string + forbiddenConstructs: [ + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + ] + allowedAxioms: Array + maxFiles: number + completeManifestRequired: true + warningPolicy: "fail" + semanticPolicy: "formal_statement_only" + blueprint?: { + protocolVersion: "proof-blueprint-v1" + graphSchemaSHA256: string + compilerArtifactSHA256: string + sketchValidatorArtifactSHA256: string + reviewerArtifactSHA256: string + reviewerPromptSHA256: string + nodePolicy: "and-or-monotone-v1" + failurePolicy: "preserve-and-refine" + memoization: "goal-sha256" + finalAuthority: "formal-proof-v1" + directAttemptFirst: true + verifiedSketchRequired: true + completeFailureHistoryRequired: true + maxNodes: number + maxDepth: number + maxParallel: number + maxAttemptsPerGoal: number + maxRefinementsPerGoal: number + leaseDurationMs: number + } + } + replication?: { + protocolVersion: "replicated-evaluation-v1" + validatorSHA256: string + environmentSHA256: string + sampling: { + design: "crossed-stratified-cluster-v1" + stratumKind: string + clusterKind: string + strata: Array<{ + id: string + commitmentSHA256: string + }> + clusters: Array<{ + id: string + commitmentSHA256: string + }> + } + estimator: "mean" | "median" | "iqm" | "pass_rate" + interval: + | { + method: "stratified-bootstrap-percentile-v1" + confidence: 0.95 + resamples: number + seed: number + } + | { + method: "wilson-score-v1" + confidence: 0.95 + } + decision: { + rule: "conservative-bound-v1" + direction: "maximize" | "minimize" | "pass" + target: number + maxIntervalWidth?: number + } + failurePolicy: "fail-closed" + } + confirmation?: { + protocolVersion: "sealed-confirmation-v1" + optimization: { + split: "development" | "validation" + manifestSHA256: string + } + claim: { + taskID: string + split: "held_out" | "release" + manifestSHA256: string + validatorSHA256: string + environmentSHA256: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + source?: { + repository: string + revision: string + } + metric: string + direction: "maximize" | "minimize" + target: number + } + selection: { + rule: "terminal-verified-best-v1" + subjects: 1 + } + exposure: { + policy: "terminal-receipt-only" + searchFeedback: false + memoryCapture: false + } + failurePolicy: "fail-closed" + } + packs?: Array<"statistics" | "biology" | "physics" | "pde" | "chemistry" | "ml" | "forecast" | "formal"> + model: { + provider: string + name: string + effort?: string + } + tools?: Array + skills?: Array<{ + name: string + version?: string + sha256?: string + }> + budget: { + wallTimeMs?: number + steps?: number + candidates?: number + tokens?: number + costUSD?: number + cpuHours?: number + gpuHours?: number + } + seed: number + intervention: "autonomous" | "human_reprompted" + contamination: { + policy: string + hiddenTestsAccessible: false + publicDataCutoff?: string + } + createdAt: number + } +} + +export type HarnessBindResponse = HarnessBindResponses[keyof HarnessBindResponses] + +export type HarnessEvaluateData = { + body?: { + schemaVersion: 1 + runID: string + sessionID: string + evaluatorToken: string + candidateID?: string + stage?: string + simulationReceiptID?: string + integrityReceiptID?: string + evolutionReceiptID?: string + interventionReceiptID?: string + evaluatorAuditReceiptID?: string + semanticReceiptID?: string + replicationReceiptID?: string + auditReceiptID?: string + failureDiscoveryReceiptID?: string + synthesisReceiptID?: string + autonomyReceiptID?: string + proofReceiptID?: string + status: "passed" | "failed" | "inconclusive" + score?: number + metrics?: { + [key: string]: number + } + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + score?: number + evidence?: Array + note?: string + }> + evidence: Array + usage?: { + wallTimeMs?: number + costUSD?: number + } + evaluatedAt: number + notes?: string + } + path?: never + query?: { + directory?: string + } + url: "/harness/evaluations" +} + +export type HarnessEvaluateErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessEvaluateError = HarnessEvaluateErrors[keyof HarnessEvaluateErrors] + +export type HarnessEvaluateResponses = { + /** + * Recorded external evaluation + */ + 200: unknown +} + +export type HarnessCompareData = { + body?: { + sessionIDs: Array + baselineRunID: string + } + path?: never + query?: { + directory?: string + } + url: "/harness/compare" +} + +export type HarnessCompareErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessCompareError = HarnessCompareErrors[keyof HarnessCompareErrors] + +export type HarnessCompareResponses = { + /** + * Comparable run deltas + */ + 200: unknown +} + +export type HarnessSkillsData = { + body?: never + path?: never + query?: { + directory?: string + } + url: "/harness/skills" +} + +export type HarnessSkillsResponses = { + /** + * Learned skill qualification manifests + */ + 200: Array<{ + schemaVersion: 1 + name: string + description: string + contentSHA256: string + origin: "conversation" | "rsi" + source: { + sessionID?: string + runID?: string + } + status: "pending" | "qualified" | "promoted" | "rejected" + evidence: Array<{ + id: string + proposalSHA256: string + benchmark: { + name: string + version: string + taskID: string + split: "held_out" | "release" + metric?: string + direction: "maximize" | "minimize" | "pass" + } + candidate: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + } + control: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + } + nonregressing: boolean + improved: boolean + trigger: { + datasetSHA256: string + split: "held_out" + examples: number + truePositive: number + falsePositive: number + trueNegative: number + falseNegative: number + precision: number + recall: number + } + evaluator: { + name: string + version: string + } + recordedAt: number + }> + criteria: { + tasks: 3 + improvements: 2 + triggerPrecision: 0.8 + triggerRecall: 0.8 + } + createdAt: number + updatedAt: number + promotedAt?: number + }> +} + +export type HarnessSkillsResponse = HarnessSkillsResponses[keyof HarnessSkillsResponses] + +export type HarnessSkillProposeData = { + body?: { + name: string + description: string + content: string + origin: "conversation" | "rsi" + sessionID?: string + runID?: string + createdAt?: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/skills" +} + +export type HarnessSkillProposeErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessSkillProposeError = HarnessSkillProposeErrors[keyof HarnessSkillProposeErrors] + +export type HarnessSkillProposeResponses = { + /** + * Quarantined proposal + */ + 200: { + schemaVersion: 1 + name: string + description: string + contentSHA256: string + origin: "conversation" | "rsi" + source: { + sessionID?: string + runID?: string + } + status: "pending" | "qualified" | "promoted" | "rejected" + evidence: Array<{ + id: string + proposalSHA256: string + benchmark: { + name: string + version: string + taskID: string + split: "held_out" | "release" + metric?: string + direction: "maximize" | "minimize" | "pass" + } + candidate: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + } + control: { + sessionID: string + runID: string + status: "passed" | "failed" | "inconclusive" + score?: number + evaluationSHA256: string + } + nonregressing: boolean + improved: boolean + trigger: { + datasetSHA256: string + split: "held_out" + examples: number + truePositive: number + falsePositive: number + trueNegative: number + falseNegative: number + precision: number + recall: number + } + evaluator: { + name: string + version: string + } + recordedAt: number + }> + criteria: { + tasks: 3 + improvements: 2 + triggerPrecision: 0.8 + triggerRecall: 0.8 + } + createdAt: number + updatedAt: number + promotedAt?: number + } | null +} + +export type HarnessSkillProposeResponse = HarnessSkillProposeResponses[keyof HarnessSkillProposeResponses] + +export type HarnessSkillAttestData = { + body?: { + name: string + candidate: { + sessionID: string + evaluatorToken: string + } + control: { + sessionID: string + evaluatorToken: string + } + trigger: { + datasetSHA256: string + split: "held_out" + examples: number + truePositive: number + falsePositive: number + trueNegative: number + falseNegative: number + } + recordedAt?: number + } + path?: never + query?: { + directory?: string + } + url: "/harness/skills/evidence" +} + +export type HarnessSkillAttestErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessSkillAttestError = HarnessSkillAttestErrors[keyof HarnessSkillAttestErrors] + +export type HarnessSkillAttestResponses = { + /** + * Updated qualification state + */ + 200: unknown +} + +export type HarnessSkillPromoteData = { + body?: never + path: { + name: string + } + query?: { + directory?: string + } + url: "/harness/skills/{name}/promotion" +} + +export type HarnessSkillPromoteErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessSkillPromoteError = HarnessSkillPromoteErrors[keyof HarnessSkillPromoteErrors] + +export type HarnessSkillPromoteResponses = { + /** + * Promoted skill + */ + 200: unknown +} + +export type HarnessContractData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/contract" +} + +export type HarnessContractErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessContractError = HarnessContractErrors[keyof HarnessContractErrors] + +export type HarnessContractResponses = { + /** + * Harness contract + */ + 200: { + schemaVersion: 1 + runID: string + sessionID: string + objective: string + benchmark: { + name: string + title?: string + family?: "data" | "biology" | "physics" | "chemistry" | "ml" | "generalist" | "custom" + task?: string + version: string + taskID: string + split: "development" | "validation" | "held_out" | "release" + evaluator: string + evaluatorVersion?: string + evaluatorSource?: "benchmark" | "gate" | "human" | "external" + fidelities?: Array<{ + id: string + final: boolean + maxWallTimeMs?: number + maxCostUSD?: number + }> + metric?: string + direction?: "maximize" | "minimize" | "pass" + target?: number + objectives?: Array<{ + metric: string + direction: "maximize" | "minimize" + }> + objectiveAudit?: { + schemaVersion: 1 + planSHA256: string + validatorSHA256: string + contractSHA256: string + guardIDs: Array + } + } + profile: "react" | "optimize" | "reproduce" | "theory" | "numerical" | "training" | "forecast" + orchestration?: { + topology: "auto" | "solo" | "centralized" | "fork_join" | "tournament" | "evolution" | "verifier_loop" + traits?: { + decomposability: number + sequentiality: number + toolIntensity: number + uncertainty: number + verificationRisk: number + novelty: number + crossDomain: number + } + maxWorkers: number + maxRounds: number + roles?: Array< + | "generation" + | "proximity" + | "reflection" + | "ranking" + | "evolution" + | "revision" + | "verification" + | "investigation" + | "simulation" + | "synthesis" + > + minIndependentVerifiers: number + adaptive?: { + protocolVersion: "marginal-utility-v1" + minRounds: number + patience: number + minUtilityGain: number + maxUncertainty: number + targetUtility?: number + } + repair?: { + protocolVersion: "verifier-routed-v1" + minConfidence: number + } + } + search?: { + protocolVersion: "adaptive-search-v1" + signal: { + source: "verified-final-evaluations" + decay: 0.9 + epsilon: 1e-8 + } + local: { + minIntensity: 0.15 + maxIntensity: 0.5 + } + global: { + exploration: 1.4142135623730951 + minVisits: 2 + } + stagnation: { + patience: 5 + maxSignal: 0.02 + } + } + audit?: { + mode: "performance" | "failure" | "hybrid" + budget: number + minSamples: number + noiseVariance?: number + lengthscale?: number + beta?: number + failureThreshold?: number + tolerance?: number + maxUncertainty?: number + estimationWeight?: number + diversityWeight?: number + coverageWeight?: number + targetFailures?: number + transfer?: { + protocolVersion: "score-history-prior-v1" + poolSHA256: string + sourceManifestSHA256: string + selectionSHA256: string + selectionMethod: "pca-gmm-profile-v1" | "holdout-embedding-gmm-v1" + sourceModels: Array + calibrationSamples: number + maxCalibrationMAE: number + } + promotionRequired?: boolean + } + failureDiscovery?: { + protocolVersion: "topic-aware-failure-v1" + sourcePoolSHA256: string + topicModel: { + kind: "predefined" | "bertopic" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + topics: Array<{ + id: string + commitment: string + }> + generator: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + validators: [ + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + { + kind: "correctness" | "topic" | "novelty" + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + }, + ] + embedding: { + identity: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + dimensions: number + regularization?: number + } + budget: number + anchorsPerAttempt: number + exploration?: number + failureThreshold: number + targetFailures?: number + } + integrity?: { + protocolVersion: "benchmark-integrity-v1" + validatorSHA256: string + traceSchemaSHA256: string + minEvents: number + minCoverage: number + assignedModel: { + name: string + baseArtifactSHA256: string + configSHA256: string + } + forbiddenModelArtifacts?: Array + policy: { + testItemDerivation: "forbidden" + unapprovedExternalModels: "forbidden" + benchmarkLookup: "forbidden" + } + auditors: [ + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + { + kind: "test_item_contamination" | "external_model_use" | "benchmark_lookup" + name: string + version: string + promptSHA256: string + }, + ] + hiddenCanaryManifestSHA256: string + minHiddenCanaries: number + } + evolution?: { + protocolVersion: "evolution-trace-v1" + validatorSHA256: string + manifestSchemaSHA256: string + lineAlgorithm: "sha256-exact-line-v1" + roots: Array + extensions: Array + exclude?: Array + maxFiles: number + maxFileBytes: number + maxTotalBytes: number + maxSourceLines: number + maxChangedLines: number + } + metaHarness?: { + protocolVersion: "meta-harness-v1" + validatorSHA256: string + archiveSchemaSHA256: string + traceSchemaSHA256: string + baseline: { + artifactSHA256: string + manifestSHA256: string + } + mutable: Array<{ + root: string + component: "prompt" | "memory" | "skill" | "tool" | "middleware" | "subagent" | "scaffold" + }> + protected: { + manifestSHA256: string + roots: Array + } + archive: { + contents: "full-source-scores-traces" + query: "filesystem" + summariesOnly: false + hiddenContent: "excluded" + evaluatorContent: "excluded" + } + updater: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judge: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + search: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + heldout: { + models: Array<{ + id: string + commitment: string + }> + tasks: Array<{ + id: string + commitment: string + activationRequired: boolean + }> + } + thresholds: { + minSearchGain: number + minHeldoutGain: number + maxModelRegression: number + minActivationRate: number + minRequiredAdherence: number + minFinalAdherence: number + maxPhaseDrift: number + minPredictionPrecision: number + maxRiskRegressions: number + maxContextTokens: number + maxMeanContextIncrease: number + } + promotionRequired: true + } + interventions?: { + protocolVersion: "intervention-study-v1" + validatorSHA256: string + requiredForPromotion: boolean + minPairs: number + maxPairs: number + maxTotalPairs: number + confidence: 0.95 + required: Array< + | "replay" + | "retune" + | "ablation" + | "repair" + | "model_transfer" + | "context_transfer" + | "evaluator_transfer" + | "split_transfer" + > + rules: Array< + | { + family: "replay" + mode: "max_absolute_effect" + threshold: number + } + | { + family: "retune" | "ablation" | "repair" + mode: "min_effect" + threshold: number + } + | { + family: "model_transfer" | "context_transfer" | "evaluator_transfer" | "split_transfer" + mode: "max_regression" + threshold: number + } + > + } + simulation?: { + kind: "ode" | "pde" | "cfd" | "materials" | "molecular" | "agentic" + engine: { + name: string + version: string + commandSHA256: string + configSHA256: string + } + problemSHA256: string + reference: { + kind: "analytic" | "manufactured" | "benchmark" | "independent_solver" | "limiting_case" + identity: string + sha256: string + } + validation: { + errorNorm: string + minLevels: number + maxLevels?: number + expectedOrder: number + orderTolerance: number + maxResidual: number + invariantTolerances: { + [key: string]: number + } + requiredStressTests: Array< + | "timestep_sensitivity" + | "solver_tolerance_sensitivity" + | "reference_replay" + | "independent_implementation" + | "unit_convention" + | "boundary_sensitivity" + | "perturbation_stability" + > + } + } + evaluatorAudit?: { + protocolVersion: "evaluator-audit-v1" + auditor: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + suite: { + name: string + version: string + commitmentSHA256: string + } + minCleanCases: number + minCasesPerFault: number + requiredFaults: Array< + | "wrong_answer" + | "unsupported_claim" + | "missing_evidence" + | "data_leakage" + | "non_reproducible" + | "reward_hacking" + | "invalid_statistics" + | "invalid_simulation" + | "distribution_shift" + | "evaluation_awareness" + > + minSensitivity: number + minSpecificity: number + minBalancedAccuracy: number + minFaultRecall: number + maxBrierScore: number + } + semanticAudit?: { + protocolVersion: "semantic-audit-v1" + reviewer: { + name: string + version: string + source: "gate" | "human" | "external" + } + scope: { + objectiveSHA256: string + criteria: Array<{ + id: string + requirement: string + }> + forbiddenShortcuts: Array<{ + id: string + description: string + }> + literature: { + cutoff: string + corpusSHA256: string + } + noveltyFloor: "not_required" | "known" | "rediscovery" | "minor" | "publication" | "major" + } + minReviewers: number + minConfidence: number + } + synthesis?: { + protocolVersion: "scientific-synthesis-v1" + querySHA256: string + referenceSHA256: string + referenceFactsSHA256: string + referenceFactCount: number + cutoff: string + tools: Array<"google_search" | "paper_search" | "web_browse"> + traceSchemaSHA256: string + filterPolicySHA256: string + maxToolEvents: number + decomposer: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + judges: { + precision: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + recall: { + name: string + version: string + promptSHA256: string + configSHA256: string + } + } + minGeneratedFacts: number + minPrecision: number + minRecall: number + minF1: number + cleanRoomRequired: true + judgeFailurePolicy: "inconclusive" + } + autonomy?: { + protocolVersion: "human-ai-autonomy-v1" + claimedLevel: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + recorder: { + name: string + version: string + artifactSHA256: string + source: "evaluator_runtime" + } + traceSchemaSHA256: string + classificationPolicySHA256: string + maxEvents: number + rawRetention: "required" + disclosure: "evaluator_retained" | "public_essential_after_release" + completeTraceRequired: true + uncertaintyPolicy: "inconclusive" + } + formalProof?: { + protocolVersion: "formal-proof-v1" + language: "lean4" + tier: "kernel" | "fresh_recheck" | "external_crosscheck" + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + challengeSHA256: string + statementSHA256: string + declaration: string + module: string + leanVersion: string + leanToolchainSHA256: string + lakeManifestSHA256: string + dependencyTreeSHA256: string + verifiers: Array<{ + role: + | "lean_kernel" + | "source_auditor" + | "axiom_auditor" + | "fresh_rechecker" + | "sandbox_comparator" + | "external_checker" + name: string + version: string + artifactSHA256: string + }> + sandboxImageSHA256?: string + forbiddenConstructs: [ + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + "sorry" | "admit" | "debug.skipKernelTC" | "native_decide", + ] + allowedAxioms: Array + maxFiles: number + completeManifestRequired: true + warningPolicy: "fail" + semanticPolicy: "formal_statement_only" + blueprint?: { + protocolVersion: "proof-blueprint-v1" + graphSchemaSHA256: string + compilerArtifactSHA256: string + sketchValidatorArtifactSHA256: string + reviewerArtifactSHA256: string + reviewerPromptSHA256: string + nodePolicy: "and-or-monotone-v1" + failurePolicy: "preserve-and-refine" + memoization: "goal-sha256" + finalAuthority: "formal-proof-v1" + directAttemptFirst: true + verifiedSketchRequired: true + completeFailureHistoryRequired: true + maxNodes: number + maxDepth: number + maxParallel: number + maxAttemptsPerGoal: number + maxRefinementsPerGoal: number + leaseDurationMs: number + } + } + replication?: { + protocolVersion: "replicated-evaluation-v1" + validatorSHA256: string + environmentSHA256: string + sampling: { + design: "crossed-stratified-cluster-v1" + stratumKind: string + clusterKind: string + strata: Array<{ + id: string + commitmentSHA256: string + }> + clusters: Array<{ + id: string + commitmentSHA256: string + }> + } + estimator: "mean" | "median" | "iqm" | "pass_rate" + interval: + | { + method: "stratified-bootstrap-percentile-v1" + confidence: 0.95 + resamples: number + seed: number + } + | { + method: "wilson-score-v1" + confidence: 0.95 + } + decision: { + rule: "conservative-bound-v1" + direction: "maximize" | "minimize" | "pass" + target: number + maxIntervalWidth?: number + } + failurePolicy: "fail-closed" + } + confirmation?: { + protocolVersion: "sealed-confirmation-v1" + optimization: { + split: "development" | "validation" + manifestSHA256: string + } + claim: { + taskID: string + split: "held_out" | "release" + manifestSHA256: string + validatorSHA256: string + environmentSHA256: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "external" + } + source?: { + repository: string + revision: string + } + metric: string + direction: "maximize" | "minimize" + target: number + } + selection: { + rule: "terminal-verified-best-v1" + subjects: 1 + } + exposure: { + policy: "terminal-receipt-only" + searchFeedback: false + memoryCapture: false + } + failurePolicy: "fail-closed" + } + packs?: Array<"statistics" | "biology" | "physics" | "pde" | "chemistry" | "ml" | "forecast" | "formal"> + model: { + provider: string + name: string + effort?: string + } + tools?: Array + skills?: Array<{ + name: string + version?: string + sha256?: string + }> + budget: { + wallTimeMs?: number + steps?: number + candidates?: number + tokens?: number + costUSD?: number + cpuHours?: number + gpuHours?: number + } + seed: number + intervention: "autonomous" | "human_reprompted" + contamination: { + policy: string + hiddenTestsAccessible: false + publicDataCutoff?: string + } + createdAt: number + } | null +} + +export type HarnessContractResponse = HarnessContractResponses[keyof HarnessContractResponses] + +export type HarnessEvaluationsData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/evaluations" +} + +export type HarnessEvaluationsErrors = { + /** + * Bad request + */ + 400: BadRequestError +} + +export type HarnessEvaluationsError = HarnessEvaluationsErrors[keyof HarnessEvaluationsErrors] + +export type HarnessEvaluationsResponses = { + /** + * Evaluation journal + */ + 200: Array<{ + schemaVersion: 1 + runID: string + sessionID: string + subject?: { + type: "run" | "candidate" + id: string + } + fidelity?: { + stage: string + final: boolean + } + simulationReceiptID?: string + integrityReceiptID?: string + evolutionReceiptID?: string + interventionReceiptID?: string + evaluatorAuditReceiptID?: string + semanticReceiptID?: string + replicationReceiptID?: string + auditReceiptID?: string + failureDiscoveryReceiptID?: string + synthesisReceiptID?: string + autonomyReceiptID?: string + proofReceiptID?: string + evaluator: { + name: string + version: string + source: "benchmark" | "gate" | "human" | "external" + } + status: "passed" | "failed" | "inconclusive" + score?: number + metrics?: { + [key: string]: number + } + checks: Array<{ + id: string + status: "passed" | "failed" | "inconclusive" + blocking: boolean + score?: number + evidence?: Array + note?: string + }> + evidence: Array + usage?: { + wallTimeMs?: number + costUSD?: number + } + evaluatedAt: number + recordedAt?: number + notes?: string + }> +} + +export type HarnessEvaluationsResponse = HarnessEvaluationsResponses[keyof HarnessEvaluationsResponses] + +export type HarnessReportData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + } + url: "/harness/runs/{sessionID}/report" +} + +export type HarnessReportErrors = { + /** + * Bad request + */ + 400: BadRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type HarnessReportError = HarnessReportErrors[keyof HarnessReportErrors] + +export type HarnessReportResponses = { + /** + * Quality-cost report + */ + 200: { + schemaVersion: 1 + runID: string + sessionID: string + contractFingerprint: string + comparisonKey: string + benchmark: { + id: string + title: string + family: "data" | "biology" | "physics" | "chemistry" | "ml" | "generalist" | "custom" + version: string + taskID: string + split: "development" | "validation" | "held_out" | "release" + } + execution: { + profile: "react" | "optimize" | "reproduce" | "theory" | "numerical" | "training" | "forecast" + packs: Array + provider: string + model: string + effort?: string + intervention: "autonomous" | "human_reprompted" + autonomy?: { + claimedLevel: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + derivedLevel?: "essentially_autonomous" | "human_ai_collaboration" | "primarily_human" + status?: "passed" | "failed" | "inconclusive" + } + formal?: { + tier: "kernel" | "fresh_recheck" | "external_crosscheck" + relation: "exact_proof" | "exact_refutation" | "repaired_proof" + status?: "passed" | "failed" + blueprint?: { + blueprintID: string + status: "open" | "proved" | "refuted" | "exhausted" + goals: number + proved: number + refuted: number + exhausted: number + decompositions: number + attempts: number + rejected: number + refinements: number + openLeases: number + revision: number + } + } + seed: number + } + quality: { + source: "optimization" | "sealed_confirmation" + provisional: boolean + status?: "passed" | "failed" | "inconclusive" + metric?: string + direction: "maximize" | "minimize" | "pass" + score?: number + target?: number + targetReached: boolean + evaluator: string + evaluatorVersion?: string + simulationReceiptID?: string + integrityReceiptID?: string + evolutionReceiptID?: string + interventionReceiptID?: string + evaluatorAuditReceiptID?: string + semanticReceiptID?: string + replicationReceiptID?: string + auditReceiptID?: string + failureDiscoveryReceiptID?: string + synthesisReceiptID?: string + autonomyReceiptID?: string + proofReceiptID?: string + metaReceiptID?: string + confirmationReceiptID?: string + evaluations: number + } + efficiency: { + costUSD?: number + evaluatorCostUSD?: number + tokens?: { + input: number + output: number + reasoning: number + cacheRead: number + cacheWrite: number + total: number + } + wallTimeMs?: number + evaluatorWallTimeMs?: number + toolCalls?: number + searches?: number + dedupeHits?: number + retries?: number + failures?: number + candidates?: number + } + search?: { + status: "active" | "completed" + stopReason?: "budget_exhausted" | "objective_met" | "no_improvement" | "user_cancelled" | "runtime_error" + bestID?: string + candidates: number + verified: number + generations: number + stalled: number + proposalPolicy: "advisory-v2" | "leased-v3" | "adaptive-v4" + controller?: { + protocolVersion: "adaptive-search-v1" + signal: { + source: "verified-final-evaluations" + decay: 0.9 + epsilon: 1e-8 + } + local: { + minIntensity: 0.15 + maxIntensity: 0.5 + } + global: { + exploration: 1.4142135623730951 + minVisits: 2 + } + stagnation: { + patience: 5 + maxSignal: 0.02 + } + } + adaptation?: { + protocolVersion: "adaptive-search-v1" + policySHA256: string + events: number + stalled: number + selectedIsland?: number + globalStagnation: boolean + islands: Array<{ + island: number + visits: number + decayedVisits: number + improvements: number + accumulatedImprovement: number + decayedReward: number + rewardMean: number + intensity: number + ucb: number + bestID?: string + bestFitness?: number + }> + } + objectives: Array<{ + metric: string + direction: "maximize" | "minimize" + }> + objectiveAudit?: { + schemaVersion: 1 + planSHA256: string + validatorSHA256: string + contractSHA256: string + guardIDs: Array + } + archive: number + } + metaHarness?: { + status: "passed" | "failed" | "inconclusive" + selectionID: string + diagnostics: { + updaterGain?: number + beneficiaryGain?: number + worstHeldoutModelGain?: number + activationRate: number + requiredAdherence?: number + finalAdherence?: number + maxPhaseDrift?: number + predictionPrecision: number + riskRegressions: number + maxContextTokens: number + meanContextIncrease: number + loadedBenefit?: number + searchPairs: number + heldoutPairs: number + } + failures: Array + } + generatedAt: number + } +} + +export type HarnessReportResponse = HarnessReportResponses[keyof HarnessReportResponses] + export type SearchQueryData = { body?: never path?: never diff --git a/tooling/sdk/openapi.json b/tooling/sdk/openapi.json index e096dc23..c1568470 100644 --- a/tooling/sdk/openapi.json +++ b/tooling/sdk/openapi.json @@ -893,6 +893,13 @@ "description": { "type": "string" }, + "category": { + "type": "string", + "enum": [ + "compute", + "integration" + ] + }, "custom": { "type": "boolean" }, @@ -954,6 +961,7 @@ "id", "label", "description", + "category", "custom", "fields", "connected", @@ -1006,6 +1014,13 @@ "description": { "type": "string" }, + "category": { + "type": "string", + "enum": [ + "compute", + "integration" + ] + }, "custom": { "type": "boolean" }, @@ -1067,6 +1082,7 @@ "id", "label", "description", + "category", "custom", "fields", "connected", @@ -1155,6 +1171,13 @@ "description": { "type": "string" }, + "category": { + "type": "string", + "enum": [ + "compute", + "integration" + ] + }, "custom": { "type": "boolean" }, @@ -1216,6 +1239,7 @@ "id", "label", "description", + "category", "custom", "fields", "connected", @@ -16552,6 +16576,59 @@ ] } }, + "profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "messageID": { + "type": "string" + }, + "id": { + "type": "string", + "enum": [ + "react", + "optimize", + "reproduce", + "theory", + "numerical", + "training", + "forecast" + ] + }, + "source": { + "type": "string", + "enum": [ + "contract", + "heuristic", + "control" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "selectedAt": { + "type": "number" + } + }, + "required": [ + "messageID", + "id", + "source", + "confidence", + "reasons", + "selectedAt" + ] + } + }, "privacy": { "type": "object", "properties": { @@ -16597,6 +16674,7 @@ "reviewerFindings", "failures", "retries", + "profiles", "privacy" ] } @@ -18919,9 +18997,9 @@ ] } }, - "/search": { - "get": { - "operationId": "search.query", + "/harness/audits": { + "post": { + "operationId": "harness.audit.initialize", "parameters": [ { "in": "query", @@ -18929,142 +19007,568 @@ "schema": { "type": "string" } - }, - { - "in": "query", - "name": "q", - "schema": { - "type": "string", - "minLength": 2, - "maxLength": 200 - }, - "required": true } ], - "summary": "Search sessions, messages, and artifacts", - "description": "Case-insensitive plain-text search across session titles, recent conversation text, and artifact files in the project.", + "summary": "Initialize an evaluator-owned active audit", + "description": "Commits an opaque probe pool and binds uncertainty-aware selection to the evaluator capability and audited artifact.", "responses": { "200": { - "description": "Grouped plain-text matches", + "description": "Active audit state", "content": { "application/json": { "schema": { "type": "object", "properties": { - "sessions": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "title": { - "type": "string" - } + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "enum": [ + "active-audit-v1", + "proactive-audit-v2" + ] + }, + "auditID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "poolFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] }, - "required": [ - "id", - "title" - ] - } + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false }, - "messages": { - "type": "array", - "items": { + "config": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "performance", + "failure", + "hybrid" + ] + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minSamples": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "noiseVariance": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 2 + }, + "lengthscale": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 100 + }, + "beta": { + "default": 1.96, + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "failureThreshold": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tolerance": { + "default": 0.02, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "estimationWeight": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "diversityWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "coverageWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + }, + "transfer": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "score-history-prior-v1" + }, + "poolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionMethod": { + "type": "string", + "enum": [ + "pca-gmm-profile-v1", + "holdout-embedding-gmm-v1" + ] + }, + "sourceModels": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "calibrationSamples": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "maxCalibrationMAE": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "poolSHA256", + "sourceManifestSHA256", + "selectionSHA256", + "selectionMethod", + "sourceModels", + "calibrationSamples", + "maxCalibrationMAE" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean" + } + }, + "required": [ + "mode", + "budget", + "minSamples" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "precision_reached", + "failure_target_reached", + "pool_exhausted" + ] + }, + "pool": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { "type": "object", "properties": { - "sessionID": { - "type": "string" + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 }, - "messageID": { - "type": "string" + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" }, - "role": { - "type": "string" + "features": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "number" + } }, - "snippet": { - "type": "string" + "stratum": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "weight": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1000 + }, + "priorLoss": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sourceLosses": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "selection": { + "type": "object", + "properties": { + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "phase": { + "type": "string", + "enum": [ + "calibration", + "adaptive", + "fallback" + ] + }, + "acquisition": { + "type": "object", + "properties": { + "posteriorLoss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "posteriorStd": { + "type": "number", + "minimum": 0 + }, + "failureUCB": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "varianceReduction": { + "type": "number", + "minimum": 0 + }, + "diversity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "coverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "score": { + "type": "number" + } + }, + "required": [ + "posteriorLoss", + "posteriorStd", + "failureUCB", + "varianceReduction", + "diversity", + "coverage", + "score" + ], + "additionalProperties": false + } + }, + "required": [ + "round", + "selectedAt", + "acquisition" + ], + "additionalProperties": false + }, + "observation": { + "type": "object", + "properties": { + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "loss", + "failure", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false } }, "required": [ - "sessionID", - "messageID", - "role", - "snippet" - ] + "id", + "commitment", + "features", + "stratum" + ], + "additionalProperties": false } }, - "artifacts": { + "order": { "type": "array", "items": { - "type": "object", - "properties": { - "path": { - "type": "string" + "type": "string", + "minLength": 1 + } + }, + "estimate": { + "type": "object", + "properties": { + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanLoss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "lower95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "upper95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "abstain": { + "type": "boolean" + }, + "effectivePoolSize": { + "type": "number", + "exclusiveMinimum": 0 + }, + "stratumCoverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "transfer": { + "default": { + "status": "not_configured", + "observed": 0, + "required": 0 }, - "name": { - "type": "string" + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "not_configured", + "calibrating", + "accepted", + "rejected" + ] + }, + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "required": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanAbsoluteError": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "threshold": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } }, - "kind": { - "type": "string" - } - }, - "required": [ - "path", - "name", - "kind" - ] - } + "required": [ + "status", + "observed", + "required" + ], + "additionalProperties": false + } + }, + "required": [ + "observed", + "failures", + "meanLoss", + "standardDeviation", + "lower95", + "upper95", + "abstain", + "effectivePoolSize", + "stratumCoverage" + ], + "additionalProperties": false + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 } }, "required": [ - "sessions", - "messages", - "artifacts" - ] - } - } - } - } - }, - "x-codeSamples": [ - { - "lang": "js", - "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.search.query({\n ...\n})" - } - ] - } - }, - "/permission/{requestID}/reply": { - "post": { - "operationId": "permission.reply", - "parameters": [ - { - "in": "query", - "name": "directory", - "schema": { - "type": "string" - } - }, - { - "in": "path", - "name": "requestID", - "schema": { - "type": "string" - }, - "required": true - } - ], - "summary": "Respond to permission request", - "description": "Approve or deny a permission request from the AI assistant.", - "responses": { - "200": { - "description": "Permission processed successfully", - "content": { - "application/json": { - "schema": { - "type": "boolean" + "schemaVersion", + "protocolVersion", + "auditID", + "runID", + "sessionID", + "evaluator", + "contractFingerprint", + "poolFingerprint", + "subject", + "config", + "status", + "pool", + "order", + "estimate", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false } } } @@ -19078,16 +19582,6 @@ } } } - }, - "404": { - "description": "Not found", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/NotFoundError" - } - } - } } }, "requestBody": { @@ -19096,42 +19590,45116 @@ "schema": { "type": "object", "properties": { - "reply": { + "sessionID": { "type": "string", - "enum": [ - "once", - "session", - "project", - "always", - "reject" - ] + "minLength": 1, + "maxLength": 240 }, - "message": { - "type": "string" - } - }, - "required": [ - "reply" - ] - } - } - } - }, - "x-codeSamples": [ - { - "lang": "js", - "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.permission.reply({\n ...\n})" - } - ] - } - }, - "/permission": { - "get": { - "operationId": "permission.list", - "parameters": [ - { - "in": "query", - "name": "directory", + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "probes": { + "minItems": 2, + "maxItems": 2000, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "features": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "number" + } + }, + "stratum": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "weight": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1000 + }, + "priorLoss": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "features", + "stratum" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceLosses": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "stratum": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "weight": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1000 + } + }, + "required": [ + "id", + "commitment", + "sourceLosses", + "stratum" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "subject", + "probes" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.audit.initialize({\n ...\n})" + } + ] + } + }, + "/harness/audits/{auditID}/status": { + "post": { + "operationId": "harness.audit.status", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "auditID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected active audit", + "responses": { + "200": { + "description": "Active audit state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "enum": [ + "active-audit-v1", + "proactive-audit-v2" + ] + }, + "auditID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "poolFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "config": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "performance", + "failure", + "hybrid" + ] + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minSamples": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "noiseVariance": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 2 + }, + "lengthscale": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 100 + }, + "beta": { + "default": 1.96, + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "failureThreshold": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tolerance": { + "default": 0.02, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "estimationWeight": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "diversityWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "coverageWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + }, + "transfer": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "score-history-prior-v1" + }, + "poolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionMethod": { + "type": "string", + "enum": [ + "pca-gmm-profile-v1", + "holdout-embedding-gmm-v1" + ] + }, + "sourceModels": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "calibrationSamples": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "maxCalibrationMAE": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "poolSHA256", + "sourceManifestSHA256", + "selectionSHA256", + "selectionMethod", + "sourceModels", + "calibrationSamples", + "maxCalibrationMAE" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean" + } + }, + "required": [ + "mode", + "budget", + "minSamples" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "precision_reached", + "failure_target_reached", + "pool_exhausted" + ] + }, + "pool": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "features": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "number" + } + }, + "stratum": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "weight": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1000 + }, + "priorLoss": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sourceLosses": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "selection": { + "type": "object", + "properties": { + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "phase": { + "type": "string", + "enum": [ + "calibration", + "adaptive", + "fallback" + ] + }, + "acquisition": { + "type": "object", + "properties": { + "posteriorLoss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "posteriorStd": { + "type": "number", + "minimum": 0 + }, + "failureUCB": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "varianceReduction": { + "type": "number", + "minimum": 0 + }, + "diversity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "coverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "score": { + "type": "number" + } + }, + "required": [ + "posteriorLoss", + "posteriorStd", + "failureUCB", + "varianceReduction", + "diversity", + "coverage", + "score" + ], + "additionalProperties": false + } + }, + "required": [ + "round", + "selectedAt", + "acquisition" + ], + "additionalProperties": false + }, + "observation": { + "type": "object", + "properties": { + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "loss", + "failure", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "commitment", + "features", + "stratum" + ], + "additionalProperties": false + } + }, + "order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "estimate": { + "type": "object", + "properties": { + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanLoss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "lower95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "upper95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "abstain": { + "type": "boolean" + }, + "effectivePoolSize": { + "type": "number", + "exclusiveMinimum": 0 + }, + "stratumCoverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "transfer": { + "default": { + "status": "not_configured", + "observed": 0, + "required": 0 + }, + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "not_configured", + "calibrating", + "accepted", + "rejected" + ] + }, + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "required": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanAbsoluteError": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "threshold": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "status", + "observed", + "required" + ], + "additionalProperties": false + } + }, + "required": [ + "observed", + "failures", + "meanLoss", + "standardDeviation", + "lower95", + "upper95", + "abstain", + "effectivePoolSize", + "stratumCoverage" + ], + "additionalProperties": false + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "auditID", + "runID", + "sessionID", + "evaluator", + "contractFingerprint", + "poolFingerprint", + "subject", + "config", + "status", + "pool", + "order", + "estimate", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.audit.status({\n ...\n})" + } + ] + } + }, + "/harness/audits/{auditID}/selection": { + "post": { + "operationId": "harness.audit.select", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "auditID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Select the next opaque active-audit probe", + "description": "Combines weighted integral-variance reduction, failure UCB, failure-region diversity, and stratum coverage.", + "responses": { + "200": { + "description": "Selected opaque probe commitment" + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.audit.select({\n ...\n})" + } + ] + } + }, + "/harness/audits/{auditID}/observations": { + "post": { + "operationId": "harness.audit.observe", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "auditID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Record an evaluator-authenticated probe outcome", + "description": "Updates the GP posterior and stopping rule without promoting the audit estimate into benchmark evidence.", + "responses": { + "200": { + "description": "Updated active audit state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "enum": [ + "active-audit-v1", + "proactive-audit-v2" + ] + }, + "auditID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "poolFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "config": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "performance", + "failure", + "hybrid" + ] + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minSamples": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "noiseVariance": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 2 + }, + "lengthscale": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 100 + }, + "beta": { + "default": 1.96, + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "failureThreshold": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tolerance": { + "default": 0.02, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "estimationWeight": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "diversityWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "coverageWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + }, + "transfer": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "score-history-prior-v1" + }, + "poolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionMethod": { + "type": "string", + "enum": [ + "pca-gmm-profile-v1", + "holdout-embedding-gmm-v1" + ] + }, + "sourceModels": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "calibrationSamples": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "maxCalibrationMAE": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "poolSHA256", + "sourceManifestSHA256", + "selectionSHA256", + "selectionMethod", + "sourceModels", + "calibrationSamples", + "maxCalibrationMAE" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean" + } + }, + "required": [ + "mode", + "budget", + "minSamples" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "precision_reached", + "failure_target_reached", + "pool_exhausted" + ] + }, + "pool": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "features": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "number" + } + }, + "stratum": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "weight": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1000 + }, + "priorLoss": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sourceLosses": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "selection": { + "type": "object", + "properties": { + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "phase": { + "type": "string", + "enum": [ + "calibration", + "adaptive", + "fallback" + ] + }, + "acquisition": { + "type": "object", + "properties": { + "posteriorLoss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "posteriorStd": { + "type": "number", + "minimum": 0 + }, + "failureUCB": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "varianceReduction": { + "type": "number", + "minimum": 0 + }, + "diversity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "coverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "score": { + "type": "number" + } + }, + "required": [ + "posteriorLoss", + "posteriorStd", + "failureUCB", + "varianceReduction", + "diversity", + "coverage", + "score" + ], + "additionalProperties": false + } + }, + "required": [ + "round", + "selectedAt", + "acquisition" + ], + "additionalProperties": false + }, + "observation": { + "type": "object", + "properties": { + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "loss", + "failure", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "commitment", + "features", + "stratum" + ], + "additionalProperties": false + } + }, + "order": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "estimate": { + "type": "object", + "properties": { + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanLoss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "lower95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "upper95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "abstain": { + "type": "boolean" + }, + "effectivePoolSize": { + "type": "number", + "exclusiveMinimum": 0 + }, + "stratumCoverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "transfer": { + "default": { + "status": "not_configured", + "observed": 0, + "required": 0 + }, + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "not_configured", + "calibrating", + "accepted", + "rejected" + ] + }, + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "required": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanAbsoluteError": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "threshold": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "status", + "observed", + "required" + ], + "additionalProperties": false + } + }, + "required": [ + "observed", + "failures", + "meanLoss", + "standardDeviation", + "lower95", + "upper95", + "abstain", + "effectivePoolSize", + "stratumCoverage" + ], + "additionalProperties": false + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "auditID", + "runID", + "sessionID", + "evaluator", + "contractFingerprint", + "poolFingerprint", + "subject", + "config", + "status", + "pool", + "order", + "estimate", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "probeID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "probeID", + "loss", + "failure", + "evidence" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.audit.observe({\n ...\n})" + } + ] + } + }, + "/harness/audits/{auditID}/receipt": { + "post": { + "operationId": "harness.audit.seal", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "auditID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Seal a terminal active-audit receipt", + "description": "Content-addresses the completed audit, exact subject artifact, committed pool, derived estimate, transfer qualification, and terminal revision for optional promotion gating.", + "responses": { + "200": { + "description": "Immutable active-audit receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "proactive-audit-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "auditID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "poolFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "config": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "performance", + "failure", + "hybrid" + ] + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minSamples": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "noiseVariance": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 2 + }, + "lengthscale": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 100 + }, + "beta": { + "default": 1.96, + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "failureThreshold": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tolerance": { + "default": 0.02, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "estimationWeight": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "diversityWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "coverageWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + }, + "transfer": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "score-history-prior-v1" + }, + "poolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionMethod": { + "type": "string", + "enum": [ + "pca-gmm-profile-v1", + "holdout-embedding-gmm-v1" + ] + }, + "sourceModels": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "calibrationSamples": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "maxCalibrationMAE": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "poolSHA256", + "sourceManifestSHA256", + "selectionSHA256", + "selectionMethod", + "sourceModels", + "calibrationSamples", + "maxCalibrationMAE" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean" + } + }, + "required": [ + "mode", + "budget", + "minSamples" + ], + "additionalProperties": false + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "precision_reached", + "failure_target_reached", + "pool_exhausted" + ] + }, + "estimate": { + "type": "object", + "properties": { + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanLoss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "lower95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "upper95": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "abstain": { + "type": "boolean" + }, + "effectivePoolSize": { + "type": "number", + "exclusiveMinimum": 0 + }, + "stratumCoverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "transfer": { + "default": { + "status": "not_configured", + "observed": 0, + "required": 0 + }, + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "not_configured", + "calibrating", + "accepted", + "rejected" + ] + }, + "observed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "required": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanAbsoluteError": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "threshold": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "status", + "observed", + "required" + ], + "additionalProperties": false + } + }, + "required": [ + "observed", + "failures", + "meanLoss", + "standardDeviation", + "lower95", + "upper95", + "abstain", + "effectivePoolSize", + "stratumCoverage" + ], + "additionalProperties": false + }, + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "qualified": { + "type": "boolean" + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "sealedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "auditID", + "runID", + "sessionID", + "contractFingerprint", + "poolFingerprint", + "subject", + "config", + "stopReason", + "estimate", + "revision", + "qualified", + "completedAt", + "sealedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.audit.seal({\n ...\n})" + } + ] + } + }, + "/harness/failure-streams": { + "post": { + "operationId": "harness.failure.initialize", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Initialize a topic-aware adversarial failure stream", + "description": "Binds deterministic UCB1 topic allocation and server-derived failure anchors to a terminal active-audit receipt without adding generated cases to the population estimate.", + "responses": { + "200": { + "description": "Topic-aware failure discovery state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "streamID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "anchors": { + "minItems": 1, + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "config": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "topicModel": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "predefined", + "bertopic" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + }, + "topics": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "generator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "validators": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + } + }, + "embedding": { + "type": "object", + "properties": { + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "dimensions": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "regularization": { + "default": 0.000001, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.01 + } + }, + "required": [ + "identity", + "dimensions" + ], + "additionalProperties": false + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "anchorsPerAttempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "exploration": { + "default": 1.4142135623730951, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "failureThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + } + }, + "required": [ + "protocolVersion", + "sourcePoolSHA256", + "topicModel", + "topics", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "failureThreshold" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "failure_target_reached" + ] + }, + "pending": { + "type": "object", + "properties": { + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "topic": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + }, + "anchors": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "allocation": { + "type": "object", + "properties": { + "phase": { + "type": "string", + "enum": [ + "initialization", + "ucb1" + ] + }, + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "score": { + "type": "number" + } + }, + "required": [ + "phase", + "pulls", + "rewards", + "score" + ], + "additionalProperties": false + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "selectionID", + "round", + "topic", + "anchors", + "allocation", + "selectedAt" + ], + "additionalProperties": false + }, + "attempts": { + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selection": { + "type": "object", + "properties": { + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "topic": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + }, + "anchors": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "allocation": { + "type": "object", + "properties": { + "phase": { + "type": "string", + "enum": [ + "initialization", + "ucb1" + ] + }, + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "score": { + "type": "number" + } + }, + "required": [ + "phase", + "pulls", + "rewards", + "score" + ], + "additionalProperties": false + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "selectionID", + "round", + "topic", + "anchors", + "allocation", + "selectedAt" + ], + "additionalProperties": false + }, + "generation": { + "anyOf": [ + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "failed" + }, + "mode": { + "type": "string", + "enum": [ + "generator_error", + "timeout", + "invalid_output", + "other" + ] + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "mode", + "evidence" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "generated" + }, + "caseSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "embedding": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "number" + } + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "caseSHA256", + "outputSHA256", + "embedding", + "evidence" + ], + "additionalProperties": false + } + ] + }, + "validations": { + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "kind", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "outcome": { + "type": "object", + "properties": { + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "loss", + "failure", + "outputSHA256", + "evidence" + ], + "additionalProperties": false + }, + "admissible": { + "type": "boolean" + }, + "reward": { + "anyOf": [ + { + "type": "number", + "const": 0 + }, + { + "type": "number", + "const": 1 + } + ] + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "attemptID", + "selection", + "generation", + "validations", + "admissible", + "reward", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "statistics": { + "type": "object", + "properties": { + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "generated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "admissible": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "invalid": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "samplesToFirstFailure": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "failureRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "topicEntropy": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "embeddingLogDet": { + "type": "number" + }, + "topics": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rate": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "pulls", + "rewards", + "rate" + ], + "additionalProperties": false + } + } + }, + "required": [ + "attempts", + "generated", + "admissible", + "failures", + "invalid", + "failureRate", + "topicEntropy", + "embeddingLogDet", + "topics" + ], + "additionalProperties": false + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "streamID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "auditReceiptID", + "sourcePoolSHA256", + "anchors", + "config", + "status", + "attempts", + "statistics", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "subject", + "auditReceiptID" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.failure.initialize({\n ...\n})" + } + ] + } + }, + "/harness/failure-streams/{streamID}/status": { + "post": { + "operationId": "harness.failure.status", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "streamID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected failure discovery stream", + "responses": { + "200": { + "description": "Topic-aware failure discovery state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "streamID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "anchors": { + "minItems": 1, + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "config": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "topicModel": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "predefined", + "bertopic" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + }, + "topics": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "generator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "validators": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + } + }, + "embedding": { + "type": "object", + "properties": { + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "dimensions": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "regularization": { + "default": 0.000001, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.01 + } + }, + "required": [ + "identity", + "dimensions" + ], + "additionalProperties": false + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "anchorsPerAttempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "exploration": { + "default": 1.4142135623730951, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "failureThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + } + }, + "required": [ + "protocolVersion", + "sourcePoolSHA256", + "topicModel", + "topics", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "failureThreshold" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "failure_target_reached" + ] + }, + "pending": { + "type": "object", + "properties": { + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "topic": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + }, + "anchors": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "allocation": { + "type": "object", + "properties": { + "phase": { + "type": "string", + "enum": [ + "initialization", + "ucb1" + ] + }, + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "score": { + "type": "number" + } + }, + "required": [ + "phase", + "pulls", + "rewards", + "score" + ], + "additionalProperties": false + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "selectionID", + "round", + "topic", + "anchors", + "allocation", + "selectedAt" + ], + "additionalProperties": false + }, + "attempts": { + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selection": { + "type": "object", + "properties": { + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "topic": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + }, + "anchors": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "allocation": { + "type": "object", + "properties": { + "phase": { + "type": "string", + "enum": [ + "initialization", + "ucb1" + ] + }, + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "score": { + "type": "number" + } + }, + "required": [ + "phase", + "pulls", + "rewards", + "score" + ], + "additionalProperties": false + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "selectionID", + "round", + "topic", + "anchors", + "allocation", + "selectedAt" + ], + "additionalProperties": false + }, + "generation": { + "anyOf": [ + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "failed" + }, + "mode": { + "type": "string", + "enum": [ + "generator_error", + "timeout", + "invalid_output", + "other" + ] + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "mode", + "evidence" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "generated" + }, + "caseSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "embedding": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "number" + } + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "caseSHA256", + "outputSHA256", + "embedding", + "evidence" + ], + "additionalProperties": false + } + ] + }, + "validations": { + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "kind", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "outcome": { + "type": "object", + "properties": { + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "loss", + "failure", + "outputSHA256", + "evidence" + ], + "additionalProperties": false + }, + "admissible": { + "type": "boolean" + }, + "reward": { + "anyOf": [ + { + "type": "number", + "const": 0 + }, + { + "type": "number", + "const": 1 + } + ] + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "attemptID", + "selection", + "generation", + "validations", + "admissible", + "reward", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "statistics": { + "type": "object", + "properties": { + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "generated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "admissible": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "invalid": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "samplesToFirstFailure": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "failureRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "topicEntropy": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "embeddingLogDet": { + "type": "number" + }, + "topics": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rate": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "pulls", + "rewards", + "rate" + ], + "additionalProperties": false + } + } + }, + "required": [ + "attempts", + "generated", + "admissible", + "failures", + "invalid", + "failureRate", + "topicEntropy", + "embeddingLogDet", + "topics" + ], + "additionalProperties": false + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "streamID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "auditReceiptID", + "sourcePoolSHA256", + "anchors", + "config", + "status", + "attempts", + "statistics", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.failure.status({\n ...\n})" + } + ] + } + }, + "/harness/failure-streams/{streamID}/selection": { + "post": { + "operationId": "harness.failure.select", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "streamID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Select the next topic and authenticated failure anchors", + "description": "Forces every frozen topic once, then derives UCB1 from the immutable attempt journal with deterministic tie-breaking.", + "responses": { + "200": { + "description": "Server-selected topic, anchors, and allocation evidence", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "topic": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + }, + "anchors": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "allocation": { + "type": "object", + "properties": { + "phase": { + "type": "string", + "enum": [ + "initialization", + "ucb1" + ] + }, + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "score": { + "type": "number" + } + }, + "required": [ + "phase", + "pulls", + "rewards", + "score" + ], + "additionalProperties": false + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "selectionID", + "round", + "topic", + "anchors", + "allocation", + "selectedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.failure.select({\n ...\n})" + } + ] + } + }, + "/harness/failure-streams/{streamID}/attempts": { + "post": { + "operationId": "harness.failure.observe", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "streamID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Record a validated adversarial generation attempt", + "description": "Consumes one attempt budget and derives admissibility and reward from the frozen correctness, topic, and novelty validators plus the target outcome.", + "responses": { + "200": { + "description": "Updated topic-aware failure discovery state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "streamID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "anchors": { + "minItems": 1, + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "config": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "topicModel": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "predefined", + "bertopic" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + }, + "topics": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "generator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "validators": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + } + }, + "embedding": { + "type": "object", + "properties": { + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "dimensions": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "regularization": { + "default": 0.000001, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.01 + } + }, + "required": [ + "identity", + "dimensions" + ], + "additionalProperties": false + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "anchorsPerAttempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "exploration": { + "default": 1.4142135623730951, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "failureThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + } + }, + "required": [ + "protocolVersion", + "sourcePoolSHA256", + "topicModel", + "topics", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "failureThreshold" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "active", + "completed" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "failure_target_reached" + ] + }, + "pending": { + "type": "object", + "properties": { + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "topic": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + }, + "anchors": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "allocation": { + "type": "object", + "properties": { + "phase": { + "type": "string", + "enum": [ + "initialization", + "ucb1" + ] + }, + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "score": { + "type": "number" + } + }, + "required": [ + "phase", + "pulls", + "rewards", + "score" + ], + "additionalProperties": false + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "selectionID", + "round", + "topic", + "anchors", + "allocation", + "selectedAt" + ], + "additionalProperties": false + }, + "attempts": { + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selection": { + "type": "object", + "properties": { + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "round": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "topic": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + }, + "anchors": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "id", + "commitment", + "loss" + ], + "additionalProperties": false + } + }, + "allocation": { + "type": "object", + "properties": { + "phase": { + "type": "string", + "enum": [ + "initialization", + "ucb1" + ] + }, + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "score": { + "type": "number" + } + }, + "required": [ + "phase", + "pulls", + "rewards", + "score" + ], + "additionalProperties": false + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "selectionID", + "round", + "topic", + "anchors", + "allocation", + "selectedAt" + ], + "additionalProperties": false + }, + "generation": { + "anyOf": [ + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "failed" + }, + "mode": { + "type": "string", + "enum": [ + "generator_error", + "timeout", + "invalid_output", + "other" + ] + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "mode", + "evidence" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "generated" + }, + "caseSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "embedding": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "number" + } + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "caseSHA256", + "outputSHA256", + "embedding", + "evidence" + ], + "additionalProperties": false + } + ] + }, + "validations": { + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "kind", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "outcome": { + "type": "object", + "properties": { + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "loss", + "failure", + "outputSHA256", + "evidence" + ], + "additionalProperties": false + }, + "admissible": { + "type": "boolean" + }, + "reward": { + "anyOf": [ + { + "type": "number", + "const": 0 + }, + { + "type": "number", + "const": 1 + } + ] + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "attemptID", + "selection", + "generation", + "validations", + "admissible", + "reward", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "statistics": { + "type": "object", + "properties": { + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "generated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "admissible": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "invalid": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "samplesToFirstFailure": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "failureRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "topicEntropy": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "embeddingLogDet": { + "type": "number" + }, + "topics": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rate": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "pulls", + "rewards", + "rate" + ], + "additionalProperties": false + } + } + }, + "required": [ + "attempts", + "generated", + "admissible", + "failures", + "invalid", + "failureRate", + "topicEntropy", + "embeddingLogDet", + "topics" + ], + "additionalProperties": false + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "streamID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "auditReceiptID", + "sourcePoolSHA256", + "anchors", + "config", + "status", + "attempts", + "statistics", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "generation": { + "anyOf": [ + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "failed" + }, + "mode": { + "type": "string", + "enum": [ + "generator_error", + "timeout", + "invalid_output", + "other" + ] + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "mode", + "evidence" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "status": { + "type": "string", + "const": "generated" + }, + "caseSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "embedding": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "number" + } + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "caseSHA256", + "outputSHA256", + "embedding", + "evidence" + ], + "additionalProperties": false + } + ] + }, + "validations": { + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "kind", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "outcome": { + "type": "object", + "properties": { + "loss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "failure": { + "type": "boolean" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "loss", + "failure", + "outputSHA256", + "evidence" + ], + "additionalProperties": false + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "selectionID", + "generation", + "validations", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.failure.observe({\n ...\n})" + } + ] + } + }, + "/harness/failure-streams/{streamID}/receipt": { + "post": { + "operationId": "harness.failure.seal", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "streamID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Seal a terminal failure discovery receipt", + "description": "Content-addresses the exact audit source, subject, topic contract, attempt journal, replayed UCB statistics, failure yield, and diversity evidence.", + "responses": { + "200": { + "description": "Immutable topic-aware failure discovery receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "streamID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "type", + "id", + "artifactSHA256" + ], + "additionalProperties": false + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "config": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "topicModel": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "predefined", + "bertopic" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + }, + "topics": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "generator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "validators": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + } + }, + "embedding": { + "type": "object", + "properties": { + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "dimensions": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "regularization": { + "default": 0.000001, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.01 + } + }, + "required": [ + "identity", + "dimensions" + ], + "additionalProperties": false + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "anchorsPerAttempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "exploration": { + "default": 1.4142135623730951, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "failureThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + } + }, + "required": [ + "protocolVersion", + "sourcePoolSHA256", + "topicModel", + "topics", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "failureThreshold" + ], + "additionalProperties": false + }, + "attemptIDs": { + "minItems": 2, + "maxItems": 512, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "statistics": { + "type": "object", + "properties": { + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "generated": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "admissible": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "invalid": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "samplesToFirstFailure": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "failureRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "topicEntropy": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "embeddingLogDet": { + "type": "number" + }, + "topics": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "pulls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rewards": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rate": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "pulls", + "rewards", + "rate" + ], + "additionalProperties": false + } + } + }, + "required": [ + "attempts", + "generated", + "admissible", + "failures", + "invalid", + "failureRate", + "topicEntropy", + "embeddingLogDet", + "topics" + ], + "additionalProperties": false + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "failure_target_reached" + ] + }, + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "sealedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "streamID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "auditReceiptID", + "sourcePoolSHA256", + "config", + "attemptIDs", + "statistics", + "stopReason", + "revision", + "completedAt", + "sealedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.failure.seal({\n ...\n})" + } + ] + } + }, + "/harness/ablations": { + "post": { + "operationId": "harness.ablation.initialize", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Freeze a matched scientific ablation plan", + "description": "Binds at least three evaluator-authenticated seed pairs before evaluation and permits exactly one declared contract factor to differ.", + "responses": { + "200": { + "description": "Immutable matched ablation plan", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "plan": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "studyID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "factor": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "profile", + "orchestration", + "search", + "audit", + "simulation", + "evaluator_audit", + "semantic_audit", + "synthesis", + "autonomy", + "formal_proof", + "replication", + "fidelities", + "skill", + "tool" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "baselineValueSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "armValueSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contextSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "evaluatorVersion": { + "type": "string", + "minLength": 1 + }, + "metric": { + "type": "string", + "minLength": 1 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "name", + "version", + "taskID", + "split", + "evaluator", + "evaluatorVersion", + "metric", + "direction" + ], + "additionalProperties": false + }, + "minEffect": { + "type": "number", + "minimum": 0 + }, + "maxPairRegression": { + "type": "number", + "minimum": 0 + }, + "pairs": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "baseline": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "contractFingerprint" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "contractFingerprint" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "baseline", + "arm" + ], + "additionalProperties": false + } + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "planID", + "studyID", + "factor", + "baselineValueSHA256", + "armValueSHA256", + "contextSHA256", + "benchmark", + "minEffect", + "maxPairRegression", + "pairs", + "createdAt" + ], + "additionalProperties": false + }, + "receipt": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "studyID": { + "type": "string", + "minLength": 1 + }, + "factor": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "profile", + "orchestration", + "search", + "audit", + "simulation", + "evaluator_audit", + "semantic_audit", + "synthesis", + "autonomy", + "formal_proof", + "replication", + "fidelities", + "skill", + "tool" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "pairs": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "baseline": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + }, + "effect": { + "type": "number" + } + }, + "required": [ + "seed", + "baseline", + "arm" + ], + "additionalProperties": false + } + }, + "statistics": { + "type": "object", + "properties": { + "pairs": { + "type": "integer", + "minimum": 3, + "maximum": 9007199254740991 + }, + "validPairs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanEffect": { + "type": "number" + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "standardError": { + "type": "number", + "minimum": 0 + }, + "confidence95": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "regressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "minEffect": { + "type": "number", + "minimum": 0 + }, + "maxPairRegression": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "pairs", + "validPairs", + "regressions", + "minEffect", + "maxPairRegression" + ], + "additionalProperties": false + }, + "verdict": { + "type": "string", + "enum": [ + "supported", + "rejected", + "inconclusive" + ] + }, + "assessedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "planID", + "studyID", + "factor", + "pairs", + "statistics", + "verdict", + "assessedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "plan" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "studyID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "factor": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "profile", + "orchestration", + "search", + "audit", + "simulation", + "evaluator_audit", + "semantic_audit", + "synthesis", + "autonomy", + "formal_proof", + "replication", + "fidelities", + "skill", + "tool" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "minEffect": { + "type": "number", + "minimum": 0 + }, + "maxPairRegression": { + "default": 0, + "type": "number", + "minimum": 0 + }, + "pairs": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "baseline": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + }, + "required": [ + "baseline", + "arm" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "studyID", + "factor", + "minEffect", + "pairs" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.ablation.initialize({\n ...\n})" + } + ] + } + }, + "/harness/ablations/{planID}/assessment": { + "post": { + "operationId": "harness.ablation.assess", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "planID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Assess a frozen matched ablation", + "description": "Authenticates every paired run, verifies immutable contracts and final evaluations, then derives paired effects and a 95% interval.", + "responses": { + "200": { + "description": "Immutable matched ablation assessment", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "plan": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "studyID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "factor": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "profile", + "orchestration", + "search", + "audit", + "simulation", + "evaluator_audit", + "semantic_audit", + "synthesis", + "autonomy", + "formal_proof", + "replication", + "fidelities", + "skill", + "tool" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "baselineValueSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "armValueSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contextSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "evaluatorVersion": { + "type": "string", + "minLength": 1 + }, + "metric": { + "type": "string", + "minLength": 1 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "name", + "version", + "taskID", + "split", + "evaluator", + "evaluatorVersion", + "metric", + "direction" + ], + "additionalProperties": false + }, + "minEffect": { + "type": "number", + "minimum": 0 + }, + "maxPairRegression": { + "type": "number", + "minimum": 0 + }, + "pairs": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "baseline": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "contractFingerprint" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "contractFingerprint" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "baseline", + "arm" + ], + "additionalProperties": false + } + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "planID", + "studyID", + "factor", + "baselineValueSHA256", + "armValueSHA256", + "contextSHA256", + "benchmark", + "minEffect", + "maxPairRegression", + "pairs", + "createdAt" + ], + "additionalProperties": false + }, + "receipt": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "studyID": { + "type": "string", + "minLength": 1 + }, + "factor": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "profile", + "orchestration", + "search", + "audit", + "simulation", + "evaluator_audit", + "semantic_audit", + "synthesis", + "autonomy", + "formal_proof", + "replication", + "fidelities", + "skill", + "tool" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + "pairs": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "baseline": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + }, + "effect": { + "type": "number" + } + }, + "required": [ + "seed", + "baseline", + "arm" + ], + "additionalProperties": false + } + }, + "statistics": { + "type": "object", + "properties": { + "pairs": { + "type": "integer", + "minimum": 3, + "maximum": 9007199254740991 + }, + "validPairs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanEffect": { + "type": "number" + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "standardError": { + "type": "number", + "minimum": 0 + }, + "confidence95": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "regressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "minEffect": { + "type": "number", + "minimum": 0 + }, + "maxPairRegression": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "pairs", + "validPairs", + "regressions", + "minEffect", + "maxPairRegression" + ], + "additionalProperties": false + }, + "verdict": { + "type": "string", + "enum": [ + "supported", + "rejected", + "inconclusive" + ] + }, + "assessedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "planID", + "studyID", + "factor", + "pairs", + "statistics", + "verdict", + "assessedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "plan" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "runs": { + "minItems": 6, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + }, + "required": [ + "runs" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.ablation.assess({\n ...\n})" + } + ] + } + }, + "/harness/interventions": { + "post": { + "operationId": "harness.intervention.initialize", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Freeze an evaluator-owned controlled replay study", + "description": "Binds a candidate and exact evolution receipt to predeclared replay, retuning, ablation, repair, or transfer pairs before the candidate's final evaluation.", + "responses": { + "200": { + "description": "Immutable controlled intervention plan", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "plan": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "intervention-study-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "requiredForPromotion": { + "type": "boolean" + }, + "minPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxTotalPairs": { + "type": "integer", + "minimum": 3, + "maximum": 256 + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "required": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + } + }, + "rules": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "family": { + "type": "string", + "const": "replay" + }, + "mode": { + "type": "string", + "const": "max_absolute_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "retune", + "ablation", + "repair" + ] + }, + "mode": { + "type": "string", + "const": "min_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "const": "max_regression" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "requiredForPromotion", + "minPairs", + "maxPairs", + "maxTotalPairs", + "confidence", + "required", + "rules" + ], + "additionalProperties": false + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "metric": { + "type": "string", + "minLength": 1 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "name", + "version", + "taskID", + "metric", + "direction" + ], + "additionalProperties": false + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "design-replay-interventions" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "pairs": { + "minItems": 3, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "index": { + "type": "integer", + "minimum": 0, + "maximum": 31 + }, + "control": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "condition": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "provider", + "name", + "version" + ], + "additionalProperties": false + }, + "context": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "split": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "manifest": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "manifest" + ], + "additionalProperties": false + }, + "environment": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "budget": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "model", + "context", + "evaluator", + "split", + "environment", + "budget" + ], + "additionalProperties": false + } + }, + "required": [ + "artifact", + "condition" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "condition": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "provider", + "name", + "version" + ], + "additionalProperties": false + }, + "context": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "split": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "manifest": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "manifest" + ], + "additionalProperties": false + }, + "environment": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "budget": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "model", + "context", + "evaluator", + "split", + "environment", + "budget" + ], + "additionalProperties": false + } + }, + "required": [ + "artifact", + "condition" + ], + "additionalProperties": false + }, + "change": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "pairID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "family", + "index", + "control", + "arm", + "change", + "pairID" + ], + "additionalProperties": false + } + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "planID", + "runID", + "sessionID", + "contractFingerprint", + "protocol", + "benchmark", + "subject", + "evolutionReceiptID", + "validator", + "pairs", + "createdAt" + ], + "additionalProperties": false + }, + "outcomes": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "outcomeID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "submissionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "pairID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "enum": [ + "control", + "arm" + ] + }, + "targetSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "outcomeID", + "submissionID", + "pairID", + "role", + "targetSHA256", + "status", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "order": { + "type": "array", + "items": { + "type": "string" + } + }, + "receipt": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "families": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "enum": [ + "max_absolute_effect", + "min_effect", + "max_regression" + ] + }, + "threshold": { + "type": "number" + }, + "pairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "validPairs": { + "type": "integer", + "minimum": 0, + "maximum": 32 + }, + "meanEffect": { + "type": "number" + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "standardError": { + "type": "number", + "minimum": 0 + }, + "confidence95": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "maxAbsoluteEffect": { + "type": "number", + "minimum": 0 + }, + "regressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "verdict": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + } + }, + "required": [ + "family", + "mode", + "threshold", + "pairs", + "validPairs", + "regressions", + "verdict" + ], + "additionalProperties": false + } + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "observedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "assessedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "planID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evolutionReceiptID", + "families", + "status", + "observedAt", + "assessedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "plan", + "outcomes", + "order" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "design-replay-interventions" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "pairs": { + "minItems": 3, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "index": { + "type": "integer", + "minimum": 0, + "maximum": 31 + }, + "control": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "condition": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "provider", + "name", + "version" + ], + "additionalProperties": false + }, + "context": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "split": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "manifest": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "manifest" + ], + "additionalProperties": false + }, + "environment": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "budget": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "model", + "context", + "evaluator", + "split", + "environment", + "budget" + ], + "additionalProperties": false + } + }, + "required": [ + "artifact", + "condition" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "condition": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "provider", + "name", + "version" + ], + "additionalProperties": false + }, + "context": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "split": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "manifest": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "manifest" + ], + "additionalProperties": false + }, + "environment": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "budget": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "model", + "context", + "evaluator", + "split", + "environment", + "budget" + ], + "additionalProperties": false + } + }, + "required": [ + "artifact", + "condition" + ], + "additionalProperties": false + }, + "change": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "family", + "index", + "control", + "arm", + "change" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "evaluatorToken", + "subject", + "evolutionReceiptID", + "validator", + "pairs" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.intervention.initialize({\n ...\n})" + } + ] + } + }, + "/harness/interventions/{candidateID}/observations": { + "post": { + "operationId": "harness.intervention.observe", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "candidateID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Record an evaluator-authenticated intervention outcome", + "description": "Binds one numeric outcome to an exact frozen pair target without adding it to candidate fitness or the benchmark evaluation journal.", + "responses": { + "200": { + "description": "Immutable intervention outcome", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "outcomeID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "submissionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "pairID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "enum": [ + "control", + "arm" + ] + }, + "targetSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "outcomeID", + "submissionID", + "pairID", + "role", + "targetSHA256", + "status", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "pairID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "enum": [ + "control", + "arm" + ] + }, + "targetSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "sessionID", + "evaluatorToken", + "pairID", + "role", + "targetSHA256", + "status", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.intervention.observe({\n ...\n})" + } + ] + } + }, + "/harness/interventions/{candidateID}/assessment": { + "post": { + "operationId": "harness.intervention.assess", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "candidateID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Assess a complete controlled replay study", + "description": "Recomputes direction-aware paired effects, confidence intervals, stability, tuning gap, component dependence, and transfer robustness from every frozen outcome.", + "responses": { + "200": { + "description": "Immutable controlled intervention receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "families": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "enum": [ + "max_absolute_effect", + "min_effect", + "max_regression" + ] + }, + "threshold": { + "type": "number" + }, + "pairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "validPairs": { + "type": "integer", + "minimum": 0, + "maximum": 32 + }, + "meanEffect": { + "type": "number" + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "standardError": { + "type": "number", + "minimum": 0 + }, + "confidence95": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "maxAbsoluteEffect": { + "type": "number", + "minimum": 0 + }, + "regressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "verdict": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + } + }, + "required": [ + "family", + "mode", + "threshold", + "pairs", + "validPairs", + "regressions", + "verdict" + ], + "additionalProperties": false + } + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "observedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "assessedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "planID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evolutionReceiptID", + "families", + "status", + "observedAt", + "assessedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.intervention.assess({\n ...\n})" + } + ] + } + }, + "/harness/interventions/{candidateID}/status": { + "post": { + "operationId": "harness.intervention.status", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "candidateID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected controlled replay study", + "responses": { + "200": { + "description": "Controlled intervention state", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "plan": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "intervention-study-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "requiredForPromotion": { + "type": "boolean" + }, + "minPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxTotalPairs": { + "type": "integer", + "minimum": 3, + "maximum": 256 + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "required": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + } + }, + "rules": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "family": { + "type": "string", + "const": "replay" + }, + "mode": { + "type": "string", + "const": "max_absolute_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "retune", + "ablation", + "repair" + ] + }, + "mode": { + "type": "string", + "const": "min_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "const": "max_regression" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "requiredForPromotion", + "minPairs", + "maxPairs", + "maxTotalPairs", + "confidence", + "required", + "rules" + ], + "additionalProperties": false + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "metric": { + "type": "string", + "minLength": 1 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "name", + "version", + "taskID", + "metric", + "direction" + ], + "additionalProperties": false + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "design-replay-interventions" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "pairs": { + "minItems": 3, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "index": { + "type": "integer", + "minimum": 0, + "maximum": 31 + }, + "control": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "condition": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "provider", + "name", + "version" + ], + "additionalProperties": false + }, + "context": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "split": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "manifest": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "manifest" + ], + "additionalProperties": false + }, + "environment": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "budget": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "model", + "context", + "evaluator", + "split", + "environment", + "budget" + ], + "additionalProperties": false + } + }, + "required": [ + "artifact", + "condition" + ], + "additionalProperties": false + }, + "arm": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "condition": { + "type": "object", + "properties": { + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + }, + "required": [ + "provider", + "name", + "version" + ], + "additionalProperties": false + }, + "context": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "split": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "manifest": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "manifest" + ], + "additionalProperties": false + }, + "environment": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "budget": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "seed", + "model", + "context", + "evaluator", + "split", + "environment", + "budget" + ], + "additionalProperties": false + } + }, + "required": [ + "artifact", + "condition" + ], + "additionalProperties": false + }, + "change": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "pairID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "family", + "index", + "control", + "arm", + "change", + "pairID" + ], + "additionalProperties": false + } + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "planID", + "runID", + "sessionID", + "contractFingerprint", + "protocol", + "benchmark", + "subject", + "evolutionReceiptID", + "validator", + "pairs", + "createdAt" + ], + "additionalProperties": false + }, + "outcomes": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "outcomeID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "submissionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "pairID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "enum": [ + "control", + "arm" + ] + }, + "targetSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "outcomeID", + "submissionID", + "pairID", + "role", + "targetSHA256", + "status", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "order": { + "type": "array", + "items": { + "type": "string" + } + }, + "receipt": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "planID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "families": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "enum": [ + "max_absolute_effect", + "min_effect", + "max_regression" + ] + }, + "threshold": { + "type": "number" + }, + "pairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "validPairs": { + "type": "integer", + "minimum": 0, + "maximum": 32 + }, + "meanEffect": { + "type": "number" + }, + "standardDeviation": { + "type": "number", + "minimum": 0 + }, + "standardError": { + "type": "number", + "minimum": 0 + }, + "confidence95": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "maxAbsoluteEffect": { + "type": "number", + "minimum": 0 + }, + "regressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "verdict": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + } + }, + "required": [ + "family", + "mode", + "threshold", + "pairs", + "validPairs", + "regressions", + "verdict" + ], + "additionalProperties": false + } + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "observedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "assessedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "planID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evolutionReceiptID", + "families", + "status", + "observedAt", + "assessedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "schemaVersion", + "plan", + "outcomes", + "order" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.intervention.status({\n ...\n})" + } + ] + } + }, + "/harness/evaluators/qualifications": { + "post": { + "operationId": "harness.judge.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Qualify a bound benchmark evaluator", + "description": "Uses an independent auditor capability and a committed hidden fault suite to recompute evaluator discrimination and calibration metrics.", + "responses": { + "200": { + "description": "Immutable evaluator audit receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "evaluator-audit-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1 + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "auditor": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "suite": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commitmentSHA256" + ], + "additionalProperties": false + }, + "cases": { + "minItems": 3, + "maxItems": 2048, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "type": "string", + "enum": [ + "clean", + "fault" + ] + }, + "fault": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + }, + "decision": { + "type": "string", + "enum": [ + "accept", + "reject", + "abstain" + ] + }, + "failureProbability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "kind", + "decision", + "failureProbability", + "evidence" + ], + "additionalProperties": false + } + }, + "metrics": { + "type": "object", + "properties": { + "cases": { + "type": "integer", + "minimum": 3, + "maximum": 9007199254740991 + }, + "cleanCases": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "faultCases": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "truePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falseNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "trueNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falsePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sensitivity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "specificity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "balancedAccuracy": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "brierScore": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "perFault": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + }, + "additionalProperties": { + "type": "object", + "properties": { + "cases": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "detected": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "recall": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "cases", + "detected", + "recall" + ], + "additionalProperties": false + } + } + }, + "required": [ + "cases", + "cleanCases", + "faultCases", + "truePositive", + "falseNegative", + "trueNegative", + "falsePositive", + "sensitivity", + "specificity", + "balancedAccuracy", + "brierScore", + "perFault" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "failures": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "protocolSHA256", + "sourceSessionID", + "evaluator", + "auditor", + "suite", + "cases", + "metrics", + "status", + "failures", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "auditorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "cases": { + "minItems": 3, + "maxItems": 2048, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "type": "string", + "enum": [ + "clean", + "fault" + ] + }, + "fault": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + }, + "decision": { + "type": "string", + "enum": [ + "accept", + "reject", + "abstain" + ] + }, + "failureProbability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "kind", + "decision", + "failureProbability", + "evidence" + ], + "additionalProperties": false + } + } + }, + "required": [ + "sessionID", + "auditorToken", + "cases" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.judge.record({\n ...\n})" + } + ] + } + }, + "/harness/evaluators/qualifications/{receiptID}": { + "post": { + "operationId": "harness.judge.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected evaluator qualification", + "responses": { + "200": { + "description": "Evaluator audit receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "evaluator-audit-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1 + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "auditor": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "suite": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commitmentSHA256" + ], + "additionalProperties": false + }, + "cases": { + "minItems": 3, + "maxItems": 2048, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "type": "string", + "enum": [ + "clean", + "fault" + ] + }, + "fault": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + }, + "decision": { + "type": "string", + "enum": [ + "accept", + "reject", + "abstain" + ] + }, + "failureProbability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "kind", + "decision", + "failureProbability", + "evidence" + ], + "additionalProperties": false + } + }, + "metrics": { + "type": "object", + "properties": { + "cases": { + "type": "integer", + "minimum": 3, + "maximum": 9007199254740991 + }, + "cleanCases": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "faultCases": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "truePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falseNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "trueNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falsePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sensitivity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "specificity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "balancedAccuracy": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "brierScore": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "perFault": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + }, + "additionalProperties": { + "type": "object", + "properties": { + "cases": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "detected": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "recall": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "cases", + "detected", + "recall" + ], + "additionalProperties": false + } + } + }, + "required": [ + "cases", + "cleanCases", + "faultCases", + "truePositive", + "falseNegative", + "trueNegative", + "falsePositive", + "sensitivity", + "specificity", + "balancedAccuracy", + "brierScore", + "perFault" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "failures": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "protocolSHA256", + "sourceSessionID", + "evaluator", + "auditor", + "suite", + "cases", + "metrics", + "status", + "failures", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "auditorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "auditorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.judge.receipt({\n ...\n})" + } + ] + } + }, + "/harness/replications/receipts": { + "post": { + "operationId": "harness.replication.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record an evaluator-authenticated replicated evaluation", + "description": "Requires the complete frozen stratum-by-cluster grid, then recomputes a robust estimate, uncertainty interval, and conservative promotion verdict.", + "responses": { + "200": { + "description": "Immutable replicated evaluation receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "replicated-evaluation-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "replicated-evaluation-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sampling": { + "type": "object", + "properties": { + "design": { + "type": "string", + "const": "crossed-stratified-cluster-v1" + }, + "stratumKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "strata": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + }, + "clusters": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "design", + "stratumKind", + "clusterKind", + "strata", + "clusters" + ], + "additionalProperties": false + }, + "estimator": { + "type": "string", + "enum": [ + "mean", + "median", + "iqm", + "pass_rate" + ] + }, + "interval": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "stratified-bootstrap-percentile-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "resamples": { + "type": "integer", + "minimum": 1000, + "maximum": 50000 + }, + "seed": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + }, + "required": [ + "method", + "confidence", + "resamples", + "seed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "wilson-score-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + } + }, + "required": [ + "method", + "confidence" + ], + "additionalProperties": false + } + ] + }, + "decision": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "conservative-bound-v1" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + }, + "maxIntervalWidth": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "rule", + "direction", + "target" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "environmentSHA256", + "sampling", + "estimator", + "interval", + "decision", + "failurePolicy" + ], + "additionalProperties": false + }, + "observations": { + "minItems": 3, + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "stratumID": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterID": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "stratumSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "clusterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "stratumID", + "clusterID", + "stratumSHA256", + "clusterSHA256", + "status", + "outputSHA256", + "environmentSHA256", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + }, + "statistics": { + "type": "object", + "properties": { + "units": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "passed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inconclusive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "estimator": { + "type": "string", + "enum": [ + "mean", + "median", + "iqm", + "pass_rate" + ] + }, + "estimate": { + "type": "number" + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "interval": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "intervalWidth": { + "type": "number", + "minimum": 0 + }, + "conservativeBound": { + "type": "number" + }, + "method": { + "type": "string", + "enum": [ + "stratified-bootstrap-percentile-v1", + "wilson-score-v1" + ] + }, + "resamples": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "units", + "passed", + "failed", + "inconclusive", + "estimator", + "confidence", + "method" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 1024, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 16384, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "protocolSHA256", + "contractSHA256", + "sourceSessionID", + "subject", + "metric", + "protocol", + "observations", + "statistics", + "status", + "failures", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "observations": { + "minItems": 3, + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "stratumID": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterID": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "stratumSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "clusterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "stratumID", + "clusterID", + "stratumSHA256", + "clusterSHA256", + "status", + "outputSHA256", + "environmentSHA256", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "subject", + "observations" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.replication.record({\n ...\n})" + } + ] + } + }, + "/harness/replications/receipts/{receiptID}": { + "post": { + "operationId": "harness.replication.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected replicated evaluation receipt", + "responses": { + "200": { + "description": "Replicated evaluation receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "replicated-evaluation-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "replicated-evaluation-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sampling": { + "type": "object", + "properties": { + "design": { + "type": "string", + "const": "crossed-stratified-cluster-v1" + }, + "stratumKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "strata": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + }, + "clusters": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "design", + "stratumKind", + "clusterKind", + "strata", + "clusters" + ], + "additionalProperties": false + }, + "estimator": { + "type": "string", + "enum": [ + "mean", + "median", + "iqm", + "pass_rate" + ] + }, + "interval": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "stratified-bootstrap-percentile-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "resamples": { + "type": "integer", + "minimum": 1000, + "maximum": 50000 + }, + "seed": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + }, + "required": [ + "method", + "confidence", + "resamples", + "seed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "wilson-score-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + } + }, + "required": [ + "method", + "confidence" + ], + "additionalProperties": false + } + ] + }, + "decision": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "conservative-bound-v1" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + }, + "maxIntervalWidth": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "rule", + "direction", + "target" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "environmentSHA256", + "sampling", + "estimator", + "interval", + "decision", + "failurePolicy" + ], + "additionalProperties": false + }, + "observations": { + "minItems": 3, + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "stratumID": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterID": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "stratumSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "clusterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "stratumID", + "clusterID", + "stratumSHA256", + "clusterSHA256", + "status", + "outputSHA256", + "environmentSHA256", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + }, + "statistics": { + "type": "object", + "properties": { + "units": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "passed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "inconclusive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "estimator": { + "type": "string", + "enum": [ + "mean", + "median", + "iqm", + "pass_rate" + ] + }, + "estimate": { + "type": "number" + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "interval": { + "type": "array", + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "intervalWidth": { + "type": "number", + "minimum": 0 + }, + "conservativeBound": { + "type": "number" + }, + "method": { + "type": "string", + "enum": [ + "stratified-bootstrap-percentile-v1", + "wilson-score-v1" + ] + }, + "resamples": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "units", + "passed", + "failed", + "inconclusive", + "estimator", + "confidence", + "method" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 1024, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 16384, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "protocolSHA256", + "contractSHA256", + "sourceSessionID", + "subject", + "metric", + "protocol", + "observations", + "statistics", + "status", + "failures", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.replication.receipt({\n ...\n})" + } + ] + } + }, + "/harness/meta/selection": { + "post": { + "operationId": "harness.meta.selection", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Resolve the terminal meta-harness qualification subject", + "description": "Returns the backend-selected verified winner after search termination, isolated behind the independent meta-harness qualifier capability.", + "responses": { + "200": { + "description": "Content-addressed terminal meta-harness selection", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "meta-harness-selection-v1" + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "searchRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "objective_met", + "no_improvement", + "user_cancelled", + "runtime_error" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateArtifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "optimizationResultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "optimizationEvaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "selectionID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "searchRevision", + "stopReason", + "candidateID", + "candidateArtifact", + "optimizationResultSHA256", + "optimizationEvaluationSHA256", + "selectedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "metaToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "metaToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.meta.selection({\n ...\n})" + } + ] + } + }, + "/harness/meta/receipts": { + "post": { + "operationId": "harness.meta.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record a one-shot continual-harness qualification", + "description": "Freezes the complete refinement lineage, full trace archive, cross-model held-out matrix, activation/adherence diagnostics, and backend-derived promotion verdict.", + "responses": { + "200": { + "description": "Immutable meta-harness qualification receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "meta-harness-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "selection": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "meta-harness-selection-v1" + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "searchRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "objective_met", + "no_improvement", + "user_cancelled", + "runtime_error" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateArtifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "optimizationResultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "optimizationEvaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "selectionID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "searchRevision", + "stopReason", + "candidateID", + "candidateArtifact", + "optimizationResultSHA256", + "optimizationEvaluationSHA256", + "selectedAt" + ], + "additionalProperties": false + }, + "candidateManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protectedManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "archive": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "indexSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contents": { + "type": "string", + "const": "full-source-scores-traces" + }, + "query": { + "type": "string", + "const": "filesystem" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + }, + "entries": { + "minItems": 1, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "string", + "enum": [ + "evaluated", + "unevaluated" + ] + }, + "scoresSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "resultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + } + }, + "required": [ + "candidateID", + "artifactSHA256", + "sourceSHA256", + "state" + ], + "additionalProperties": false + } + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "indexSHA256", + "contents", + "query", + "complete", + "hiddenContent", + "evaluatorContent", + "entries" + ], + "additionalProperties": false + }, + "refinements": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "scope": { + "type": "string", + "const": "session" + }, + "parentSnapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "snapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trigger": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "diagnosis": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "implementation", + "fundamental", + "inconclusive" + ] + }, + "rationale": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + } + }, + "required": [ + "kind", + "rationale" + ], + "additionalProperties": false + }, + "rootCause": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "expectedOutcome": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "changes": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "create", + "update", + "delete", + "rollback" + ] + }, + "component": { + "type": "string", + "enum": [ + "prompt", + "memory", + "skill", + "tool", + "middleware", + "subagent", + "scaffold" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "beforeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "afterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + } + }, + "required": [ + "action", + "component", + "path", + "reason" + ], + "additionalProperties": false + } + }, + "evidence": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "messageIndex": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "excerptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "candidateID", + "traceSHA256", + "messageIndex", + "excerptSHA256" + ], + "additionalProperties": false + } + }, + "predictions": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "expected": { + "type": "string", + "enum": [ + "fail_to_pass", + "remain_pass" + ] + } + }, + "required": [ + "modelID", + "taskID", + "expected" + ], + "additionalProperties": false + } + } + }, + "required": [ + "revision", + "scope", + "parentSnapshotSHA256", + "snapshotSHA256", + "trigger", + "diagnosis", + "rootCause", + "expectedOutcome", + "changes", + "evidence", + "predictions" + ], + "additionalProperties": false + } + }, + "cells": { + "minItems": 4, + "maxItems": 65536, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "search", + "held_out" + ] + }, + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "modelCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "taskCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "passed": { + "type": "boolean" + }, + "contextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "role": { + "type": "string", + "const": "baseline" + } + }, + "required": [ + "split", + "modelID", + "modelCommitment", + "taskID", + "taskCommitment", + "outcome", + "contextTokens", + "outputSHA256", + "trace", + "evidence", + "role" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "search", + "held_out" + ] + }, + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "modelCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "taskCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "passed": { + "type": "boolean" + }, + "contextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "role": { + "type": "string", + "const": "candidate" + }, + "loaded": { + "type": "boolean" + }, + "phases": { + "maxItems": 4, + "type": "array", + "items": { + "type": "object", + "properties": { + "followed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violatedCommission": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violatedOmission": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "requiredUnobserved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "notApplicable": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "insufficientEvidence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "phase": { + "type": "string", + "enum": [ + "loaded", + "midpoint", + "pre_final", + "final_validation" + ] + } + }, + "required": [ + "followed", + "violatedCommission", + "violatedOmission", + "requiredUnobserved", + "notApplicable", + "insufficientEvidence", + "phase" + ], + "additionalProperties": false + } + } + }, + "required": [ + "split", + "modelID", + "modelCommitment", + "taskID", + "taskCommitment", + "outcome", + "contextTokens", + "outputSHA256", + "trace", + "evidence", + "role", + "loaded", + "phases" + ], + "additionalProperties": false + } + ] + } + }, + "diagnostics": { + "type": "object", + "properties": { + "updaterGain": { + "type": "number" + }, + "beneficiaryGain": { + "type": "number" + }, + "worstHeldoutModelGain": { + "type": "number" + }, + "activationRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "requiredAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "finalAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxPhaseDrift": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "predictionPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "riskRegressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "maxContextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanContextIncrease": { + "type": "number" + }, + "loadedBenefit": { + "type": "number" + }, + "searchPairs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "heldoutPairs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "activationRate", + "predictionPrecision", + "riskRegressions", + "maxContextTokens", + "meanContextIncrease", + "searchPairs", + "heldoutPairs" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 1024, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "selection", + "candidateManifestSHA256", + "protectedManifestSHA256", + "validatorSHA256", + "archive", + "refinements", + "cells", + "diagnostics", + "status", + "failures", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "metaToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protectedManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "archive": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "indexSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contents": { + "type": "string", + "const": "full-source-scores-traces" + }, + "query": { + "type": "string", + "const": "filesystem" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + }, + "entries": { + "minItems": 1, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "string", + "enum": [ + "evaluated", + "unevaluated" + ] + }, + "scoresSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "resultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + } + }, + "required": [ + "candidateID", + "artifactSHA256", + "sourceSHA256", + "state" + ], + "additionalProperties": false + } + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "indexSHA256", + "contents", + "query", + "complete", + "hiddenContent", + "evaluatorContent", + "entries" + ], + "additionalProperties": false + }, + "refinements": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "scope": { + "type": "string", + "const": "session" + }, + "parentSnapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "snapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trigger": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "diagnosis": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "implementation", + "fundamental", + "inconclusive" + ] + }, + "rationale": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + } + }, + "required": [ + "kind", + "rationale" + ], + "additionalProperties": false + }, + "rootCause": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "expectedOutcome": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "changes": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "create", + "update", + "delete", + "rollback" + ] + }, + "component": { + "type": "string", + "enum": [ + "prompt", + "memory", + "skill", + "tool", + "middleware", + "subagent", + "scaffold" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "beforeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "afterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + } + }, + "required": [ + "action", + "component", + "path", + "reason" + ], + "additionalProperties": false + } + }, + "evidence": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "messageIndex": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "excerptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "candidateID", + "traceSHA256", + "messageIndex", + "excerptSHA256" + ], + "additionalProperties": false + } + }, + "predictions": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "expected": { + "type": "string", + "enum": [ + "fail_to_pass", + "remain_pass" + ] + } + }, + "required": [ + "modelID", + "taskID", + "expected" + ], + "additionalProperties": false + } + } + }, + "required": [ + "revision", + "scope", + "parentSnapshotSHA256", + "snapshotSHA256", + "trigger", + "diagnosis", + "rootCause", + "expectedOutcome", + "changes", + "evidence", + "predictions" + ], + "additionalProperties": false + } + }, + "cells": { + "minItems": 4, + "maxItems": 65536, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "search", + "held_out" + ] + }, + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "modelCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "taskCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "passed": { + "type": "boolean" + }, + "contextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "role": { + "type": "string", + "const": "baseline" + } + }, + "required": [ + "split", + "modelID", + "modelCommitment", + "taskID", + "taskCommitment", + "outcome", + "contextTokens", + "outputSHA256", + "trace", + "evidence", + "role" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "search", + "held_out" + ] + }, + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "modelCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "taskCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "passed": { + "type": "boolean" + }, + "contextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "role": { + "type": "string", + "const": "candidate" + }, + "loaded": { + "type": "boolean" + }, + "phases": { + "maxItems": 4, + "type": "array", + "items": { + "type": "object", + "properties": { + "followed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violatedCommission": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violatedOmission": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "requiredUnobserved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "notApplicable": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "insufficientEvidence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "phase": { + "type": "string", + "enum": [ + "loaded", + "midpoint", + "pre_final", + "final_validation" + ] + } + }, + "required": [ + "followed", + "violatedCommission", + "violatedOmission", + "requiredUnobserved", + "notApplicable", + "insufficientEvidence", + "phase" + ], + "additionalProperties": false + } + } + }, + "required": [ + "split", + "modelID", + "modelCommitment", + "taskID", + "taskCommitment", + "outcome", + "contextTokens", + "outputSHA256", + "trace", + "evidence", + "role", + "loaded", + "phases" + ], + "additionalProperties": false + } + ] + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "sessionID", + "metaToken", + "selectionID", + "candidateArtifactSHA256", + "candidateManifestSHA256", + "protectedManifestSHA256", + "validatorSHA256", + "archive", + "refinements", + "cells", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.meta.record({\n ...\n})" + } + ] + } + }, + "/harness/meta/receipts/{receiptID}": { + "post": { + "operationId": "harness.meta.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected meta-harness receipt", + "responses": { + "200": { + "description": "Canonical meta-harness qualification receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "meta-harness-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "selection": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "meta-harness-selection-v1" + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "searchRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "objective_met", + "no_improvement", + "user_cancelled", + "runtime_error" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateArtifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "optimizationResultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "optimizationEvaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "selectionID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "searchRevision", + "stopReason", + "candidateID", + "candidateArtifact", + "optimizationResultSHA256", + "optimizationEvaluationSHA256", + "selectedAt" + ], + "additionalProperties": false + }, + "candidateManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protectedManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "archive": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "indexSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contents": { + "type": "string", + "const": "full-source-scores-traces" + }, + "query": { + "type": "string", + "const": "filesystem" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + }, + "entries": { + "minItems": 1, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "string", + "enum": [ + "evaluated", + "unevaluated" + ] + }, + "scoresSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "resultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + } + }, + "required": [ + "candidateID", + "artifactSHA256", + "sourceSHA256", + "state" + ], + "additionalProperties": false + } + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "indexSHA256", + "contents", + "query", + "complete", + "hiddenContent", + "evaluatorContent", + "entries" + ], + "additionalProperties": false + }, + "refinements": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "scope": { + "type": "string", + "const": "session" + }, + "parentSnapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "snapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trigger": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "diagnosis": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "implementation", + "fundamental", + "inconclusive" + ] + }, + "rationale": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + } + }, + "required": [ + "kind", + "rationale" + ], + "additionalProperties": false + }, + "rootCause": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "expectedOutcome": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "changes": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": [ + "create", + "update", + "delete", + "rollback" + ] + }, + "component": { + "type": "string", + "enum": [ + "prompt", + "memory", + "skill", + "tool", + "middleware", + "subagent", + "scaffold" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "beforeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "afterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reason": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + } + }, + "required": [ + "action", + "component", + "path", + "reason" + ], + "additionalProperties": false + } + }, + "evidence": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "messageIndex": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "excerptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "candidateID", + "traceSHA256", + "messageIndex", + "excerptSHA256" + ], + "additionalProperties": false + } + }, + "predictions": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "expected": { + "type": "string", + "enum": [ + "fail_to_pass", + "remain_pass" + ] + } + }, + "required": [ + "modelID", + "taskID", + "expected" + ], + "additionalProperties": false + } + } + }, + "required": [ + "revision", + "scope", + "parentSnapshotSHA256", + "snapshotSHA256", + "trigger", + "diagnosis", + "rootCause", + "expectedOutcome", + "changes", + "evidence", + "predictions" + ], + "additionalProperties": false + } + }, + "cells": { + "minItems": 4, + "maxItems": 65536, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "search", + "held_out" + ] + }, + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "modelCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "taskCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "passed": { + "type": "boolean" + }, + "contextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "role": { + "type": "string", + "const": "baseline" + } + }, + "required": [ + "split", + "modelID", + "modelCommitment", + "taskID", + "taskCommitment", + "outcome", + "contextTokens", + "outputSHA256", + "trace", + "evidence", + "role" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "search", + "held_out" + ] + }, + "modelID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "modelCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "taskCommitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "passed": { + "type": "boolean" + }, + "contextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean", + "const": true + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "uri", + "sha256", + "schemaSHA256", + "complete", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "role": { + "type": "string", + "const": "candidate" + }, + "loaded": { + "type": "boolean" + }, + "phases": { + "maxItems": 4, + "type": "array", + "items": { + "type": "object", + "properties": { + "followed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violatedCommission": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violatedOmission": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "requiredUnobserved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "notApplicable": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "insufficientEvidence": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "phase": { + "type": "string", + "enum": [ + "loaded", + "midpoint", + "pre_final", + "final_validation" + ] + } + }, + "required": [ + "followed", + "violatedCommission", + "violatedOmission", + "requiredUnobserved", + "notApplicable", + "insufficientEvidence", + "phase" + ], + "additionalProperties": false + } + } + }, + "required": [ + "split", + "modelID", + "modelCommitment", + "taskID", + "taskCommitment", + "outcome", + "contextTokens", + "outputSHA256", + "trace", + "evidence", + "role", + "loaded", + "phases" + ], + "additionalProperties": false + } + ] + } + }, + "diagnostics": { + "type": "object", + "properties": { + "updaterGain": { + "type": "number" + }, + "beneficiaryGain": { + "type": "number" + }, + "worstHeldoutModelGain": { + "type": "number" + }, + "activationRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "requiredAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "finalAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxPhaseDrift": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "predictionPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "riskRegressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "maxContextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanContextIncrease": { + "type": "number" + }, + "loadedBenefit": { + "type": "number" + }, + "searchPairs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "heldoutPairs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "activationRate", + "predictionPrecision", + "riskRegressions", + "maxContextTokens", + "meanContextIncrease", + "searchPairs", + "heldoutPairs" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 1024, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "selection", + "candidateManifestSHA256", + "protectedManifestSHA256", + "validatorSHA256", + "archive", + "refinements", + "cells", + "diagnostics", + "status", + "failures", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "metaToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "metaToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.meta.receipt({\n ...\n})" + } + ] + } + }, + "/harness/confirmations/selection": { + "post": { + "operationId": "harness.confirmation.selection", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Resolve the sealed post-search confirmation subject", + "description": "Returns exactly one backend-selected verified winner only after adaptive search is terminal. The endpoint is isolated behind the claim evaluator capability.", + "responses": { + "200": { + "description": "Immutable terminal winner selection", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "terminal-verified-best-selection-v1" + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "searchRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "objective_met", + "no_improvement", + "user_cancelled", + "runtime_error" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateArtifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "candidateCreatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "optimizationResultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "optimizationEvaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "selectionID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "searchRevision", + "stopReason", + "candidateID", + "candidateArtifact", + "candidateCreatedAt", + "optimizationResultSHA256", + "optimizationEvaluationSHA256", + "selectedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "confirmationToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "confirmationToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.confirmation.selection({\n ...\n})" + } + ] + } + }, + "/harness/confirmations/receipts": { + "post": { + "operationId": "harness.confirmation.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record a one-shot sealed claim evaluation", + "description": "Freezes the claim result for the server-selected terminal winner without feeding the result into search, adaptive control, hindsight memory, or skill learning.", + "responses": { + "200": { + "description": "Immutable sealed confirmation receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "sealed-confirmation-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "selection": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "terminal-verified-best-selection-v1" + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "searchRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "objective_met", + "no_improvement", + "user_cancelled", + "runtime_error" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateArtifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "candidateCreatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "optimizationResultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "optimizationEvaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "selectionID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "searchRevision", + "stopReason", + "candidateID", + "candidateArtifact", + "candidateCreatedAt", + "optimizationResultSHA256", + "optimizationEvaluationSHA256", + "selectedAt" + ], + "additionalProperties": false + }, + "claim": { + "type": "object", + "properties": { + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "format": "uri" + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "repository", + "revision" + ], + "additionalProperties": false + }, + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + }, + "target": { + "type": "number" + } + }, + "required": [ + "taskID", + "split", + "manifestSHA256", + "validatorSHA256", + "environmentSHA256", + "evaluator", + "metric", + "direction", + "target" + ], + "additionalProperties": false + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "metrics": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "number" + } + }, + "checks": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "blocking": { + "type": "boolean" + }, + "score": { + "type": "number" + }, + "evidence": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "id", + "status", + "blocking" + ], + "additionalProperties": false + } + }, + "failures": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 4224, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "usage": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "number", + "minimum": 0 + }, + "costUSD": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "selection", + "claim", + "outcome", + "status", + "metrics", + "checks", + "failures", + "evidence", + "outputSHA256", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "confirmationToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "candidateSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "metrics": { + "default": {}, + "type": "object", + "propertyNames": { + "type": "string", + "maxLength": 200 + }, + "additionalProperties": { + "type": "number" + } + }, + "checks": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "blocking": { + "type": "boolean" + }, + "score": { + "type": "number" + }, + "evidence": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "id", + "status", + "blocking" + ], + "additionalProperties": false + } + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "usage": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "number", + "minimum": 0 + }, + "costUSD": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "sessionID", + "confirmationToken", + "candidateSHA256", + "manifestSHA256", + "validatorSHA256", + "environmentSHA256", + "outcome", + "checks", + "evidence", + "outputSHA256", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.confirmation.record({\n ...\n})" + } + ] + } + }, + "/harness/confirmations/receipts/{receiptID}": { + "post": { + "operationId": "harness.confirmation.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected sealed confirmation receipt", + "responses": { + "200": { + "description": "Canonical sealed confirmation receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "sealed-confirmation-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "selection": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "terminal-verified-best-selection-v1" + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "searchRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "objective_met", + "no_improvement", + "user_cancelled", + "runtime_error" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "candidateArtifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "candidateCreatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "optimizationResultSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "optimizationEvaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "selectionID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "searchRevision", + "stopReason", + "candidateID", + "candidateArtifact", + "candidateCreatedAt", + "optimizationResultSHA256", + "optimizationEvaluationSHA256", + "selectedAt" + ], + "additionalProperties": false + }, + "claim": { + "type": "object", + "properties": { + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "format": "uri" + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "repository", + "revision" + ], + "additionalProperties": false + }, + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + }, + "target": { + "type": "number" + } + }, + "required": [ + "taskID", + "split", + "manifestSHA256", + "validatorSHA256", + "environmentSHA256", + "evaluator", + "metric", + "direction", + "target" + ], + "additionalProperties": false + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed", + "inconclusive" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "metrics": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "number" + } + }, + "checks": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "blocking": { + "type": "boolean" + }, + "score": { + "type": "number" + }, + "evidence": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "id", + "status", + "blocking" + ], + "additionalProperties": false + } + }, + "failures": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 4224, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "usage": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "number", + "minimum": 0 + }, + "costUSD": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "contractSHA256", + "protocolSHA256", + "sourceSessionID", + "runID", + "selection", + "claim", + "outcome", + "status", + "metrics", + "checks", + "failures", + "evidence", + "outputSHA256", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "confirmationToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "confirmationToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.confirmation.receipt({\n ...\n})" + } + ] + } + }, + "/harness/semantics/receipts": { + "post": { + "operationId": "harness.semantic.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record an independent semantic audit", + "description": "Derives whether one bound result is meaningful, merely technically valid, ambiguous, or incorrect from independent evidence-backed reviews of frozen intent, shortcuts, and literature-relative novelty.", + "responses": { + "200": { + "description": "Immutable semantic audit receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "semantic-audit-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "reviewer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "enum": [ + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "scope": { + "type": "object", + "properties": { + "objectiveSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "requirement": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "requirement" + ], + "additionalProperties": false + } + }, + "forbiddenShortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "description" + ], + "additionalProperties": false + } + }, + "literature": { + "type": "object", + "properties": { + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "corpusSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "cutoff", + "corpusSHA256" + ], + "additionalProperties": false + }, + "noveltyFloor": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + } + }, + "required": [ + "objectiveSHA256", + "criteria", + "forbiddenShortcuts", + "literature", + "noveltyFloor" + ], + "additionalProperties": false + }, + "reviews": { + "minItems": 2, + "maxItems": 5, + "type": "array", + "items": { + "type": "object", + "properties": { + "actor": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "correctness": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "alignment": { + "type": "string", + "enum": [ + "intended", + "reasonable_alternative", + "misinterpreted", + "ambiguous" + ] + }, + "novelty": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + }, + "vacuous": { + "type": "boolean" + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "shortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "observed": { + "type": "boolean" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "observed", + "evidence" + ], + "additionalProperties": false + } + }, + "literatureRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "reviewedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "actor", + "sessionID", + "correctness", + "alignment", + "novelty", + "vacuous", + "confidence", + "criteria", + "shortcuts", + "evidence", + "summary", + "reviewedAt" + ], + "additionalProperties": false + } + }, + "status": { + "type": "string", + "enum": [ + "meaningful", + "technical_only", + "ambiguous", + "failed" + ] + }, + "failures": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "reviewedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "protocolSHA256", + "sourceSessionID", + "subject", + "reviewer", + "scope", + "reviews", + "status", + "failures", + "evidence", + "reviewedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "reviewerToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "reviews": { + "minItems": 2, + "maxItems": 5, + "type": "array", + "items": { + "type": "object", + "properties": { + "actor": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "correctness": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "alignment": { + "type": "string", + "enum": [ + "intended", + "reasonable_alternative", + "misinterpreted", + "ambiguous" + ] + }, + "novelty": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + }, + "vacuous": { + "type": "boolean" + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "shortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "observed": { + "type": "boolean" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "observed", + "evidence" + ], + "additionalProperties": false + } + }, + "literatureRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "reviewedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "actor", + "sessionID", + "correctness", + "alignment", + "novelty", + "vacuous", + "confidence", + "criteria", + "shortcuts", + "evidence", + "summary", + "reviewedAt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "sessionID", + "reviewerToken", + "subject", + "reviews" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.semantic.record({\n ...\n})" + } + ] + } + }, + "/harness/semantics/receipts/{receiptID}": { + "post": { + "operationId": "harness.semantic.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected semantic audit receipt", + "responses": { + "200": { + "description": "Semantic audit receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "semantic-audit-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSessionID": { + "type": "string", + "minLength": 1 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "reviewer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "enum": [ + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "scope": { + "type": "object", + "properties": { + "objectiveSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "requirement": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "requirement" + ], + "additionalProperties": false + } + }, + "forbiddenShortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "description" + ], + "additionalProperties": false + } + }, + "literature": { + "type": "object", + "properties": { + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "corpusSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "cutoff", + "corpusSHA256" + ], + "additionalProperties": false + }, + "noveltyFloor": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + } + }, + "required": [ + "objectiveSHA256", + "criteria", + "forbiddenShortcuts", + "literature", + "noveltyFloor" + ], + "additionalProperties": false + }, + "reviews": { + "minItems": 2, + "maxItems": 5, + "type": "array", + "items": { + "type": "object", + "properties": { + "actor": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "correctness": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "alignment": { + "type": "string", + "enum": [ + "intended", + "reasonable_alternative", + "misinterpreted", + "ambiguous" + ] + }, + "novelty": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + }, + "vacuous": { + "type": "boolean" + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "shortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "observed": { + "type": "boolean" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "observed", + "evidence" + ], + "additionalProperties": false + } + }, + "literatureRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "reviewedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "actor", + "sessionID", + "correctness", + "alignment", + "novelty", + "vacuous", + "confidence", + "criteria", + "shortcuts", + "evidence", + "summary", + "reviewedAt" + ], + "additionalProperties": false + } + }, + "status": { + "type": "string", + "enum": [ + "meaningful", + "technical_only", + "ambiguous", + "failed" + ] + }, + "failures": { + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evidence": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "reviewedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "protocolSHA256", + "sourceSessionID", + "subject", + "reviewer", + "scope", + "reviews", + "status", + "failures", + "evidence", + "reviewedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "reviewerToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "reviewerToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.semantic.receipt({\n ...\n})" + } + ] + } + }, + "/harness/syntheses/receipts": { + "post": { + "operationId": "harness.synthesis.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record an evaluator-authenticated clean-room synthesis", + "description": "Binds a complete retrieval trace and hidden atomic-fact manifest, rejects clean-room policy drift, and derives factual precision, recall, contradiction penalty, and F1 without trusting caller-authored metrics.", + "responses": { + "200": { + "description": "Immutable scientific synthesis receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "scientific-synthesis-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "conclusionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatorAuditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "events": { + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "tool": { + "type": "string", + "enum": [ + "google_search", + "paper_search", + "web_browse" + ] + }, + "requestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "responseSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "publishedAt": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "matches": { + "type": "object", + "properties": { + "forbiddenDomain": { + "type": "boolean" + }, + "referenceTitle": { + "type": "boolean" + } + }, + "required": [ + "forbiddenDomain", + "referenceTitle" + ], + "additionalProperties": false + }, + "decision": { + "type": "string", + "enum": [ + "allowed", + "blocked" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "eventID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "violations": { + "maxItems": 5, + "type": "array", + "items": { + "type": "string", + "enum": [ + "forbidden_domain", + "reference_title", + "post_cutoff", + "unknown_date", + "duplicate_output" + ] + } + } + }, + "required": [ + "sequence", + "tool", + "requestSHA256", + "responseSHA256", + "sourceSHA256", + "matches", + "decision", + "evidence", + "eventID", + "violations" + ], + "additionalProperties": false + } + }, + "decomposition": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "evidence" + ], + "additionalProperties": false + }, + "generatedFacts": { + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verdict": { + "type": "string", + "enum": [ + "supported", + "contradicted", + "unsupported", + "judge_error" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "verdict", + "evidence" + ], + "additionalProperties": false + } + }, + "referenceFacts": { + "minItems": 1, + "maxItems": 2048, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "coverage": { + "type": "string", + "enum": [ + "covered", + "missed", + "judge_error" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "coverage", + "evidence" + ], + "additionalProperties": false + } + }, + "metrics": { + "type": "object", + "properties": { + "toolEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "allowedSources": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "blockedSources": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violations": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "forbidden_domain", + "reference_title", + "post_cutoff", + "unknown_date", + "duplicate_output" + ] + }, + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "generatedFacts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "supported": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "contradicted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "unsupported": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "precisionJudgeErrors": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "referenceFacts": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "covered": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "missed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "recallJudgeErrors": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "precision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "recall": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "f1": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "toolEvents", + "allowedSources", + "blockedSources", + "violations", + "generatedFacts", + "supported", + "contradicted", + "unsupported", + "precisionJudgeErrors", + "referenceFacts", + "covered", + "missed", + "recallJudgeErrors" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "protocolSHA256", + "subject", + "conclusionSHA256", + "evaluatorAuditReceiptID", + "traceSHA256", + "events", + "decomposition", + "generatedFacts", + "referenceFacts", + "metrics", + "status", + "failures", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "conclusionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatorAuditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "owner": { + "type": "string", + "const": "evaluator_runtime" + }, + "complete": { + "type": "boolean", + "const": true + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "filterPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "events": { + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "tool": { + "type": "string", + "enum": [ + "google_search", + "paper_search", + "web_browse" + ] + }, + "requestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "responseSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "publishedAt": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "matches": { + "type": "object", + "properties": { + "forbiddenDomain": { + "type": "boolean" + }, + "referenceTitle": { + "type": "boolean" + } + }, + "required": [ + "forbiddenDomain", + "referenceTitle" + ], + "additionalProperties": false + }, + "decision": { + "type": "string", + "enum": [ + "allowed", + "blocked" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "sequence", + "tool", + "requestSHA256", + "responseSHA256", + "sourceSHA256", + "matches", + "decision", + "evidence" + ], + "additionalProperties": false + } + } + }, + "required": [ + "owner", + "complete", + "schemaSHA256", + "filterPolicySHA256", + "events" + ], + "additionalProperties": false + }, + "decomposition": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "evidence" + ], + "additionalProperties": false + }, + "generatedFacts": { + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verdict": { + "type": "string", + "enum": [ + "supported", + "contradicted", + "unsupported", + "judge_error" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "verdict", + "evidence" + ], + "additionalProperties": false + } + }, + "referenceFacts": { + "minItems": 1, + "maxItems": 2048, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "coverage": { + "type": "string", + "enum": [ + "covered", + "missed", + "judge_error" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "coverage", + "evidence" + ], + "additionalProperties": false + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "subject", + "conclusionSHA256", + "evaluatorAuditReceiptID", + "trace", + "decomposition", + "generatedFacts", + "referenceFacts", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.synthesis.record({\n ...\n})" + } + ] + } + }, + "/harness/syntheses/receipts/{receiptID}": { + "post": { + "operationId": "harness.synthesis.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected scientific synthesis receipt", + "responses": { + "200": { + "description": "Canonical scientific synthesis receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "scientific-synthesis-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "conclusionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatorAuditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "events": { + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "tool": { + "type": "string", + "enum": [ + "google_search", + "paper_search", + "web_browse" + ] + }, + "requestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "responseSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "publishedAt": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "matches": { + "type": "object", + "properties": { + "forbiddenDomain": { + "type": "boolean" + }, + "referenceTitle": { + "type": "boolean" + } + }, + "required": [ + "forbiddenDomain", + "referenceTitle" + ], + "additionalProperties": false + }, + "decision": { + "type": "string", + "enum": [ + "allowed", + "blocked" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "eventID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "violations": { + "maxItems": 5, + "type": "array", + "items": { + "type": "string", + "enum": [ + "forbidden_domain", + "reference_title", + "post_cutoff", + "unknown_date", + "duplicate_output" + ] + } + } + }, + "required": [ + "sequence", + "tool", + "requestSHA256", + "responseSHA256", + "sourceSHA256", + "matches", + "decision", + "evidence", + "eventID", + "violations" + ], + "additionalProperties": false + } + }, + "decomposition": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "outputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "status", + "evidence" + ], + "additionalProperties": false + }, + "generatedFacts": { + "maxItems": 512, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verdict": { + "type": "string", + "enum": [ + "supported", + "contradicted", + "unsupported", + "judge_error" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "verdict", + "evidence" + ], + "additionalProperties": false + } + }, + "referenceFacts": { + "minItems": 1, + "maxItems": 2048, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,239}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "coverage": { + "type": "string", + "enum": [ + "covered", + "missed", + "judge_error" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "commitment", + "coverage", + "evidence" + ], + "additionalProperties": false + } + }, + "metrics": { + "type": "object", + "properties": { + "toolEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "allowedSources": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "blockedSources": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "violations": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "forbidden_domain", + "reference_title", + "post_cutoff", + "unknown_date", + "duplicate_output" + ] + }, + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "generatedFacts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "supported": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "contradicted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "unsupported": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "precisionJudgeErrors": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "referenceFacts": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "covered": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "missed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "recallJudgeErrors": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "precision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "recall": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "f1": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "toolEvents", + "allowedSources", + "blockedSources", + "violations", + "generatedFacts", + "supported", + "contradicted", + "unsupported", + "precisionJudgeErrors", + "referenceFacts", + "covered", + "missed", + "recallJudgeErrors" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "protocolSHA256", + "subject", + "conclusionSHA256", + "evaluatorAuditReceiptID", + "traceSHA256", + "events", + "decomposition", + "generatedFacts", + "referenceFacts", + "metrics", + "status", + "failures", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.synthesis.receipt({\n ...\n})" + } + ] + } + }, + "/harness/autonomy/receipts": { + "post": { + "operationId": "harness.autonomy.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record an evaluator-authenticated human-AI autonomy trace", + "description": "Binds a complete interaction log to the exact run or candidate artifact and derives the Aletheia-inspired essentially-autonomous, collaborative, or primarily-human contribution level without trusting the caller's claim.", + "responses": { + "200": { + "description": "Immutable backend-derived human-AI autonomy receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "human-ai-autonomy-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "recorderArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "rawLogSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "events": { + "minItems": 2, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "at": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "actor": { + "type": "string", + "enum": [ + "benchmark", + "human", + "agent" + ] + }, + "kind": { + "type": "string", + "enum": [ + "problem_statement", + "clarification", + "resource_provision", + "strategy", + "technical_correction", + "artifact_edit", + "candidate_selection", + "evaluation_feedback", + "exposition", + "other" + ] + }, + "contribution": { + "type": "string", + "enum": [ + "problem", + "auxiliary", + "essential", + "core", + "unclear" + ] + }, + "contentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactBeforeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactAfterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "priorEventID": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "eventID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sequence", + "at", + "actor", + "kind", + "contribution", + "contentSHA256", + "evidence", + "priorEventID", + "eventID" + ], + "additionalProperties": false + } + }, + "claimedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "derivedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "metrics": { + "type": "object", + "properties": { + "events": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "counts": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "benchmark", + "human", + "agent" + ] + }, + "additionalProperties": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "problem", + "auxiliary", + "essential", + "core", + "unclear" + ] + }, + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "problemEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "humanSubstantiveEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "agentSubstantiveEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "unclearEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "linkedArtifactEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "artifactTransitions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "finalArtifactLinked": { + "type": "boolean" + } + }, + "required": [ + "events", + "counts", + "problemEvents", + "humanSubstantiveEvents", + "agentSubstantiveEvents", + "unclearEvents", + "linkedArtifactEvents", + "artifactTransitions", + "finalArtifactLinked" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "protocolSHA256", + "subject", + "artifactSHA256", + "traceSHA256", + "recorderArtifactSHA256", + "rawLogSHA256", + "startedAt", + "endedAt", + "events", + "claimedLevel", + "metrics", + "status", + "failures", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "trace": { + "type": "object", + "properties": { + "owner": { + "type": "string", + "const": "evaluator_runtime" + }, + "complete": { + "type": "boolean", + "const": true + }, + "recorderArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "classificationPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "rawLogSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "events": { + "minItems": 2, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "at": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "actor": { + "type": "string", + "enum": [ + "benchmark", + "human", + "agent" + ] + }, + "kind": { + "type": "string", + "enum": [ + "problem_statement", + "clarification", + "resource_provision", + "strategy", + "technical_correction", + "artifact_edit", + "candidate_selection", + "evaluation_feedback", + "exposition", + "other" + ] + }, + "contribution": { + "type": "string", + "enum": [ + "problem", + "auxiliary", + "essential", + "core", + "unclear" + ] + }, + "contentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactBeforeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactAfterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "sequence", + "at", + "actor", + "kind", + "contribution", + "contentSHA256", + "evidence" + ], + "additionalProperties": false + } + } + }, + "required": [ + "owner", + "complete", + "recorderArtifactSHA256", + "schemaSHA256", + "classificationPolicySHA256", + "rawLogSHA256", + "startedAt", + "endedAt", + "events" + ], + "additionalProperties": false + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "subject", + "artifactSHA256", + "trace" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.autonomy.record({\n ...\n})" + } + ] + } + }, + "/harness/autonomy/receipts/{receiptID}": { + "post": { + "operationId": "harness.autonomy.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected human-AI autonomy receipt", + "responses": { + "200": { + "description": "Canonical human-AI autonomy receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "human-ai-autonomy-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "recorderArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "rawLogSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "events": { + "minItems": 2, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "sequence": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "at": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "actor": { + "type": "string", + "enum": [ + "benchmark", + "human", + "agent" + ] + }, + "kind": { + "type": "string", + "enum": [ + "problem_statement", + "clarification", + "resource_provision", + "strategy", + "technical_correction", + "artifact_edit", + "candidate_selection", + "evaluation_feedback", + "exposition", + "other" + ] + }, + "contribution": { + "type": "string", + "enum": [ + "problem", + "auxiliary", + "essential", + "core", + "unclear" + ] + }, + "contentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactBeforeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactAfterSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "priorEventID": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + { + "type": "null" + } + ] + }, + "eventID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sequence", + "at", + "actor", + "kind", + "contribution", + "contentSHA256", + "evidence", + "priorEventID", + "eventID" + ], + "additionalProperties": false + } + }, + "claimedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "derivedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "metrics": { + "type": "object", + "properties": { + "events": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "counts": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "benchmark", + "human", + "agent" + ] + }, + "additionalProperties": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "problem", + "auxiliary", + "essential", + "core", + "unclear" + ] + }, + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + } + }, + "problemEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "humanSubstantiveEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "agentSubstantiveEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "unclearEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "linkedArtifactEvents": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "artifactTransitions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "finalArtifactLinked": { + "type": "boolean" + } + }, + "required": [ + "events", + "counts", + "problemEvents", + "humanSubstantiveEvents", + "agentSubstantiveEvents", + "unclearEvents", + "linkedArtifactEvents", + "artifactTransitions", + "finalArtifactLinked" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "failures": { + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "protocolSHA256", + "subject", + "artifactSHA256", + "traceSHA256", + "recorderArtifactSHA256", + "rawLogSHA256", + "startedAt", + "endedAt", + "events", + "claimedLevel", + "metrics", + "status", + "failures", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.autonomy.receipt({\n ...\n})" + } + ] + } + }, + "/harness/proofs/blueprints": { + "post": { + "operationId": "harness.blueprint.initialize", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Initialize an evaluator-grounded formal proof blueprint", + "description": "Creates the content-addressed root of a bounded LEAP-inspired AND/OR proof graph without granting the graph final proof authority.", + "responses": { + "200": { + "description": "Canonical proof blueprint view", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "proof-blueprint-view-v1" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "rootGoalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "summary": { + "type": "object", + "properties": { + "blueprintID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "goals": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "proved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refuted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "exhausted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "decompositions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rejected": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refinements": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "openLeases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "blueprintID", + "status", + "goals", + "proved", + "refuted", + "exhausted", + "decompositions", + "attempts", + "rejected", + "refinements", + "openLeases", + "revision" + ], + "additionalProperties": false + }, + "goals": { + "type": "array", + "items": { + "type": "object", + "properties": { + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "ready": { + "type": "boolean" + } + }, + "required": [ + "statementSHA256", + "declaration", + "module", + "id", + "createdAt", + "status", + "ready" + ], + "additionalProperties": false + } + }, + "decompositions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "parentID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "childIDs": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "informalPlanSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "closed", + "blocked" + ] + } + }, + "required": [ + "id", + "parentID", + "childIDs", + "informalPlanSHA256", + "sketchArtifactSHA256", + "sketchTranscriptSHA256", + "reviewerTranscriptSHA256", + "attemptID", + "createdAt", + "status" + ], + "additionalProperties": false + } + }, + "attempts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "leaseID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "type": "string", + "enum": [ + "direct", + "decomposition" + ] + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "result": { + "type": "string", + "enum": [ + "proved", + "refuted", + "failed", + "accepted", + "rejected" + ] + }, + "claim": { + "type": "string", + "enum": [ + "proof", + "refutation", + "failure" + ] + }, + "decompositionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "feedbackSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "failures": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "ordinal", + "goalID", + "leaseID", + "kind", + "artifactSHA256", + "result", + "transcriptSHA256", + "feedbackSHA256", + "failures", + "startedAt", + "endedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "leases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "consumed", + "expired" + ] + }, + "issuedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "expiresAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "consumedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "goalID", + "revision", + "ordinal", + "status", + "issuedAt", + "expiresAt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "runID", + "sessionID", + "rootGoalID", + "summary", + "goals", + "decompositions", + "attempts", + "leases" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.blueprint.initialize({\n ...\n})" + } + ] + } + }, + "/harness/proofs/blueprints/status": { + "post": { + "operationId": "harness.blueprint.status", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Read an evaluator-grounded formal proof blueprint", + "responses": { + "200": { + "description": "Backend-derived goal, decomposition, attempt, and lease state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "proof-blueprint-view-v1" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "rootGoalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "summary": { + "type": "object", + "properties": { + "blueprintID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "goals": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "proved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refuted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "exhausted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "decompositions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rejected": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refinements": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "openLeases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "blueprintID", + "status", + "goals", + "proved", + "refuted", + "exhausted", + "decompositions", + "attempts", + "rejected", + "refinements", + "openLeases", + "revision" + ], + "additionalProperties": false + }, + "goals": { + "type": "array", + "items": { + "type": "object", + "properties": { + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "ready": { + "type": "boolean" + } + }, + "required": [ + "statementSHA256", + "declaration", + "module", + "id", + "createdAt", + "status", + "ready" + ], + "additionalProperties": false + } + }, + "decompositions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "parentID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "childIDs": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "informalPlanSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "closed", + "blocked" + ] + } + }, + "required": [ + "id", + "parentID", + "childIDs", + "informalPlanSHA256", + "sketchArtifactSHA256", + "sketchTranscriptSHA256", + "reviewerTranscriptSHA256", + "attemptID", + "createdAt", + "status" + ], + "additionalProperties": false + } + }, + "attempts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "leaseID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "type": "string", + "enum": [ + "direct", + "decomposition" + ] + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "result": { + "type": "string", + "enum": [ + "proved", + "refuted", + "failed", + "accepted", + "rejected" + ] + }, + "claim": { + "type": "string", + "enum": [ + "proof", + "refutation", + "failure" + ] + }, + "decompositionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "feedbackSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "failures": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "ordinal", + "goalID", + "leaseID", + "kind", + "artifactSHA256", + "result", + "transcriptSHA256", + "feedbackSHA256", + "failures", + "startedAt", + "endedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "leases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "consumed", + "expired" + ] + }, + "issuedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "expiresAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "consumedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "goalID", + "revision", + "ordinal", + "status", + "issuedAt", + "expiresAt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "runID", + "sessionID", + "rootGoalID", + "summary", + "goals", + "decompositions", + "attempts", + "leases" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.blueprint.status({\n ...\n})" + } + ] + } + }, + "/harness/proofs/blueprints/leases": { + "post": { + "operationId": "harness.blueprint.lease", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Lease bounded ready goals from a formal proof blueprint", + "description": "Atomically expires stale work and leases distinct deepest-ready goals up to the frozen parallelism limit.", + "responses": { + "200": { + "description": "Issued leases and updated proof blueprint", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "leases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "consumed", + "expired" + ] + }, + "issuedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "expiresAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "consumedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "goalID", + "revision", + "ordinal", + "status", + "issuedAt", + "expiresAt" + ], + "additionalProperties": false + } + }, + "state": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "proof-blueprint-view-v1" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "rootGoalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "summary": { + "type": "object", + "properties": { + "blueprintID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "goals": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "proved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refuted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "exhausted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "decompositions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rejected": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refinements": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "openLeases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "blueprintID", + "status", + "goals", + "proved", + "refuted", + "exhausted", + "decompositions", + "attempts", + "rejected", + "refinements", + "openLeases", + "revision" + ], + "additionalProperties": false + }, + "goals": { + "type": "array", + "items": { + "type": "object", + "properties": { + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "ready": { + "type": "boolean" + } + }, + "required": [ + "statementSHA256", + "declaration", + "module", + "id", + "createdAt", + "status", + "ready" + ], + "additionalProperties": false + } + }, + "decompositions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "parentID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "childIDs": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "informalPlanSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "closed", + "blocked" + ] + } + }, + "required": [ + "id", + "parentID", + "childIDs", + "informalPlanSHA256", + "sketchArtifactSHA256", + "sketchTranscriptSHA256", + "reviewerTranscriptSHA256", + "attemptID", + "createdAt", + "status" + ], + "additionalProperties": false + } + }, + "attempts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "leaseID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "type": "string", + "enum": [ + "direct", + "decomposition" + ] + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "result": { + "type": "string", + "enum": [ + "proved", + "refuted", + "failed", + "accepted", + "rejected" + ] + }, + "claim": { + "type": "string", + "enum": [ + "proof", + "refutation", + "failure" + ] + }, + "decompositionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "feedbackSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "failures": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "ordinal", + "goalID", + "leaseID", + "kind", + "artifactSHA256", + "result", + "transcriptSHA256", + "feedbackSHA256", + "failures", + "startedAt", + "endedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "leases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "consumed", + "expired" + ] + }, + "issuedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "expiresAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "consumedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "goalID", + "revision", + "ordinal", + "status", + "issuedAt", + "expiresAt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "runID", + "sessionID", + "rootGoalID", + "summary", + "goals", + "decompositions", + "attempts", + "leases" + ], + "additionalProperties": false + } + }, + "required": [ + "leases", + "state" + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "count": { + "type": "integer", + "minimum": 1, + "maximum": 32 + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "count" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.blueprint.lease({\n ...\n})" + } + ] + } + }, + "/harness/proofs/blueprints/attempts": { + "post": { + "operationId": "harness.blueprint.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record an evaluator-authenticated proof or decomposition attempt", + "description": "Consumes one active goal lease, retains failed verifier or reviewer outcomes, and admits only exact compiler-checked sketches into the monotone acyclic graph.", + "responses": { + "200": { + "description": "Recorded attempt and updated proof blueprint", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "decompositionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "state": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "proof-blueprint-view-v1" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "rootGoalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "summary": { + "type": "object", + "properties": { + "blueprintID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "goals": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "proved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refuted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "exhausted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "decompositions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rejected": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refinements": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "openLeases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "blueprintID", + "status", + "goals", + "proved", + "refuted", + "exhausted", + "decompositions", + "attempts", + "rejected", + "refinements", + "openLeases", + "revision" + ], + "additionalProperties": false + }, + "goals": { + "type": "array", + "items": { + "type": "object", + "properties": { + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "ready": { + "type": "boolean" + } + }, + "required": [ + "statementSHA256", + "declaration", + "module", + "id", + "createdAt", + "status", + "ready" + ], + "additionalProperties": false + } + }, + "decompositions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "parentID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "childIDs": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "informalPlanSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "attemptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "closed", + "blocked" + ] + } + }, + "required": [ + "id", + "parentID", + "childIDs", + "informalPlanSHA256", + "sketchArtifactSHA256", + "sketchTranscriptSHA256", + "reviewerTranscriptSHA256", + "attemptID", + "createdAt", + "status" + ], + "additionalProperties": false + } + }, + "attempts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "leaseID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "kind": { + "type": "string", + "enum": [ + "direct", + "decomposition" + ] + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "result": { + "type": "string", + "enum": [ + "proved", + "refuted", + "failed", + "accepted", + "rejected" + ] + }, + "claim": { + "type": "string", + "enum": [ + "proof", + "refutation", + "failure" + ] + }, + "decompositionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "feedbackSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "failures": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "ordinal", + "goalID", + "leaseID", + "kind", + "artifactSHA256", + "result", + "transcriptSHA256", + "feedbackSHA256", + "failures", + "startedAt", + "endedAt", + "recordedAt" + ], + "additionalProperties": false + } + }, + "leases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "goalID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ordinal": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "status": { + "type": "string", + "enum": [ + "open", + "consumed", + "expired" + ] + }, + "issuedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "expiresAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "consumedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "goalID", + "revision", + "ordinal", + "status", + "issuedAt", + "expiresAt" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "runID", + "sessionID", + "rootGoalID", + "summary", + "goals", + "decompositions", + "attempts", + "leases" + ], + "additionalProperties": false + } + }, + "required": [ + "state" + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "kind": { + "type": "string", + "const": "direct" + }, + "leaseID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "claim": { + "type": "string", + "enum": [ + "proof", + "refutation", + "failure" + ] + }, + "verification": { + "type": "object", + "properties": { + "compilerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementMatched": { + "type": "boolean" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "warnings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "feedbackSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "compilerArtifactSHA256", + "statementMatched", + "exitCode", + "warnings", + "transcriptSHA256", + "feedbackSHA256", + "startedAt", + "endedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "kind", + "leaseID", + "artifactSHA256", + "claim", + "verification" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "kind": { + "type": "string", + "const": "decomposition" + }, + "leaseID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "informalPlanSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "children": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "statementSHA256", + "declaration", + "module" + ], + "additionalProperties": false + } + }, + "verification": { + "type": "object", + "properties": { + "compilerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementMatched": { + "type": "boolean" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "warnings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "feedbackSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "validatorArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "placeholderDeclarations": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "validatorTranscriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "compilerArtifactSHA256", + "statementMatched", + "exitCode", + "warnings", + "transcriptSHA256", + "feedbackSHA256", + "startedAt", + "endedAt", + "validatorArtifactSHA256", + "placeholderDeclarations", + "validatorTranscriptSHA256" + ], + "additionalProperties": false + }, + "review": { + "type": "object", + "properties": { + "reviewerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "relevant": { + "type": "boolean" + }, + "easier": { + "type": "boolean" + }, + "plausible": { + "type": "boolean" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "reviewerArtifactSHA256", + "promptSHA256", + "relevant", + "easier", + "plausible", + "transcriptSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "kind", + "leaseID", + "informalPlanSHA256", + "artifactSHA256", + "children", + "verification", + "review" + ], + "additionalProperties": false + } + ] + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.blueprint.record({\n ...\n})" + } + ] + } + }, + "/harness/proofs/receipts": { + "post": { + "operationId": "harness.formal.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record an evaluator-authenticated formal proof verification", + "description": "Binds a trusted Lean challenge, exact proof artifact, frozen environment, transitive axiom audit, and the contract's kernel, fresh-recheck, or external-crosscheck trust tier.", + "responses": { + "200": { + "description": "Immutable backend-derived formal proof receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "formal-proof-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "relation": { + "type": "string", + "enum": [ + "exact_proof", + "exact_refutation", + "repaired_proof" + ] + }, + "challengeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "environment": { + "type": "object", + "properties": { + "leanVersion": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "leanToolchainSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lakeManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "dependencyTreeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "leanVersion", + "leanToolchainSHA256", + "lakeManifestSHA256", + "dependencyTreeSHA256" + ], + "additionalProperties": false + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "files": { + "minItems": 6, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "role": { + "type": "string", + "enum": [ + "challenge", + "statement", + "proof", + "lean_toolchain", + "lake_manifest", + "dependency_tree", + "config", + "support" + ] + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "path", + "role", + "sha256" + ], + "additionalProperties": false + } + }, + "verification": { + "type": "object", + "properties": { + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "build": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "warnings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "exitCode", + "warnings", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean" + }, + "findings": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "construct": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "construct", + "path", + "line" + ], + "additionalProperties": false + } + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "complete", + "findings", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "axioms": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean" + }, + "typesTraversed": { + "type": "boolean" + }, + "observed": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "complete", + "typesTraversed", + "observed", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "fresh": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fresh": { + "type": "boolean" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "fresh", + "exitCode", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "external": { + "type": "object", + "properties": { + "comparatorArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sandboxImageSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sandboxed": { + "type": "boolean" + }, + "challengeMatched": { + "type": "boolean" + }, + "proofTermSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "checks": { + "minItems": 2, + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": [ + "lean_kernel", + "external_checker" + ] + }, + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "accepted": { + "type": "boolean" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "role", + "verifierArtifactSHA256", + "accepted", + "transcriptSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "comparatorArtifactSHA256", + "sandboxImageSHA256", + "sandboxed", + "challengeMatched", + "proofTermSHA256", + "transcriptSHA256", + "checks" + ], + "additionalProperties": false + } + }, + "required": [ + "startedAt", + "endedAt", + "build", + "source", + "axioms" + ], + "additionalProperties": false + }, + "tier": { + "type": "string", + "enum": [ + "kernel", + "fresh_recheck", + "external_crosscheck" + ] + }, + "metrics": { + "type": "object", + "properties": { + "files": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "warnings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "observedAxioms": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "disallowedAxioms": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "manifestComplete": { + "type": "boolean" + }, + "buildAccepted": { + "type": "boolean" + }, + "sourceAuditAccepted": { + "type": "boolean" + }, + "forbiddenFindings": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "construct": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "construct", + "path", + "line" + ], + "additionalProperties": false + } + }, + "axiomAuditAccepted": { + "type": "boolean" + }, + "freshRecheckAccepted": { + "type": "boolean" + }, + "externalCrosscheckAccepted": { + "type": "boolean" + }, + "statementMatched": { + "type": "boolean" + } + }, + "required": [ + "files", + "warnings", + "observedAxioms", + "disallowedAxioms", + "manifestComplete", + "buildAccepted", + "sourceAuditAccepted", + "forbiddenFindings", + "axiomAuditAccepted", + "freshRecheckAccepted", + "externalCrosscheckAccepted", + "statementMatched" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "failures": { + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "protocolSHA256", + "subject", + "artifactSHA256", + "relation", + "challengeSHA256", + "statementSHA256", + "declaration", + "module", + "environment", + "manifestSHA256", + "files", + "verification", + "tier", + "metrics", + "status", + "failures", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "relation": { + "type": "string", + "enum": [ + "exact_proof", + "exact_refutation", + "repaired_proof" + ] + }, + "challengeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "environment": { + "type": "object", + "properties": { + "leanVersion": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "leanToolchainSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lakeManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "dependencyTreeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "leanVersion", + "leanToolchainSHA256", + "lakeManifestSHA256", + "dependencyTreeSHA256" + ], + "additionalProperties": false + }, + "manifest": { + "type": "object", + "properties": { + "complete": { + "type": "boolean" + }, + "files": { + "minItems": 6, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "role": { + "type": "string", + "enum": [ + "challenge", + "statement", + "proof", + "lean_toolchain", + "lake_manifest", + "dependency_tree", + "config", + "support" + ] + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "path", + "role", + "sha256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "complete", + "files" + ], + "additionalProperties": false + }, + "verification": { + "type": "object", + "properties": { + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "build": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "warnings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "exitCode", + "warnings", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean" + }, + "findings": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "construct": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "construct", + "path", + "line" + ], + "additionalProperties": false + } + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "complete", + "findings", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "axioms": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean" + }, + "typesTraversed": { + "type": "boolean" + }, + "observed": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "complete", + "typesTraversed", + "observed", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "fresh": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fresh": { + "type": "boolean" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "fresh", + "exitCode", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "external": { + "type": "object", + "properties": { + "comparatorArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sandboxImageSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sandboxed": { + "type": "boolean" + }, + "challengeMatched": { + "type": "boolean" + }, + "proofTermSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "checks": { + "minItems": 2, + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": [ + "lean_kernel", + "external_checker" + ] + }, + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "accepted": { + "type": "boolean" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "role", + "verifierArtifactSHA256", + "accepted", + "transcriptSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "comparatorArtifactSHA256", + "sandboxImageSHA256", + "sandboxed", + "challengeMatched", + "proofTermSHA256", + "transcriptSHA256", + "checks" + ], + "additionalProperties": false + } + }, + "required": [ + "startedAt", + "endedAt", + "build", + "source", + "axioms" + ], + "additionalProperties": false + } + }, + "required": [ + "sessionID", + "evaluatorToken", + "subject", + "artifactSHA256", + "relation", + "challengeSHA256", + "statementSHA256", + "declaration", + "module", + "environment", + "manifest", + "verification" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.formal.record({\n ...\n})" + } + ] + } + }, + "/harness/proofs/receipts/{receiptID}": { + "post": { + "operationId": "harness.formal.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected formal proof receipt", + "responses": { + "200": { + "description": "Canonical formal proof receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "protocolVersion": { + "type": "string", + "const": "formal-proof-receipt-v1" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "protocolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "relation": { + "type": "string", + "enum": [ + "exact_proof", + "exact_refutation", + "repaired_proof" + ] + }, + "challengeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "environment": { + "type": "object", + "properties": { + "leanVersion": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "leanToolchainSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lakeManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "dependencyTreeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "leanVersion", + "leanToolchainSHA256", + "lakeManifestSHA256", + "dependencyTreeSHA256" + ], + "additionalProperties": false + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "files": { + "minItems": 6, + "maxItems": 10000, + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "role": { + "type": "string", + "enum": [ + "challenge", + "statement", + "proof", + "lean_toolchain", + "lake_manifest", + "dependency_tree", + "config", + "support" + ] + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "path", + "role", + "sha256" + ], + "additionalProperties": false + } + }, + "verification": { + "type": "object", + "properties": { + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "build": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "warnings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "exitCode", + "warnings", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean" + }, + "findings": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "construct": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "construct", + "path", + "line" + ], + "additionalProperties": false + } + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "complete", + "findings", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "axioms": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "complete": { + "type": "boolean" + }, + "typesTraversed": { + "type": "boolean" + }, + "observed": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "complete", + "typesTraversed", + "observed", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "fresh": { + "type": "object", + "properties": { + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "fresh": { + "type": "boolean" + }, + "exitCode": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "verifierArtifactSHA256", + "fresh", + "exitCode", + "transcriptSHA256" + ], + "additionalProperties": false + }, + "external": { + "type": "object", + "properties": { + "comparatorArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sandboxImageSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sandboxed": { + "type": "boolean" + }, + "challengeMatched": { + "type": "boolean" + }, + "proofTermSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "checks": { + "minItems": 2, + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": [ + "lean_kernel", + "external_checker" + ] + }, + "verifierArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "accepted": { + "type": "boolean" + }, + "transcriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "role", + "verifierArtifactSHA256", + "accepted", + "transcriptSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "comparatorArtifactSHA256", + "sandboxImageSHA256", + "sandboxed", + "challengeMatched", + "proofTermSHA256", + "transcriptSHA256", + "checks" + ], + "additionalProperties": false + } + }, + "required": [ + "startedAt", + "endedAt", + "build", + "source", + "axioms" + ], + "additionalProperties": false + }, + "tier": { + "type": "string", + "enum": [ + "kernel", + "fresh_recheck", + "external_crosscheck" + ] + }, + "metrics": { + "type": "object", + "properties": { + "files": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "warnings": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "observedAxioms": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "disallowedAxioms": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "manifestComplete": { + "type": "boolean" + }, + "buildAccepted": { + "type": "boolean" + }, + "sourceAuditAccepted": { + "type": "boolean" + }, + "forbiddenFindings": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "construct": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + }, + "path": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "line": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "construct", + "path", + "line" + ], + "additionalProperties": false + } + }, + "axiomAuditAccepted": { + "type": "boolean" + }, + "freshRecheckAccepted": { + "type": "boolean" + }, + "externalCrosscheckAccepted": { + "type": "boolean" + }, + "statementMatched": { + "type": "boolean" + } + }, + "required": [ + "files", + "warnings", + "observedAxioms", + "disallowedAxioms", + "manifestComplete", + "buildAccepted", + "sourceAuditAccepted", + "forbiddenFindings", + "axiomAuditAccepted", + "freshRecheckAccepted", + "externalCrosscheckAccepted", + "statementMatched" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "failures": { + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "protocolSHA256", + "subject", + "artifactSHA256", + "relation", + "challengeSHA256", + "statementSHA256", + "declaration", + "module", + "environment", + "manifestSHA256", + "files", + "verification", + "tier", + "metrics", + "status", + "failures", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.formal.receipt({\n ...\n})" + } + ] + } + }, + "/harness/integrity/receipts": { + "post": { + "operationId": "harness.integrity.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record evaluator-authenticated runtime integrity", + "description": "Derives trace-completeness, model-identity, contamination, external-model, benchmark-lookup, and hidden-canary gates against an immutable protocol.", + "responses": { + "200": { + "description": "Immutable runtime integrity receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "submissionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "benchmark-integrity-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "minCoverage": { + "type": "number", + "minimum": 0.9, + "maximum": 1 + }, + "assignedModel": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "forbiddenModelArtifacts": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "policy": { + "type": "object", + "properties": { + "testItemDerivation": { + "type": "string", + "const": "forbidden" + }, + "unapprovedExternalModels": { + "type": "string", + "const": "forbidden" + }, + "benchmarkLookup": { + "type": "string", + "const": "forbidden" + } + }, + "required": [ + "testItemDerivation", + "unapprovedExternalModels", + "benchmarkLookup" + ], + "additionalProperties": false + }, + "auditors": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256" + ], + "additionalProperties": false + } + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minHiddenCanaries": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "traceSchemaSHA256", + "minEvents", + "minCoverage", + "assignedModel", + "policy", + "auditors", + "hiddenCanaryManifestSHA256", + "minHiddenCanaries" + ], + "additionalProperties": false + }, + "trace": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "events": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "dropped": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "artifact", + "schemaSHA256", + "events", + "dropped", + "startedAt", + "endedAt" + ], + "additionalProperties": false + }, + "traceCoverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "model": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outputArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineageVerified": { + "type": "boolean" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256", + "outputArtifactSHA256", + "lineageVerified" + ], + "additionalProperties": false + }, + "audits": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "decision": { + "type": "string", + "enum": [ + "clean", + "flagged", + "abstain" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256", + "decision", + "confidence", + "evidence" + ], + "additionalProperties": false + } + }, + "activity": { + "type": "object", + "properties": { + "unapprovedExternalModelCalls": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "benchmarkLookupEvents": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "hiddenCanariesTested": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "hiddenCanaryViolations": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + } + }, + "required": [ + "unapprovedExternalModelCalls", + "benchmarkLookupEvents", + "hiddenCanaryManifestSHA256", + "hiddenCanariesTested", + "hiddenCanaryViolations" + ], + "additionalProperties": false + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "verify-benchmark-integrity" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "checks": { + "type": "object", + "properties": { + "traceCompleteness": { + "type": "boolean" + }, + "modelIdentity": { + "type": "boolean" + }, + "testItemContamination": { + "type": "boolean" + }, + "externalModelUse": { + "type": "boolean" + }, + "benchmarkLookup": { + "type": "boolean" + }, + "hiddenCanary": { + "type": "boolean" + } + }, + "required": [ + "traceCompleteness", + "modelIdentity", + "testItemContamination", + "externalModelUse", + "benchmarkLookup", + "hiddenCanary" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "failures": { + "maxItems": 14, + "type": "array", + "items": { + "type": "string", + "enum": [ + "trace_schema", + "trace_event_floor", + "trace_coverage", + "model_name", + "model_base_artifact", + "model_config", + "model_lineage", + "forbidden_model_artifact", + "test_item_contamination", + "external_model_use", + "benchmark_lookup", + "hidden_canary_manifest", + "hidden_canary_coverage", + "hidden_canary_violation" + ] + } + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "submissionID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evaluator", + "protocol", + "trace", + "traceCoverage", + "model", + "audits", + "activity", + "validator", + "checks", + "status", + "failures", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "benchmark-integrity-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "minCoverage": { + "type": "number", + "minimum": 0.9, + "maximum": 1 + }, + "assignedModel": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "forbiddenModelArtifacts": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "policy": { + "type": "object", + "properties": { + "testItemDerivation": { + "type": "string", + "const": "forbidden" + }, + "unapprovedExternalModels": { + "type": "string", + "const": "forbidden" + }, + "benchmarkLookup": { + "type": "string", + "const": "forbidden" + } + }, + "required": [ + "testItemDerivation", + "unapprovedExternalModels", + "benchmarkLookup" + ], + "additionalProperties": false + }, + "auditors": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256" + ], + "additionalProperties": false + } + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minHiddenCanaries": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "traceSchemaSHA256", + "minEvents", + "minCoverage", + "assignedModel", + "policy", + "auditors", + "hiddenCanaryManifestSHA256", + "minHiddenCanaries" + ], + "additionalProperties": false + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "trace": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "events": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "dropped": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "artifact", + "schemaSHA256", + "events", + "dropped", + "startedAt", + "endedAt" + ], + "additionalProperties": false + }, + "model": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outputArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineageVerified": { + "type": "boolean" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256", + "outputArtifactSHA256", + "lineageVerified" + ], + "additionalProperties": false + }, + "audits": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "decision": { + "type": "string", + "enum": [ + "clean", + "flagged", + "abstain" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256", + "decision", + "confidence", + "evidence" + ], + "additionalProperties": false + } + }, + "activity": { + "type": "object", + "properties": { + "unapprovedExternalModelCalls": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "benchmarkLookupEvents": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "hiddenCanariesTested": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "hiddenCanaryViolations": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + } + }, + "required": [ + "unapprovedExternalModelCalls", + "benchmarkLookupEvents", + "hiddenCanaryManifestSHA256", + "hiddenCanariesTested", + "hiddenCanaryViolations" + ], + "additionalProperties": false + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "verify-benchmark-integrity" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "evaluatorToken", + "protocol", + "subject", + "trace", + "model", + "audits", + "activity", + "validator", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.integrity.record({\n ...\n})" + } + ] + } + }, + "/harness/integrity/receipts/{receiptID}": { + "post": { + "operationId": "harness.integrity.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected runtime integrity receipt", + "responses": { + "200": { + "description": "Runtime integrity receipt", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "submissionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "benchmark-integrity-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "minCoverage": { + "type": "number", + "minimum": 0.9, + "maximum": 1 + }, + "assignedModel": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "forbiddenModelArtifacts": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "policy": { + "type": "object", + "properties": { + "testItemDerivation": { + "type": "string", + "const": "forbidden" + }, + "unapprovedExternalModels": { + "type": "string", + "const": "forbidden" + }, + "benchmarkLookup": { + "type": "string", + "const": "forbidden" + } + }, + "required": [ + "testItemDerivation", + "unapprovedExternalModels", + "benchmarkLookup" + ], + "additionalProperties": false + }, + "auditors": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256" + ], + "additionalProperties": false + } + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minHiddenCanaries": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "traceSchemaSHA256", + "minEvents", + "minCoverage", + "assignedModel", + "policy", + "auditors", + "hiddenCanaryManifestSHA256", + "minHiddenCanaries" + ], + "additionalProperties": false + }, + "trace": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "events": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "dropped": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "endedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "artifact", + "schemaSHA256", + "events", + "dropped", + "startedAt", + "endedAt" + ], + "additionalProperties": false + }, + "traceCoverage": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "model": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outputArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineageVerified": { + "type": "boolean" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256", + "outputArtifactSHA256", + "lineageVerified" + ], + "additionalProperties": false + }, + "audits": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "decision": { + "type": "string", + "enum": [ + "clean", + "flagged", + "abstain" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256", + "decision", + "confidence", + "evidence" + ], + "additionalProperties": false + } + }, + "activity": { + "type": "object", + "properties": { + "unapprovedExternalModelCalls": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "benchmarkLookupEvents": { + "type": "integer", + "minimum": 0, + "maximum": 10000000 + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "hiddenCanariesTested": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "hiddenCanaryViolations": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + } + }, + "required": [ + "unapprovedExternalModelCalls", + "benchmarkLookupEvents", + "hiddenCanaryManifestSHA256", + "hiddenCanariesTested", + "hiddenCanaryViolations" + ], + "additionalProperties": false + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "verify-benchmark-integrity" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "checks": { + "type": "object", + "properties": { + "traceCompleteness": { + "type": "boolean" + }, + "modelIdentity": { + "type": "boolean" + }, + "testItemContamination": { + "type": "boolean" + }, + "externalModelUse": { + "type": "boolean" + }, + "benchmarkLookup": { + "type": "boolean" + }, + "hiddenCanary": { + "type": "boolean" + } + }, + "required": [ + "traceCompleteness", + "modelIdentity", + "testItemContamination", + "externalModelUse", + "benchmarkLookup", + "hiddenCanary" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "failures": { + "maxItems": 14, + "type": "array", + "items": { + "type": "string", + "enum": [ + "trace_schema", + "trace_event_floor", + "trace_coverage", + "model_name", + "model_base_artifact", + "model_config", + "model_lineage", + "forbidden_model_artifact", + "test_item_contamination", + "external_model_use", + "benchmark_lookup", + "hidden_canary_manifest", + "hidden_canary_coverage", + "hidden_canary_violation" + ] + } + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "submissionID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evaluator", + "protocol", + "trace", + "traceCoverage", + "model", + "audits", + "activity", + "validator", + "checks", + "status", + "failures", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.integrity.receipt({\n ...\n})" + } + ] + } + }, + "/harness/evolution/receipts": { + "post": { + "operationId": "harness.evolution.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record evaluator-authenticated evolutionary provenance", + "description": "Binds a candidate snapshot and every parent delta to immutable search lineage, then derives replay and ancestral line-reintroduction diagnostics without changing fitness.", + "responses": { + "200": { + "description": "Immutable evolution trace receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "submissionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evolution-trace-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineAlgorithm": { + "type": "string", + "const": "sha256-exact-line-v1" + }, + "roots": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "extensions": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^\\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}$" + } + }, + "exclude": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 1, + "maximum": 100000 + }, + "maxFileBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1000000000 + }, + "maxTotalBytes": { + "type": "integer", + "minimum": 1, + "maximum": 10000000000 + }, + "maxSourceLines": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "maxChangedLines": { + "type": "integer", + "minimum": 1, + "maximum": 2000000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "manifestSchemaSHA256", + "lineAlgorithm", + "roots", + "extensions", + "maxFiles", + "maxFileBytes", + "maxTotalBytes", + "maxSourceLines", + "maxChangedLines" + ], + "additionalProperties": false + }, + "snapshot": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "files": { + "minItems": 1, + "maxItems": 100000, + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 1000000000 + }, + "lineHashes": { + "maxItems": 1000000, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } + }, + "required": [ + "path", + "sha256", + "bytes", + "lineHashes" + ], + "additionalProperties": false + } + } + }, + "required": [ + "artifact", + "schemaSHA256", + "files" + ], + "additionalProperties": false + }, + "parents": { + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "snapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "delta": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "artifact", + "receiptID", + "snapshotSHA256", + "delta" + ], + "additionalProperties": false + } + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "trace-evolutionary-candidate" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "diagnostics": { + "type": "object", + "properties": { + "files": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sourceLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "depth": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ancestors": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "addedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deletedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ancestralDeletedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reintroducedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reintroducedHashes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reintroducedFraction": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sourceChanged": { + "type": "boolean" + }, + "cycleDetected": { + "type": "boolean" + }, + "parents": { + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "filesChanged": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "addedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deletedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "receiptID", + "filesChanged", + "addedLines", + "deletedLines" + ], + "additionalProperties": false + } + } + }, + "required": [ + "files", + "bytes", + "sourceLines", + "depth", + "ancestors", + "addedLines", + "deletedLines", + "ancestralDeletedLines", + "reintroducedLines", + "reintroducedHashes", + "reintroducedFraction", + "novelLines", + "sourceChanged", + "cycleDetected", + "parents" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "submissionID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evaluator", + "protocol", + "snapshot", + "parents", + "validator", + "diagnostics", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evolution-trace-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineAlgorithm": { + "type": "string", + "const": "sha256-exact-line-v1" + }, + "roots": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "extensions": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^\\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}$" + } + }, + "exclude": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 1, + "maximum": 100000 + }, + "maxFileBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1000000000 + }, + "maxTotalBytes": { + "type": "integer", + "minimum": 1, + "maximum": 10000000000 + }, + "maxSourceLines": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "maxChangedLines": { + "type": "integer", + "minimum": 1, + "maximum": 2000000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "manifestSchemaSHA256", + "lineAlgorithm", + "roots", + "extensions", + "maxFiles", + "maxFileBytes", + "maxTotalBytes", + "maxSourceLines", + "maxChangedLines" + ], + "additionalProperties": false + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "snapshot": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "files": { + "minItems": 1, + "maxItems": 100000, + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 1000000000 + }, + "lineHashes": { + "maxItems": 1000000, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } + }, + "required": [ + "path", + "sha256", + "bytes", + "lineHashes" + ], + "additionalProperties": false + } + } + }, + "required": [ + "artifact", + "schemaSHA256", + "files" + ], + "additionalProperties": false + }, + "parents": { + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "snapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "delta": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "artifact", + "receiptID", + "snapshotSHA256", + "delta" + ], + "additionalProperties": false + } + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "trace-evolutionary-candidate" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "evaluatorToken", + "protocol", + "subject", + "snapshot", + "parents", + "validator", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.evolution.record({\n ...\n})" + } + ] + } + }, + "/harness/evolution/receipts/{receiptID}": { + "post": { + "operationId": "harness.evolution.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected evolution trace receipt", + "responses": { + "200": { + "description": "Evolution trace receipt", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "submissionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "candidate" + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evolution-trace-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineAlgorithm": { + "type": "string", + "const": "sha256-exact-line-v1" + }, + "roots": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "extensions": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^\\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}$" + } + }, + "exclude": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 1, + "maximum": 100000 + }, + "maxFileBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1000000000 + }, + "maxTotalBytes": { + "type": "integer", + "minimum": 1, + "maximum": 10000000000 + }, + "maxSourceLines": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "maxChangedLines": { + "type": "integer", + "minimum": 1, + "maximum": 2000000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "manifestSchemaSHA256", + "lineAlgorithm", + "roots", + "extensions", + "maxFiles", + "maxFileBytes", + "maxTotalBytes", + "maxSourceLines", + "maxChangedLines" + ], + "additionalProperties": false + }, + "snapshot": { + "type": "object", + "properties": { + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "schemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "files": { + "minItems": 1, + "maxItems": 100000, + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 1000000000 + }, + "lineHashes": { + "maxItems": 1000000, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + } + }, + "required": [ + "path", + "sha256", + "bytes", + "lineHashes" + ], + "additionalProperties": false + } + } + }, + "required": [ + "artifact", + "schemaSHA256", + "files" + ], + "additionalProperties": false + }, + "parents": { + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "snapshotSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "delta": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "artifact", + "receiptID", + "snapshotSHA256", + "delta" + ], + "additionalProperties": false + } + }, + "validator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "trace-evolutionary-candidate" + }, + "version": { + "type": "number", + "const": 1 + }, + "scriptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "scriptSHA256" + ], + "additionalProperties": false + }, + "diagnostics": { + "type": "object", + "properties": { + "files": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "bytes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sourceLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "depth": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ancestors": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "addedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deletedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "ancestralDeletedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reintroducedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reintroducedHashes": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reintroducedFraction": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "sourceChanged": { + "type": "boolean" + }, + "cycleDetected": { + "type": "boolean" + }, + "parents": { + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "filesChanged": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "addedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "deletedLines": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "receiptID", + "filesChanged", + "addedLines", + "deletedLines" + ], + "additionalProperties": false + } + } + }, + "required": [ + "files", + "bytes", + "sourceLines", + "depth", + "ancestors", + "addedLines", + "deletedLines", + "ancestralDeletedLines", + "reintroducedLines", + "reintroducedHashes", + "reintroducedFraction", + "novelLines", + "sourceChanged", + "cycleDetected", + "parents" + ], + "additionalProperties": false + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "submissionID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evaluator", + "protocol", + "snapshot", + "parents", + "validator", + "diagnostics", + "evidence", + "evaluatedAt", + "recordedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.evolution.receipt({\n ...\n})" + } + ] + } + }, + "/harness/simulations/receipts": { + "post": { + "operationId": "harness.simulation.record", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Record an evaluator-authenticated simulator validation", + "description": "Recomputes convergence, residual, invariant, and stress-test gates against the immutable simulator protocol and exact subject artifact.", + "responses": { + "200": { + "description": "Immutable simulator validation receipt", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "engine": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commandSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commandSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "problemSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reference": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "analytic", + "manufactured", + "benchmark", + "independent_solver", + "limiting_case" + ] + }, + "identity": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "identity", + "sha256" + ], + "additionalProperties": false + }, + "validationInputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "levels": { + "minItems": 3, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "h": { + "type": "number", + "exclusiveMinimum": 0 + }, + "error": { + "type": "number", + "exclusiveMinimum": 0 + }, + "residual": { + "type": "number", + "minimum": 0 + }, + "invariants": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "number", + "minimum": 0 + } + } + }, + "required": [ + "label", + "h", + "error", + "residual", + "invariants" + ], + "additionalProperties": false + } + }, + "observedOrders": { + "maxItems": 23, + "type": "array", + "items": { + "type": "number" + } + }, + "medianObservedOrder": { + "type": "number" + }, + "stressTests": { + "maxItems": 7, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "checks": { + "type": "object", + "properties": { + "enoughLevels": { + "type": "boolean" + }, + "resolutionDecreases": { + "type": "boolean" + }, + "errorDecreases": { + "type": "boolean" + }, + "observedOrder": { + "type": "boolean" + }, + "residualBound": { + "type": "boolean" + }, + "invariants": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "boolean" + } + }, + "stressTests": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + }, + "additionalProperties": { + "type": "boolean" + } + } + }, + "required": [ + "enoughLevels", + "resolutionDecreases", + "errorDecreases", + "observedOrder", + "residualBound", + "invariants", + "stressTests" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evaluator", + "engine", + "problemSHA256", + "reference", + "validationInputSHA256", + "levels", + "observedOrders", + "medianObservedOrder", + "stressTests", + "checks", + "status", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "engine": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commandSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commandSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "problemSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reference": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "analytic", + "manufactured", + "benchmark", + "independent_solver", + "limiting_case" + ] + }, + "identity": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "identity", + "sha256" + ], + "additionalProperties": false + }, + "validationInputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "levels": { + "minItems": 3, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "h": { + "type": "number", + "exclusiveMinimum": 0 + }, + "error": { + "type": "number", + "exclusiveMinimum": 0 + }, + "residual": { + "type": "number", + "minimum": 0 + }, + "invariants": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "number", + "minimum": 0 + } + } + }, + "required": [ + "label", + "h", + "error", + "residual", + "invariants" + ], + "additionalProperties": false + } + }, + "stressTests": { + "maxItems": 7, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "evaluatorToken", + "subject", + "engine", + "problemSHA256", + "reference", + "validationInputSHA256", + "levels", + "stressTests", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.simulation.record({\n ...\n})" + } + ] + } + }, + "/harness/simulations/receipts/{receiptID}": { + "post": { + "operationId": "harness.simulation.receipt", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "receiptID", + "schema": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "required": true + } + ], + "summary": "Read a capability-protected simulator validation receipt", + "responses": { + "200": { + "description": "Simulator validation receipt", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "receiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "artifact": { + "type": "object", + "properties": { + "uri": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "uri", + "sha256" + ], + "additionalProperties": false + } + }, + "required": [ + "type", + "id", + "artifact" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "engine": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commandSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commandSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "problemSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reference": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "analytic", + "manufactured", + "benchmark", + "independent_solver", + "limiting_case" + ] + }, + "identity": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "identity", + "sha256" + ], + "additionalProperties": false + }, + "validationInputSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "levels": { + "minItems": 3, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "h": { + "type": "number", + "exclusiveMinimum": 0 + }, + "error": { + "type": "number", + "exclusiveMinimum": 0 + }, + "residual": { + "type": "number", + "minimum": 0 + }, + "invariants": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "number", + "minimum": 0 + } + } + }, + "required": [ + "label", + "h", + "error", + "residual", + "invariants" + ], + "additionalProperties": false + } + }, + "observedOrders": { + "maxItems": 23, + "type": "array", + "items": { + "type": "number" + } + }, + "medianObservedOrder": { + "type": "number" + }, + "stressTests": { + "maxItems": 7, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "id", + "status", + "evidence" + ], + "additionalProperties": false + } + }, + "checks": { + "type": "object", + "properties": { + "enoughLevels": { + "type": "boolean" + }, + "resolutionDecreases": { + "type": "boolean" + }, + "errorDecreases": { + "type": "boolean" + }, + "observedOrder": { + "type": "boolean" + }, + "residualBound": { + "type": "boolean" + }, + "invariants": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "boolean" + } + }, + "stressTests": { + "type": "object", + "propertyNames": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + }, + "additionalProperties": { + "type": "boolean" + } + } + }, + "required": [ + "enoughLevels", + "resolutionDecreases", + "errorDecreases", + "observedOrder", + "residualBound", + "invariants", + "stressTests" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "receiptID", + "runID", + "sessionID", + "contractFingerprint", + "subject", + "evaluator", + "engine", + "problemSHA256", + "reference", + "validationInputSHA256", + "levels", + "observedOrders", + "medianObservedOrder", + "stressTests", + "checks", + "status", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.simulation.receipt({\n ...\n})" + } + ] + } + }, + "/harness/runs/{sessionID}/orchestration": { + "post": { + "operationId": "harness.orchestration.start", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Initialize contract-bound scientific orchestration", + "description": "Selects a bounded topology from immutable contract traits and creates a restart-safe provisional work DAG.", + "responses": { + "200": { + "description": "Scientific orchestration state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 3 + }, + "protocolVersion": { + "type": "string", + "enum": [ + "coalition-v1", + "coalition-v2", + "coalition-v3" + ] + }, + "sessionPolicy": { + "type": "string", + "enum": [ + "legacy-v1", + "fresh-v1", + "producer-lanes-v1" + ] + }, + "workerPolicy": { + "type": "string", + "enum": [ + "claimed-v1", + "task-attested-v1" + ] + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "objective": { + "type": "string", + "minLength": 1 + }, + "selection": { + "type": "object", + "properties": { + "topology": { + "type": "string", + "enum": [ + "solo", + "centralized", + "fork_join", + "tournament", + "evolution", + "verifier_loop" + ] + }, + "source": { + "type": "string", + "enum": [ + "contract", + "policy" + ] + }, + "reasons": { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "traits": { + "type": "object", + "properties": { + "decomposability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sequentiality": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "toolIntensity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "verificationRisk": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "crossDomain": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "decomposability", + "sequentiality", + "toolIntensity", + "uncertainty", + "verificationRisk", + "novelty", + "crossDomain" + ], + "additionalProperties": false + } + }, + "required": [ + "topology", + "source", + "reasons", + "traits" + ], + "additionalProperties": false + }, + "maxWorkers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "maxRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "minIndependentVerifiers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "status": { + "type": "string", + "enum": [ + "active", + "awaiting_checkpoint", + "completed" + ] + }, + "adaptive": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "marginal-utility-v1" + }, + "minRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "patience": { + "type": "integer", + "minimum": 1, + "maximum": 7 + }, + "minUtilityGain": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetUtility": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "checkpoints": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "round": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "utility": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "gain": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "qualified": { + "type": "boolean" + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "round", + "utility", + "uncertainty", + "evidenceRefs", + "evaluatedAt", + "id", + "gain", + "qualified", + "recordedAt" + ], + "additionalProperties": false + } + }, + "stalled": { + "type": "integer", + "minimum": 0, + "maximum": 8 + }, + "phase": { + "type": "string", + "enum": [ + "searching", + "finalizing" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "target_reached", + "marginal_utility_exhausted", + "max_rounds" + ] + } + }, + "required": [ + "protocolVersion", + "minRounds", + "patience", + "minUtilityGain", + "maxUncertainty", + "checkpoints", + "stalled", + "phase" + ], + "additionalProperties": false + }, + "repair": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "verifier-routed-v1" + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "phase": { + "type": "string", + "enum": [ + "producing", + "verifying", + "investigating", + "completed" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifierIDs": { + "maxItems": 2, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "evidenceID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "routes": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "attempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "actionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifierIDs": { + "minItems": 1, + "maxItems": 2, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "decision": { + "type": "string", + "enum": [ + "accept", + "revise", + "restart", + "investigate" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "attempt", + "candidateID", + "verifierIDs", + "decision", + "confidence", + "evidenceRefs", + "recordedAt" + ], + "additionalProperties": false + } + }, + "stopReason": { + "type": "string", + "enum": [ + "accepted", + "attempt_limit", + "work_failed" + ] + } + }, + "required": [ + "protocolVersion", + "minConfidence", + "phase", + "candidateID", + "verifierIDs", + "routes" + ], + "additionalProperties": false + }, + "consensus": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "supported", + "rejected", + "disputed", + "insufficient" + ] + }, + "verifierCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "support": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reject": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "abstain": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "provisional": { + "type": "boolean", + "const": true + }, + "derivedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "status", + "verifierCount", + "support", + "reject", + "abstain", + "confidence", + "evidenceRefs", + "provisional", + "derivedAt" + ], + "additionalProperties": false + }, + "work": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "enum": [ + "generation", + "proximity", + "reflection", + "ranking", + "evolution", + "revision", + "verification", + "investigation", + "simulation", + "synthesis" + ] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "round": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "agent": { + "type": "string", + "enum": [ + "task", + "biology", + "physics", + "ml", + "critique", + "physics-critique", + "reviewer" + ] + }, + "dependencies": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "prompt": { + "type": "string", + "minLength": 1, + "maxLength": 40000 + }, + "allocation": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "lane": { + "type": "string", + "enum": [ + "producer-a", + "producer-b" + ] + }, + "status": { + "type": "string", + "enum": [ + "pending", + "executed", + "completed", + "failed", + "cancelled" + ] + }, + "workerSessionID": { + "type": "string", + "minLength": 1 + }, + "workerReceipt": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "workID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "workerSessionID": { + "type": "string", + "minLength": 1 + }, + "turnID": { + "type": "string", + "minLength": 1 + }, + "agent": { + "type": "string", + "enum": [ + "task", + "biology", + "physics", + "ml", + "critique", + "physics-critique", + "reviewer" + ] + }, + "workPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed" + ] + }, + "usage": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "toolCalls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failedToolCalls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "provisional": { + "type": "boolean", + "const": true + } + }, + "required": [ + "id", + "workID", + "workerSessionID", + "turnID", + "agent", + "workPromptSHA256", + "taskPromptSHA256", + "outcome", + "usage", + "toolCalls", + "failedToolCalls", + "startedAt", + "completedAt", + "provisional" + ], + "additionalProperties": false + }, + "result": { + "type": "object", + "properties": { + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "artifactRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "evidenceRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "usage": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "verdict": { + "type": "object", + "properties": { + "decision": { + "type": "string", + "enum": [ + "support", + "reject", + "abstain" + ] + }, + "severity": { + "type": "string", + "enum": [ + "none", + "minor", + "critical", + "unknown" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "checks": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + } + }, + "required": [ + "id", + "status", + "evidenceRefs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "decision", + "confidence", + "checks" + ], + "additionalProperties": false + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "summary", + "completedAt" + ], + "additionalProperties": false + }, + "failure": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + } + }, + "required": [ + "id", + "role", + "label", + "round", + "agent", + "dependencies", + "prompt", + "allocation", + "status" + ], + "additionalProperties": false + } + }, + "order": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "sessionPolicy", + "workerPolicy", + "runID", + "sessionID", + "contractFingerprint", + "objective", + "selection", + "maxWorkers", + "maxRounds", + "minIndependentVerifiers", + "status", + "work", + "order", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.orchestration.start({\n ...\n})" + } + ] + }, + "get": { + "operationId": "harness.orchestration.status", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Read scientific orchestration state", + "responses": { + "200": { + "description": "Scientific orchestration state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 3 + }, + "protocolVersion": { + "type": "string", + "enum": [ + "coalition-v1", + "coalition-v2", + "coalition-v3" + ] + }, + "sessionPolicy": { + "type": "string", + "enum": [ + "legacy-v1", + "fresh-v1", + "producer-lanes-v1" + ] + }, + "workerPolicy": { + "type": "string", + "enum": [ + "claimed-v1", + "task-attested-v1" + ] + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "objective": { + "type": "string", + "minLength": 1 + }, + "selection": { + "type": "object", + "properties": { + "topology": { + "type": "string", + "enum": [ + "solo", + "centralized", + "fork_join", + "tournament", + "evolution", + "verifier_loop" + ] + }, + "source": { + "type": "string", + "enum": [ + "contract", + "policy" + ] + }, + "reasons": { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "traits": { + "type": "object", + "properties": { + "decomposability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sequentiality": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "toolIntensity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "verificationRisk": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "crossDomain": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "decomposability", + "sequentiality", + "toolIntensity", + "uncertainty", + "verificationRisk", + "novelty", + "crossDomain" + ], + "additionalProperties": false + } + }, + "required": [ + "topology", + "source", + "reasons", + "traits" + ], + "additionalProperties": false + }, + "maxWorkers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "maxRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "minIndependentVerifiers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "status": { + "type": "string", + "enum": [ + "active", + "awaiting_checkpoint", + "completed" + ] + }, + "adaptive": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "marginal-utility-v1" + }, + "minRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "patience": { + "type": "integer", + "minimum": 1, + "maximum": 7 + }, + "minUtilityGain": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetUtility": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "checkpoints": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "round": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "utility": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "gain": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "qualified": { + "type": "boolean" + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "round", + "utility", + "uncertainty", + "evidenceRefs", + "evaluatedAt", + "id", + "gain", + "qualified", + "recordedAt" + ], + "additionalProperties": false + } + }, + "stalled": { + "type": "integer", + "minimum": 0, + "maximum": 8 + }, + "phase": { + "type": "string", + "enum": [ + "searching", + "finalizing" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "target_reached", + "marginal_utility_exhausted", + "max_rounds" + ] + } + }, + "required": [ + "protocolVersion", + "minRounds", + "patience", + "minUtilityGain", + "maxUncertainty", + "checkpoints", + "stalled", + "phase" + ], + "additionalProperties": false + }, + "repair": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "verifier-routed-v1" + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "phase": { + "type": "string", + "enum": [ + "producing", + "verifying", + "investigating", + "completed" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifierIDs": { + "maxItems": 2, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "evidenceID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "routes": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "attempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "actionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifierIDs": { + "minItems": 1, + "maxItems": 2, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "decision": { + "type": "string", + "enum": [ + "accept", + "revise", + "restart", + "investigate" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "attempt", + "candidateID", + "verifierIDs", + "decision", + "confidence", + "evidenceRefs", + "recordedAt" + ], + "additionalProperties": false + } + }, + "stopReason": { + "type": "string", + "enum": [ + "accepted", + "attempt_limit", + "work_failed" + ] + } + }, + "required": [ + "protocolVersion", + "minConfidence", + "phase", + "candidateID", + "verifierIDs", + "routes" + ], + "additionalProperties": false + }, + "consensus": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "supported", + "rejected", + "disputed", + "insufficient" + ] + }, + "verifierCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "support": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reject": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "abstain": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "provisional": { + "type": "boolean", + "const": true + }, + "derivedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "status", + "verifierCount", + "support", + "reject", + "abstain", + "confidence", + "evidenceRefs", + "provisional", + "derivedAt" + ], + "additionalProperties": false + }, + "work": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "enum": [ + "generation", + "proximity", + "reflection", + "ranking", + "evolution", + "revision", + "verification", + "investigation", + "simulation", + "synthesis" + ] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "round": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "agent": { + "type": "string", + "enum": [ + "task", + "biology", + "physics", + "ml", + "critique", + "physics-critique", + "reviewer" + ] + }, + "dependencies": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "prompt": { + "type": "string", + "minLength": 1, + "maxLength": 40000 + }, + "allocation": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "lane": { + "type": "string", + "enum": [ + "producer-a", + "producer-b" + ] + }, + "status": { + "type": "string", + "enum": [ + "pending", + "executed", + "completed", + "failed", + "cancelled" + ] + }, + "workerSessionID": { + "type": "string", + "minLength": 1 + }, + "workerReceipt": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "workID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "workerSessionID": { + "type": "string", + "minLength": 1 + }, + "turnID": { + "type": "string", + "minLength": 1 + }, + "agent": { + "type": "string", + "enum": [ + "task", + "biology", + "physics", + "ml", + "critique", + "physics-critique", + "reviewer" + ] + }, + "workPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed" + ] + }, + "usage": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "toolCalls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failedToolCalls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "provisional": { + "type": "boolean", + "const": true + } + }, + "required": [ + "id", + "workID", + "workerSessionID", + "turnID", + "agent", + "workPromptSHA256", + "taskPromptSHA256", + "outcome", + "usage", + "toolCalls", + "failedToolCalls", + "startedAt", + "completedAt", + "provisional" + ], + "additionalProperties": false + }, + "result": { + "type": "object", + "properties": { + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "artifactRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "evidenceRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "usage": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "verdict": { + "type": "object", + "properties": { + "decision": { + "type": "string", + "enum": [ + "support", + "reject", + "abstain" + ] + }, + "severity": { + "type": "string", + "enum": [ + "none", + "minor", + "critical", + "unknown" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "checks": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + } + }, + "required": [ + "id", + "status", + "evidenceRefs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "decision", + "confidence", + "checks" + ], + "additionalProperties": false + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "summary", + "completedAt" + ], + "additionalProperties": false + }, + "failure": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + } + }, + "required": [ + "id", + "role", + "label", + "round", + "agent", + "dependencies", + "prompt", + "allocation", + "status" + ], + "additionalProperties": false + } + }, + "order": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "sessionPolicy", + "workerPolicy", + "runID", + "sessionID", + "contractFingerprint", + "objective", + "selection", + "maxWorkers", + "maxRounds", + "minIndependentVerifiers", + "status", + "work", + "order", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.orchestration.status({\n ...\n})" + } + ] + } + }, + "/harness/runs/{sessionID}/orchestration/checkpoints": { + "post": { + "operationId": "harness.orchestration.checkpoint", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Record an evaluator-authenticated orchestration utility checkpoint", + "description": "Gates the next evolution round and stops low-utility search without allowing worker self-scores to control budget.", + "responses": { + "200": { + "description": "Scientific orchestration state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 3 + }, + "protocolVersion": { + "type": "string", + "enum": [ + "coalition-v1", + "coalition-v2", + "coalition-v3" + ] + }, + "sessionPolicy": { + "type": "string", + "enum": [ + "legacy-v1", + "fresh-v1", + "producer-lanes-v1" + ] + }, + "workerPolicy": { + "type": "string", + "enum": [ + "claimed-v1", + "task-attested-v1" + ] + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "objective": { + "type": "string", + "minLength": 1 + }, + "selection": { + "type": "object", + "properties": { + "topology": { + "type": "string", + "enum": [ + "solo", + "centralized", + "fork_join", + "tournament", + "evolution", + "verifier_loop" + ] + }, + "source": { + "type": "string", + "enum": [ + "contract", + "policy" + ] + }, + "reasons": { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "traits": { + "type": "object", + "properties": { + "decomposability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sequentiality": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "toolIntensity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "verificationRisk": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "crossDomain": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "decomposability", + "sequentiality", + "toolIntensity", + "uncertainty", + "verificationRisk", + "novelty", + "crossDomain" + ], + "additionalProperties": false + } + }, + "required": [ + "topology", + "source", + "reasons", + "traits" + ], + "additionalProperties": false + }, + "maxWorkers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "maxRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "minIndependentVerifiers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "status": { + "type": "string", + "enum": [ + "active", + "awaiting_checkpoint", + "completed" + ] + }, + "adaptive": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "marginal-utility-v1" + }, + "minRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "patience": { + "type": "integer", + "minimum": 1, + "maximum": 7 + }, + "minUtilityGain": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetUtility": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "checkpoints": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "round": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "utility": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "gain": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "qualified": { + "type": "boolean" + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "round", + "utility", + "uncertainty", + "evidenceRefs", + "evaluatedAt", + "id", + "gain", + "qualified", + "recordedAt" + ], + "additionalProperties": false + } + }, + "stalled": { + "type": "integer", + "minimum": 0, + "maximum": 8 + }, + "phase": { + "type": "string", + "enum": [ + "searching", + "finalizing" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "target_reached", + "marginal_utility_exhausted", + "max_rounds" + ] + } + }, + "required": [ + "protocolVersion", + "minRounds", + "patience", + "minUtilityGain", + "maxUncertainty", + "checkpoints", + "stalled", + "phase" + ], + "additionalProperties": false + }, + "repair": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "verifier-routed-v1" + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "phase": { + "type": "string", + "enum": [ + "producing", + "verifying", + "investigating", + "completed" + ] + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifierIDs": { + "maxItems": 2, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "evidenceID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "routes": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "attempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "actionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifierIDs": { + "minItems": 1, + "maxItems": 2, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "decision": { + "type": "string", + "enum": [ + "accept", + "revise", + "restart", + "investigate" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "attempt", + "candidateID", + "verifierIDs", + "decision", + "confidence", + "evidenceRefs", + "recordedAt" + ], + "additionalProperties": false + } + }, + "stopReason": { + "type": "string", + "enum": [ + "accepted", + "attempt_limit", + "work_failed" + ] + } + }, + "required": [ + "protocolVersion", + "minConfidence", + "phase", + "candidateID", + "verifierIDs", + "routes" + ], + "additionalProperties": false + }, + "consensus": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "supported", + "rejected", + "disputed", + "insufficient" + ] + }, + "verifierCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "support": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reject": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "abstain": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "provisional": { + "type": "boolean", + "const": true + }, + "derivedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "status", + "verifierCount", + "support", + "reject", + "abstain", + "confidence", + "evidenceRefs", + "provisional", + "derivedAt" + ], + "additionalProperties": false + }, + "work": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "role": { + "type": "string", + "enum": [ + "generation", + "proximity", + "reflection", + "ranking", + "evolution", + "revision", + "verification", + "investigation", + "simulation", + "synthesis" + ] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 160 + }, + "round": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "agent": { + "type": "string", + "enum": [ + "task", + "biology", + "physics", + "ml", + "critique", + "physics-critique", + "reviewer" + ] + }, + "dependencies": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "prompt": { + "type": "string", + "minLength": 1, + "maxLength": 40000 + }, + "allocation": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "lane": { + "type": "string", + "enum": [ + "producer-a", + "producer-b" + ] + }, + "status": { + "type": "string", + "enum": [ + "pending", + "executed", + "completed", + "failed", + "cancelled" + ] + }, + "workerSessionID": { + "type": "string", + "minLength": 1 + }, + "workerReceipt": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "workID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "workerSessionID": { + "type": "string", + "minLength": 1 + }, + "turnID": { + "type": "string", + "minLength": 1 + }, + "agent": { + "type": "string", + "enum": [ + "task", + "biology", + "physics", + "ml", + "critique", + "physics-critique", + "reviewer" + ] + }, + "workPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "taskPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "outcome": { + "type": "string", + "enum": [ + "completed", + "failed" + ] + }, + "usage": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "toolCalls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failedToolCalls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "startedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "provisional": { + "type": "boolean", + "const": true + } + }, + "required": [ + "id", + "workID", + "workerSessionID", + "turnID", + "agent", + "workPromptSHA256", + "taskPromptSHA256", + "outcome", + "usage", + "toolCalls", + "failedToolCalls", + "startedAt", + "completedAt", + "provisional" + ], + "additionalProperties": false + }, + "result": { + "type": "object", + "properties": { + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 8000 + }, + "artifactRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "evidenceRefs": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "usage": { + "type": "object", + "properties": { + "steps": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "wallTimeMs": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "verdict": { + "type": "object", + "properties": { + "decision": { + "type": "string", + "enum": [ + "support", + "reject", + "abstain" + ] + }, + "severity": { + "type": "string", + "enum": [ + "none", + "minor", + "critical", + "unknown" + ] + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "checks": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + } + }, + "required": [ + "id", + "status", + "evidenceRefs" + ], + "additionalProperties": false + } + } + }, + "required": [ + "decision", + "confidence", + "checks" + ], + "additionalProperties": false + }, + "completedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "summary", + "completedAt" + ], + "additionalProperties": false + }, + "failure": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + } + }, + "required": [ + "id", + "role", + "label", + "round", + "agent", + "dependencies", + "prompt", + "allocation", + "status" + ], + "additionalProperties": false + } + }, + "order": { + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "protocolVersion", + "sessionPolicy", + "workerPolicy", + "runID", + "sessionID", + "contractFingerprint", + "objective", + "selection", + "maxWorkers", + "maxRounds", + "minIndependentVerifiers", + "status", + "work", + "order", + "revision", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "round": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "utility": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidenceRefs": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "evaluatorToken", + "round", + "utility", + "uncertainty", + "evidenceRefs", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.orchestration.checkpoint({\n ...\n})" + } + ] + } + }, + "/harness/runs/{sessionID}/world": { + "get": { + "operationId": "harness.world.status", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Read the session-local continual world model", + "description": "Returns confidence-graded mutable working state, event boundaries, context epoch, refinement trigger, and rollback revisions without exposing evaluator capabilities.", + "responses": { + "200": { + "description": "Continual world-model state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "basePromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "entries": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "key": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{0,79}$" + }, + "kind": { + "type": "string", + "enum": [ + "hypothesis", + "observation", + "strategy", + "memory", + "skill", + "subagent" + ] + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "confidence": { + "type": "integer", + "minimum": 1, + "maximum": 5 + }, + "evidence": { + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "authority": { + "type": "string", + "enum": [ + "self", + "tool", + "evaluator", + "human" + ] + } + }, + "required": [ + "ref", + "authority" + ], + "additionalProperties": false + } + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "key", + "kind", + "content", + "confidence", + "evidence", + "updatedAt", + "revision" + ], + "additionalProperties": false + } + }, + "events": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "type": { + "type": "string", + "enum": [ + "analysis", + "tool", + "evaluation", + "failure", + "milestone", + "stagnation", + "manual" + ] + }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "evidenceRefs": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "changed": { + "type": "boolean" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "type", + "summary", + "evidenceRefs", + "changed", + "createdAt" + ], + "additionalProperties": false + } + }, + "snapshots": { + "maxItems": 10, + "type": "array", + "items": { + "type": "object", + "properties": { + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "key": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{0,79}$" + }, + "kind": { + "type": "string", + "enum": [ + "hypothesis", + "observation", + "strategy", + "memory", + "skill", + "subagent" + ] + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "confidence": { + "type": "integer", + "minimum": 1, + "maximum": 5 + }, + "evidence": { + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "authority": { + "type": "string", + "enum": [ + "self", + "tool", + "evaluator", + "human" + ] + } + }, + "required": [ + "ref", + "authority" + ], + "additionalProperties": false + } + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "key", + "kind", + "content", + "confidence", + "evidence", + "updatedAt", + "revision" + ], + "additionalProperties": false + } + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "revision", + "entries", + "sha256", + "createdAt" + ], + "additionalProperties": false + } + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "contextEpoch": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "eventsSinceRefine": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refinement": { + "type": "object", + "properties": { + "recommended": { + "type": "boolean" + }, + "trigger": { + "type": "string", + "enum": [ + "manual", + "failure", + "stagnation", + "milestone", + "periodic" + ] + } + }, + "required": [ + "recommended" + ], + "additionalProperties": false + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "sessionID", + "runID", + "basePromptSHA256", + "entries", + "events", + "snapshots", + "revision", + "contextEpoch", + "eventsSinceRefine", + "refinement", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.world.status({\n ...\n})" + } + ] + } + }, + "/harness/runs/{sessionID}/world/refinements": { + "post": { + "operationId": "harness.world.refine", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Apply an evaluator-authenticated world-model refinement", + "description": "Applies a small revision-checked patch. Evaluator evidence may raise confidence beyond the agent's self-report ceiling while the immutable base prompt remains unchanged.", + "responses": { + "200": { + "description": "Refined continual world-model state", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "basePromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "entries": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "key": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{0,79}$" + }, + "kind": { + "type": "string", + "enum": [ + "hypothesis", + "observation", + "strategy", + "memory", + "skill", + "subagent" + ] + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "confidence": { + "type": "integer", + "minimum": 1, + "maximum": 5 + }, + "evidence": { + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "authority": { + "type": "string", + "enum": [ + "self", + "tool", + "evaluator", + "human" + ] + } + }, + "required": [ + "ref", + "authority" + ], + "additionalProperties": false + } + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "key", + "kind", + "content", + "confidence", + "evidence", + "updatedAt", + "revision" + ], + "additionalProperties": false + } + }, + "events": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "type": { + "type": "string", + "enum": [ + "analysis", + "tool", + "evaluation", + "failure", + "milestone", + "stagnation", + "manual" + ] + }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "evidenceRefs": { + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "changed": { + "type": "boolean" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "type", + "summary", + "evidenceRefs", + "changed", + "createdAt" + ], + "additionalProperties": false + } + }, + "snapshots": { + "maxItems": 10, + "type": "array", + "items": { + "type": "object", + "properties": { + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "entries": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "key": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{0,79}$" + }, + "kind": { + "type": "string", + "enum": [ + "hypothesis", + "observation", + "strategy", + "memory", + "skill", + "subagent" + ] + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "confidence": { + "type": "integer", + "minimum": 1, + "maximum": 5 + }, + "evidence": { + "maxItems": 16, + "type": "array", + "items": { + "type": "object", + "properties": { + "ref": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "authority": { + "type": "string", + "enum": [ + "self", + "tool", + "evaluator", + "human" + ] + } + }, + "required": [ + "ref", + "authority" + ], + "additionalProperties": false + } + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "key", + "kind", + "content", + "confidence", + "evidence", + "updatedAt", + "revision" + ], + "additionalProperties": false + } + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "revision", + "entries", + "sha256", + "createdAt" + ], + "additionalProperties": false + } + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "contextEpoch": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "eventsSinceRefine": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refinement": { + "type": "object", + "properties": { + "recommended": { + "type": "boolean" + }, + "trigger": { + "type": "string", + "enum": [ + "manual", + "failure", + "stagnation", + "milestone", + "periodic" + ] + } + }, + "required": [ + "recommended" + ], + "additionalProperties": false + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "sessionID", + "runID", + "basePromptSHA256", + "entries", + "events", + "snapshots", + "revision", + "contextEpoch", + "eventsSinceRefine", + "refinement", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "expectedRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reason": { + "type": "string", + "enum": [ + "manual", + "failure", + "stagnation", + "milestone", + "periodic" + ] + }, + "patches": { + "minItems": 1, + "maxItems": 6, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "upsert" + }, + "key": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{0,79}$" + }, + "kind": { + "type": "string", + "enum": [ + "hypothesis", + "observation", + "strategy", + "memory", + "skill", + "subagent" + ] + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "confidence": { + "type": "integer", + "minimum": 1, + "maximum": 5 + }, + "evidenceRefs": { + "default": [], + "maxItems": 16, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "op", + "key", + "kind", + "content", + "confidence" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "const": "remove" + }, + "key": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._:-]{0,79}$" + } + }, + "required": [ + "op", + "key" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "evaluatorToken", + "expectedRevision", + "reason", + "patches" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.world.refine({\n ...\n})" + } + ] + } + }, + "/harness/runs": { + "post": { + "operationId": "harness.bind", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Bind an immutable scientific evaluation run", + "description": "Called by a local or external evaluator before agent execution. The evaluator capability is hashed and never returned.", + "responses": { + "200": { + "description": "Bound harness contract", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "objective": { + "type": "string", + "minLength": 1 + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "title": { + "default": "Scientific evaluation", + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "family": { + "default": "custom", + "type": "string", + "enum": [ + "data", + "biology", + "physics", + "chemistry", + "ml", + "generalist", + "custom" + ] + }, + "task": { + "default": "Scientific evaluation task", + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "split": { + "type": "string", + "enum": [ + "development", + "validation", + "held_out", + "release" + ] + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "evaluatorVersion": { + "type": "string", + "minLength": 1 + }, + "evaluatorSource": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + }, + "fidelities": { + "minItems": 2, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "final": { + "type": "boolean" + }, + "maxWallTimeMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxCostUSD": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "id", + "final" + ], + "additionalProperties": false + } + }, + "metric": { + "type": "string", + "minLength": 1 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + }, + "objectives": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "metric", + "direction" + ], + "additionalProperties": false + } + }, + "objectiveAudit": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "guardIDs": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + } + }, + "required": [ + "schemaVersion", + "planSHA256", + "validatorSHA256", + "contractSHA256", + "guardIDs" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "version", + "taskID", + "split", + "evaluator" + ], + "additionalProperties": false + }, + "profile": { + "type": "string", + "enum": [ + "react", + "optimize", + "reproduce", + "theory", + "numerical", + "training", + "forecast" + ] + }, + "orchestration": { + "type": "object", + "properties": { + "topology": { + "type": "string", + "enum": [ + "auto", + "solo", + "centralized", + "fork_join", + "tournament", + "evolution", + "verifier_loop" + ] + }, + "traits": { + "type": "object", + "properties": { + "decomposability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sequentiality": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "toolIntensity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "verificationRisk": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "crossDomain": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "decomposability", + "sequentiality", + "toolIntensity", + "uncertainty", + "verificationRisk", + "novelty", + "crossDomain" + ], + "additionalProperties": false + }, + "maxWorkers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "maxRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "roles": { + "minItems": 1, + "maxItems": 10, + "type": "array", + "items": { + "type": "string", + "enum": [ + "generation", + "proximity", + "reflection", + "ranking", + "evolution", + "revision", + "verification", + "investigation", + "simulation", + "synthesis" + ] + } + }, + "minIndependentVerifiers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "adaptive": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "marginal-utility-v1" + }, + "minRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "patience": { + "type": "integer", + "minimum": 1, + "maximum": 7 + }, + "minUtilityGain": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetUtility": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "minRounds", + "patience", + "minUtilityGain", + "maxUncertainty" + ], + "additionalProperties": false + }, + "repair": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "verifier-routed-v1" + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "minConfidence" + ], + "additionalProperties": false + } + }, + "required": [ + "topology", + "maxWorkers", + "maxRounds", + "minIndependentVerifiers" + ], + "additionalProperties": false + }, + "search": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "adaptive-search-v1" + }, + "signal": { + "type": "object", + "properties": { + "source": { + "type": "string", + "const": "verified-final-evaluations" + }, + "decay": { + "type": "number", + "const": 0.9 + }, + "epsilon": { + "type": "number", + "const": 1e-8 + } + }, + "required": [ + "source", + "decay", + "epsilon" + ], + "additionalProperties": false + }, + "local": { + "type": "object", + "properties": { + "minIntensity": { + "type": "number", + "const": 0.15 + }, + "maxIntensity": { + "type": "number", + "const": 0.5 + } + }, + "required": [ + "minIntensity", + "maxIntensity" + ], + "additionalProperties": false + }, + "global": { + "type": "object", + "properties": { + "exploration": { + "type": "number", + "const": 1.4142135623730951 + }, + "minVisits": { + "type": "number", + "const": 2 + } + }, + "required": [ + "exploration", + "minVisits" + ], + "additionalProperties": false + }, + "stagnation": { + "type": "object", + "properties": { + "patience": { + "type": "number", + "const": 5 + }, + "maxSignal": { + "type": "number", + "const": 0.02 + } + }, + "required": [ + "patience", + "maxSignal" + ], + "additionalProperties": false + } + }, + "required": [ + "protocolVersion", + "signal", + "local", + "global", + "stagnation" + ], + "additionalProperties": false + }, + "audit": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "performance", + "failure", + "hybrid" + ] + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minSamples": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "noiseVariance": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 2 + }, + "lengthscale": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 100 + }, + "beta": { + "default": 1.96, + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "failureThreshold": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tolerance": { + "default": 0.02, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "estimationWeight": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "diversityWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "coverageWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + }, + "transfer": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "score-history-prior-v1" + }, + "poolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionMethod": { + "type": "string", + "enum": [ + "pca-gmm-profile-v1", + "holdout-embedding-gmm-v1" + ] + }, + "sourceModels": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "calibrationSamples": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "maxCalibrationMAE": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "poolSHA256", + "sourceManifestSHA256", + "selectionSHA256", + "selectionMethod", + "sourceModels", + "calibrationSamples", + "maxCalibrationMAE" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean" + } + }, + "required": [ + "mode", + "budget", + "minSamples" + ], + "additionalProperties": false + }, + "failureDiscovery": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "topicModel": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "predefined", + "bertopic" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + }, + "topics": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "generator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "validators": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + } + }, + "embedding": { + "type": "object", + "properties": { + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "dimensions": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "regularization": { + "default": 0.000001, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.01 + } + }, + "required": [ + "identity", + "dimensions" + ], + "additionalProperties": false + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "anchorsPerAttempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "exploration": { + "default": 1.4142135623730951, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "failureThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + } + }, + "required": [ + "protocolVersion", + "sourcePoolSHA256", + "topicModel", + "topics", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "failureThreshold" + ], + "additionalProperties": false + }, + "integrity": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "benchmark-integrity-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "minCoverage": { + "type": "number", + "minimum": 0.9, + "maximum": 1 + }, + "assignedModel": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "forbiddenModelArtifacts": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "policy": { + "type": "object", + "properties": { + "testItemDerivation": { + "type": "string", + "const": "forbidden" + }, + "unapprovedExternalModels": { + "type": "string", + "const": "forbidden" + }, + "benchmarkLookup": { + "type": "string", + "const": "forbidden" + } + }, + "required": [ + "testItemDerivation", + "unapprovedExternalModels", + "benchmarkLookup" + ], + "additionalProperties": false + }, + "auditors": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256" + ], + "additionalProperties": false + } + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minHiddenCanaries": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "traceSchemaSHA256", + "minEvents", + "minCoverage", + "assignedModel", + "policy", + "auditors", + "hiddenCanaryManifestSHA256", + "minHiddenCanaries" + ], + "additionalProperties": false + }, + "evolution": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evolution-trace-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineAlgorithm": { + "type": "string", + "const": "sha256-exact-line-v1" + }, + "roots": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "extensions": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^\\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}$" + } + }, + "exclude": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 1, + "maximum": 100000 + }, + "maxFileBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1000000000 + }, + "maxTotalBytes": { + "type": "integer", + "minimum": 1, + "maximum": 10000000000 + }, + "maxSourceLines": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "maxChangedLines": { + "type": "integer", + "minimum": 1, + "maximum": 2000000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "manifestSchemaSHA256", + "lineAlgorithm", + "roots", + "extensions", + "maxFiles", + "maxFileBytes", + "maxTotalBytes", + "maxSourceLines", + "maxChangedLines" + ], + "additionalProperties": false + }, + "metaHarness": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "meta-harness-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "archiveSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "baseline": { + "type": "object", + "properties": { + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "artifactSHA256", + "manifestSHA256" + ], + "additionalProperties": false + }, + "mutable": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "root": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "component": { + "type": "string", + "enum": [ + "prompt", + "memory", + "skill", + "tool", + "middleware", + "subagent", + "scaffold" + ] + } + }, + "required": [ + "root", + "component" + ], + "additionalProperties": false + } + }, + "protected": { + "type": "object", + "properties": { + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "roots": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "manifestSHA256", + "roots" + ], + "additionalProperties": false + }, + "archive": { + "type": "object", + "properties": { + "contents": { + "type": "string", + "const": "full-source-scores-traces" + }, + "query": { + "type": "string", + "const": "filesystem" + }, + "summariesOnly": { + "type": "boolean", + "const": false + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "contents", + "query", + "summariesOnly", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "updater": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "judge": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "search": { + "type": "object", + "properties": { + "models": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "tasks": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "activationRequired": { + "type": "boolean" + } + }, + "required": [ + "id", + "commitment", + "activationRequired" + ], + "additionalProperties": false + } + } + }, + "required": [ + "models", + "tasks" + ], + "additionalProperties": false + }, + "heldout": { + "type": "object", + "properties": { + "models": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "tasks": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "activationRequired": { + "type": "boolean" + } + }, + "required": [ + "id", + "commitment", + "activationRequired" + ], + "additionalProperties": false + } + } + }, + "required": [ + "models", + "tasks" + ], + "additionalProperties": false + }, + "thresholds": { + "type": "object", + "properties": { + "minSearchGain": { + "type": "number", + "minimum": 0 + }, + "minHeldoutGain": { + "type": "number", + "minimum": 0 + }, + "maxModelRegression": { + "type": "number", + "minimum": 0 + }, + "minActivationRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minRequiredAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minFinalAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxPhaseDrift": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minPredictionPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxRiskRegressions": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "maxContextTokens": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxMeanContextIncrease": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "minSearchGain", + "minHeldoutGain", + "maxModelRegression", + "minActivationRate", + "minRequiredAdherence", + "minFinalAdherence", + "maxPhaseDrift", + "minPredictionPrecision", + "maxRiskRegressions", + "maxContextTokens", + "maxMeanContextIncrease" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean", + "const": true + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "archiveSchemaSHA256", + "traceSchemaSHA256", + "baseline", + "mutable", + "protected", + "archive", + "updater", + "judge", + "search", + "heldout", + "thresholds", + "promotionRequired" + ], + "additionalProperties": false + }, + "interventions": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "intervention-study-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "requiredForPromotion": { + "type": "boolean" + }, + "minPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxTotalPairs": { + "type": "integer", + "minimum": 3, + "maximum": 256 + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "required": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + } + }, + "rules": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "family": { + "type": "string", + "const": "replay" + }, + "mode": { + "type": "string", + "const": "max_absolute_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "retune", + "ablation", + "repair" + ] + }, + "mode": { + "type": "string", + "const": "min_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "const": "max_regression" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "requiredForPromotion", + "minPairs", + "maxPairs", + "maxTotalPairs", + "confidence", + "required", + "rules" + ], + "additionalProperties": false + }, + "simulation": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "ode", + "pde", + "cfd", + "materials", + "molecular", + "agentic" + ] + }, + "engine": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commandSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commandSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "problemSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reference": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "analytic", + "manufactured", + "benchmark", + "independent_solver", + "limiting_case" + ] + }, + "identity": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "identity", + "sha256" + ], + "additionalProperties": false + }, + "validation": { + "type": "object", + "properties": { + "errorNorm": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "minLevels": { + "type": "integer", + "minimum": 3, + "maximum": 12 + }, + "maxLevels": { + "default": 12, + "type": "integer", + "minimum": 3, + "maximum": 24 + }, + "expectedOrder": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 20 + }, + "orderTolerance": { + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "maxResidual": { + "type": "number", + "minimum": 0 + }, + "invariantTolerances": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "number", + "minimum": 0 + } + }, + "requiredStressTests": { + "minItems": 1, + "maxItems": 7, + "type": "array", + "items": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + } + } + }, + "required": [ + "errorNorm", + "minLevels", + "expectedOrder", + "orderTolerance", + "maxResidual", + "invariantTolerances", + "requiredStressTests" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "engine", + "problemSHA256", + "reference", + "validation" + ], + "additionalProperties": false + }, + "evaluatorAudit": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evaluator-audit-v1" + }, + "auditor": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "suite": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commitmentSHA256" + ], + "additionalProperties": false + }, + "minCleanCases": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minCasesPerFault": { + "type": "integer", + "minimum": 1, + "maximum": 128 + }, + "requiredFaults": { + "minItems": 1, + "maxItems": 10, + "type": "array", + "items": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + } + }, + "minSensitivity": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minSpecificity": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minBalancedAccuracy": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minFaultRecall": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "maxBrierScore": { + "type": "number", + "minimum": 0, + "maximum": 0.5 + } + }, + "required": [ + "protocolVersion", + "auditor", + "suite", + "minCleanCases", + "minCasesPerFault", + "requiredFaults", + "minSensitivity", + "minSpecificity", + "minBalancedAccuracy", + "minFaultRecall", + "maxBrierScore" + ], + "additionalProperties": false + }, + "semanticAudit": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "semantic-audit-v1" + }, + "reviewer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "scope": { + "type": "object", + "properties": { + "objectiveSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "requirement": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "requirement" + ], + "additionalProperties": false + } + }, + "forbiddenShortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "description" + ], + "additionalProperties": false + } + }, + "literature": { + "type": "object", + "properties": { + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "corpusSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "cutoff", + "corpusSHA256" + ], + "additionalProperties": false + }, + "noveltyFloor": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + } + }, + "required": [ + "objectiveSHA256", + "criteria", + "forbiddenShortcuts", + "literature", + "noveltyFloor" + ], + "additionalProperties": false + }, + "minReviewers": { + "type": "integer", + "minimum": 2, + "maximum": 5 + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "reviewer", + "scope", + "minReviewers", + "minConfidence" + ], + "additionalProperties": false + }, + "synthesis": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "scientific-synthesis-v1" + }, + "querySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceFactsSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceFactCount": { + "type": "integer", + "minimum": 1, + "maximum": 2048 + }, + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "tools": { + "minItems": 1, + "maxItems": 3, + "type": "array", + "items": { + "type": "string", + "enum": [ + "google_search", + "paper_search", + "web_browse" + ] + } + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "filterPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "maxToolEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "decomposer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "judges": { + "type": "object", + "properties": { + "precision": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "recall": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "precision", + "recall" + ], + "additionalProperties": false + }, + "minGeneratedFacts": { + "type": "integer", + "minimum": 1, + "maximum": 512 + }, + "minPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minRecall": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minF1": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "cleanRoomRequired": { + "type": "boolean", + "const": true + }, + "judgeFailurePolicy": { + "type": "string", + "const": "inconclusive" + } + }, + "required": [ + "protocolVersion", + "querySHA256", + "referenceSHA256", + "referenceFactsSHA256", + "referenceFactCount", + "cutoff", + "tools", + "traceSchemaSHA256", + "filterPolicySHA256", + "maxToolEvents", + "decomposer", + "judges", + "minGeneratedFacts", + "minPrecision", + "minRecall", + "minF1", + "cleanRoomRequired", + "judgeFailurePolicy" + ], + "additionalProperties": false + }, + "autonomy": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "human-ai-autonomy-v1" + }, + "claimedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "recorder": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "source": { + "type": "string", + "const": "evaluator_runtime" + } + }, + "required": [ + "name", + "version", + "artifactSHA256", + "source" + ], + "additionalProperties": false + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "classificationPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "maxEvents": { + "type": "integer", + "minimum": 2, + "maximum": 10000 + }, + "rawRetention": { + "type": "string", + "const": "required" + }, + "disclosure": { + "type": "string", + "enum": [ + "evaluator_retained", + "public_essential_after_release" + ] + }, + "completeTraceRequired": { + "type": "boolean", + "const": true + }, + "uncertaintyPolicy": { + "type": "string", + "const": "inconclusive" + } + }, + "required": [ + "protocolVersion", + "claimedLevel", + "recorder", + "traceSchemaSHA256", + "classificationPolicySHA256", + "maxEvents", + "rawRetention", + "disclosure", + "completeTraceRequired", + "uncertaintyPolicy" + ], + "additionalProperties": false + }, + "formalProof": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "formal-proof-v1" + }, + "language": { + "type": "string", + "const": "lean4" + }, + "tier": { + "type": "string", + "enum": [ + "kernel", + "fresh_recheck", + "external_crosscheck" + ] + }, + "relation": { + "type": "string", + "enum": [ + "exact_proof", + "exact_refutation", + "repaired_proof" + ] + }, + "challengeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "leanVersion": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "leanToolchainSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lakeManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "dependencyTreeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifiers": { + "minItems": 2, + "maxItems": 6, + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": [ + "lean_kernel", + "source_auditor", + "axiom_auditor", + "fresh_rechecker", + "sandbox_comparator", + "external_checker" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "role", + "name", + "version", + "artifactSHA256" + ], + "additionalProperties": false + } + }, + "sandboxImageSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "forbiddenConstructs": { + "minItems": 4, + "maxItems": 4, + "type": "array", + "items": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + } + }, + "allowedAxioms": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 6, + "maximum": 10000 + }, + "completeManifestRequired": { + "type": "boolean", + "const": true + }, + "warningPolicy": { + "type": "string", + "const": "fail" + }, + "semanticPolicy": { + "type": "string", + "const": "formal_statement_only" + }, + "blueprint": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "proof-blueprint-v1" + }, + "graphSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "compilerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchValidatorArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "nodePolicy": { + "type": "string", + "const": "and-or-monotone-v1" + }, + "failurePolicy": { + "type": "string", + "const": "preserve-and-refine" + }, + "memoization": { + "type": "string", + "const": "goal-sha256" + }, + "finalAuthority": { + "type": "string", + "const": "formal-proof-v1" + }, + "directAttemptFirst": { + "type": "boolean", + "const": true + }, + "verifiedSketchRequired": { + "type": "boolean", + "const": true + }, + "completeFailureHistoryRequired": { + "type": "boolean", + "const": true + }, + "maxNodes": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "maxDepth": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "maxParallel": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "maxAttemptsPerGoal": { + "type": "integer", + "minimum": 1, + "maximum": 16 + }, + "maxRefinementsPerGoal": { + "type": "integer", + "minimum": 0, + "maximum": 16 + }, + "leaseDurationMs": { + "type": "integer", + "minimum": 1000, + "maximum": 3600000 + } + }, + "required": [ + "protocolVersion", + "graphSchemaSHA256", + "compilerArtifactSHA256", + "sketchValidatorArtifactSHA256", + "reviewerArtifactSHA256", + "reviewerPromptSHA256", + "nodePolicy", + "failurePolicy", + "memoization", + "finalAuthority", + "directAttemptFirst", + "verifiedSketchRequired", + "completeFailureHistoryRequired", + "maxNodes", + "maxDepth", + "maxParallel", + "maxAttemptsPerGoal", + "maxRefinementsPerGoal", + "leaseDurationMs" + ], + "additionalProperties": false + } + }, + "required": [ + "protocolVersion", + "language", + "tier", + "relation", + "challengeSHA256", + "statementSHA256", + "declaration", + "module", + "leanVersion", + "leanToolchainSHA256", + "lakeManifestSHA256", + "dependencyTreeSHA256", + "verifiers", + "forbiddenConstructs", + "allowedAxioms", + "maxFiles", + "completeManifestRequired", + "warningPolicy", + "semanticPolicy" + ], + "additionalProperties": false + }, + "replication": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "replicated-evaluation-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sampling": { + "type": "object", + "properties": { + "design": { + "type": "string", + "const": "crossed-stratified-cluster-v1" + }, + "stratumKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "strata": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + }, + "clusters": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "design", + "stratumKind", + "clusterKind", + "strata", + "clusters" + ], + "additionalProperties": false + }, + "estimator": { + "type": "string", + "enum": [ + "mean", + "median", + "iqm", + "pass_rate" + ] + }, + "interval": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "stratified-bootstrap-percentile-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "resamples": { + "type": "integer", + "minimum": 1000, + "maximum": 50000 + }, + "seed": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + }, + "required": [ + "method", + "confidence", + "resamples", + "seed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "wilson-score-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + } + }, + "required": [ + "method", + "confidence" + ], + "additionalProperties": false + } + ] + }, + "decision": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "conservative-bound-v1" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + }, + "maxIntervalWidth": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "rule", + "direction", + "target" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "environmentSHA256", + "sampling", + "estimator", + "interval", + "decision", + "failurePolicy" + ], + "additionalProperties": false + }, + "confirmation": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "sealed-confirmation-v1" + }, + "optimization": { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "development", + "validation" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "split", + "manifestSHA256" + ], + "additionalProperties": false + }, + "claim": { + "type": "object", + "properties": { + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "format": "uri" + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "repository", + "revision" + ], + "additionalProperties": false + }, + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + }, + "target": { + "type": "number" + } + }, + "required": [ + "taskID", + "split", + "manifestSHA256", + "validatorSHA256", + "environmentSHA256", + "evaluator", + "metric", + "direction", + "target" + ], + "additionalProperties": false + }, + "selection": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "terminal-verified-best-v1" + }, + "subjects": { + "type": "number", + "const": 1 + } + }, + "required": [ + "rule", + "subjects" + ], + "additionalProperties": false + }, + "exposure": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "const": "terminal-receipt-only" + }, + "searchFeedback": { + "type": "boolean", + "const": false + }, + "memoryCapture": { + "type": "boolean", + "const": false + } + }, + "required": [ + "policy", + "searchFeedback", + "memoryCapture" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "optimization", + "claim", + "selection", + "exposure", + "failurePolicy" + ], + "additionalProperties": false + }, + "packs": { + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "statistics", + "biology", + "physics", + "pde", + "chemistry", + "ml", + "forecast", + "formal" + ] + } + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "effort": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "provider", + "name" + ], + "additionalProperties": false + }, + "tools": { + "default": [], + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "skills": { + "default": [], + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + }, + "budget": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "steps": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "candidates": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "cpuHours": { + "type": "number", + "minimum": 0 + }, + "gpuHours": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "intervention": { + "type": "string", + "enum": [ + "autonomous", + "human_reprompted" + ] + }, + "contamination": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "minLength": 1 + }, + "hiddenTestsAccessible": { + "type": "boolean", + "const": false + }, + "publicDataCutoff": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "policy", + "hiddenTestsAccessible" + ], + "additionalProperties": false + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "objective", + "benchmark", + "profile", + "model", + "budget", + "seed", + "intervention", + "contamination", + "createdAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "benchmark": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "title": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "family": { + "default": "custom", + "type": "string", + "enum": [ + "data", + "biology", + "physics", + "chemistry", + "ml", + "generalist", + "custom" + ] + }, + "task": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "split": { + "type": "string", + "enum": [ + "development", + "validation", + "held_out", + "release" + ] + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + }, + "token": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "name", + "version", + "source", + "token" + ], + "additionalProperties": false + }, + "objective": { + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "profile": { + "type": "string", + "enum": [ + "react", + "optimize", + "reproduce", + "theory", + "numerical", + "training", + "forecast" + ] + }, + "search": { + "type": "string", + "enum": [ + "adaptive", + "static" + ] + }, + "orchestration": { + "type": "object", + "properties": { + "topology": { + "type": "string", + "enum": [ + "auto", + "solo", + "centralized", + "fork_join", + "tournament", + "evolution", + "verifier_loop" + ] + }, + "traits": { + "type": "object", + "properties": { + "decomposability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sequentiality": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "toolIntensity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "verificationRisk": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "crossDomain": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "decomposability", + "sequentiality", + "toolIntensity", + "uncertainty", + "verificationRisk", + "novelty", + "crossDomain" + ], + "additionalProperties": false + }, + "maxWorkers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "maxRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "roles": { + "minItems": 1, + "maxItems": 10, + "type": "array", + "items": { + "type": "string", + "enum": [ + "generation", + "proximity", + "reflection", + "ranking", + "evolution", + "revision", + "verification", + "investigation", + "simulation", + "synthesis" + ] + } + }, + "minIndependentVerifiers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "adaptive": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "marginal-utility-v1" + }, + "minRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "patience": { + "type": "integer", + "minimum": 1, + "maximum": 7 + }, + "minUtilityGain": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetUtility": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "minRounds", + "patience", + "minUtilityGain", + "maxUncertainty" + ], + "additionalProperties": false + }, + "repair": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "verifier-routed-v1" + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "minConfidence" + ], + "additionalProperties": false + } + }, + "required": [ + "topology", + "maxWorkers", + "maxRounds", + "minIndependentVerifiers" + ], + "additionalProperties": false + }, + "audit": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "performance", + "failure", + "hybrid" + ] + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minSamples": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "noiseVariance": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 2 + }, + "lengthscale": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 100 + }, + "beta": { + "default": 1.96, + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "failureThreshold": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tolerance": { + "default": 0.02, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "estimationWeight": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "diversityWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "coverageWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + }, + "transfer": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "score-history-prior-v1" + }, + "poolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionMethod": { + "type": "string", + "enum": [ + "pca-gmm-profile-v1", + "holdout-embedding-gmm-v1" + ] + }, + "sourceModels": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "calibrationSamples": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "maxCalibrationMAE": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "poolSHA256", + "sourceManifestSHA256", + "selectionSHA256", + "selectionMethod", + "sourceModels", + "calibrationSamples", + "maxCalibrationMAE" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean" + } + }, + "required": [ + "mode", + "budget", + "minSamples" + ], + "additionalProperties": false + }, + "failureDiscovery": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "topicModel": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "predefined", + "bertopic" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + }, + "topics": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "generator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "validators": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + } + }, + "embedding": { + "type": "object", + "properties": { + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "dimensions": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "regularization": { + "default": 0.000001, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.01 + } + }, + "required": [ + "identity", + "dimensions" + ], + "additionalProperties": false + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "anchorsPerAttempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "exploration": { + "default": 1.4142135623730951, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "failureThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + } + }, + "required": [ + "protocolVersion", + "sourcePoolSHA256", + "topicModel", + "topics", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "failureThreshold" + ], + "additionalProperties": false + }, + "integrity": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "benchmark-integrity-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "minCoverage": { + "type": "number", + "minimum": 0.9, + "maximum": 1 + }, + "assignedModel": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "forbiddenModelArtifacts": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "policy": { + "type": "object", + "properties": { + "testItemDerivation": { + "type": "string", + "const": "forbidden" + }, + "unapprovedExternalModels": { + "type": "string", + "const": "forbidden" + }, + "benchmarkLookup": { + "type": "string", + "const": "forbidden" + } + }, + "required": [ + "testItemDerivation", + "unapprovedExternalModels", + "benchmarkLookup" + ], + "additionalProperties": false + }, + "auditors": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256" + ], + "additionalProperties": false + } + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minHiddenCanaries": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "traceSchemaSHA256", + "minEvents", + "minCoverage", + "assignedModel", + "policy", + "auditors", + "hiddenCanaryManifestSHA256", + "minHiddenCanaries" + ], + "additionalProperties": false + }, + "evolution": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evolution-trace-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineAlgorithm": { + "type": "string", + "const": "sha256-exact-line-v1" + }, + "roots": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "extensions": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^\\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}$" + } + }, + "exclude": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 1, + "maximum": 100000 + }, + "maxFileBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1000000000 + }, + "maxTotalBytes": { + "type": "integer", + "minimum": 1, + "maximum": 10000000000 + }, + "maxSourceLines": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "maxChangedLines": { + "type": "integer", + "minimum": 1, + "maximum": 2000000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "manifestSchemaSHA256", + "lineAlgorithm", + "roots", + "extensions", + "maxFiles", + "maxFileBytes", + "maxTotalBytes", + "maxSourceLines", + "maxChangedLines" + ], + "additionalProperties": false + }, + "metaHarness": { + "type": "object", + "properties": { + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "meta-harness-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "archiveSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "baseline": { + "type": "object", + "properties": { + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "artifactSHA256", + "manifestSHA256" + ], + "additionalProperties": false + }, + "mutable": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "root": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "component": { + "type": "string", + "enum": [ + "prompt", + "memory", + "skill", + "tool", + "middleware", + "subagent", + "scaffold" + ] + } + }, + "required": [ + "root", + "component" + ], + "additionalProperties": false + } + }, + "protected": { + "type": "object", + "properties": { + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "roots": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "manifestSHA256", + "roots" + ], + "additionalProperties": false + }, + "archive": { + "type": "object", + "properties": { + "contents": { + "type": "string", + "const": "full-source-scores-traces" + }, + "query": { + "type": "string", + "const": "filesystem" + }, + "summariesOnly": { + "type": "boolean", + "const": false + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "contents", + "query", + "summariesOnly", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "updater": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "judge": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "search": { + "type": "object", + "properties": { + "models": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "tasks": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "activationRequired": { + "type": "boolean" + } + }, + "required": [ + "id", + "commitment", + "activationRequired" + ], + "additionalProperties": false + } + } + }, + "required": [ + "models", + "tasks" + ], + "additionalProperties": false + }, + "heldout": { + "type": "object", + "properties": { + "models": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "tasks": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "activationRequired": { + "type": "boolean" + } + }, + "required": [ + "id", + "commitment", + "activationRequired" + ], + "additionalProperties": false + } + } + }, + "required": [ + "models", + "tasks" + ], + "additionalProperties": false + }, + "thresholds": { + "type": "object", + "properties": { + "minSearchGain": { + "type": "number", + "minimum": 0 + }, + "minHeldoutGain": { + "type": "number", + "minimum": 0 + }, + "maxModelRegression": { + "type": "number", + "minimum": 0 + }, + "minActivationRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minRequiredAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minFinalAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxPhaseDrift": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minPredictionPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxRiskRegressions": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "maxContextTokens": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxMeanContextIncrease": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "minSearchGain", + "minHeldoutGain", + "maxModelRegression", + "minActivationRate", + "minRequiredAdherence", + "minFinalAdherence", + "maxPhaseDrift", + "minPredictionPrecision", + "maxRiskRegressions", + "maxContextTokens", + "maxMeanContextIncrease" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean", + "const": true + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "archiveSchemaSHA256", + "traceSchemaSHA256", + "baseline", + "mutable", + "protected", + "archive", + "updater", + "judge", + "search", + "heldout", + "thresholds", + "promotionRequired" + ], + "additionalProperties": false + }, + "token": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "protocol", + "token" + ], + "additionalProperties": false + }, + "interventions": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "intervention-study-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "requiredForPromotion": { + "type": "boolean" + }, + "minPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxTotalPairs": { + "type": "integer", + "minimum": 3, + "maximum": 256 + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "required": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + } + }, + "rules": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "family": { + "type": "string", + "const": "replay" + }, + "mode": { + "type": "string", + "const": "max_absolute_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "retune", + "ablation", + "repair" + ] + }, + "mode": { + "type": "string", + "const": "min_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "const": "max_regression" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "requiredForPromotion", + "minPairs", + "maxPairs", + "maxTotalPairs", + "confidence", + "required", + "rules" + ], + "additionalProperties": false + }, + "simulation": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "ode", + "pde", + "cfd", + "materials", + "molecular", + "agentic" + ] + }, + "engine": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commandSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commandSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "problemSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reference": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "analytic", + "manufactured", + "benchmark", + "independent_solver", + "limiting_case" + ] + }, + "identity": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "identity", + "sha256" + ], + "additionalProperties": false + }, + "validation": { + "type": "object", + "properties": { + "errorNorm": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "minLevels": { + "type": "integer", + "minimum": 3, + "maximum": 12 + }, + "maxLevels": { + "default": 12, + "type": "integer", + "minimum": 3, + "maximum": 24 + }, + "expectedOrder": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 20 + }, + "orderTolerance": { + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "maxResidual": { + "type": "number", + "minimum": 0 + }, + "invariantTolerances": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "number", + "minimum": 0 + } + }, + "requiredStressTests": { + "minItems": 1, + "maxItems": 7, + "type": "array", + "items": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + } + } + }, + "required": [ + "errorNorm", + "minLevels", + "expectedOrder", + "orderTolerance", + "maxResidual", + "invariantTolerances", + "requiredStressTests" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "engine", + "problemSHA256", + "reference", + "validation" + ], + "additionalProperties": false + }, + "evaluatorAudit": { + "type": "object", + "properties": { + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evaluator-audit-v1" + }, + "auditor": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "suite": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commitmentSHA256" + ], + "additionalProperties": false + }, + "minCleanCases": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minCasesPerFault": { + "type": "integer", + "minimum": 1, + "maximum": 128 + }, + "requiredFaults": { + "minItems": 1, + "maxItems": 10, + "type": "array", + "items": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + } + }, + "minSensitivity": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minSpecificity": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minBalancedAccuracy": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minFaultRecall": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "maxBrierScore": { + "type": "number", + "minimum": 0, + "maximum": 0.5 + } + }, + "required": [ + "protocolVersion", + "auditor", + "suite", + "minCleanCases", + "minCasesPerFault", + "requiredFaults", + "minSensitivity", + "minSpecificity", + "minBalancedAccuracy", + "minFaultRecall", + "maxBrierScore" + ], + "additionalProperties": false + }, + "token": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "protocol", + "token" + ], + "additionalProperties": false + }, + "semanticAudit": { + "type": "object", + "properties": { + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "semantic-audit-v1" + }, + "reviewer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "scope": { + "type": "object", + "properties": { + "objectiveSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "requirement": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "requirement" + ], + "additionalProperties": false + } + }, + "forbiddenShortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "description" + ], + "additionalProperties": false + } + }, + "literature": { + "type": "object", + "properties": { + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "corpusSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "cutoff", + "corpusSHA256" + ], + "additionalProperties": false + }, + "noveltyFloor": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + } + }, + "required": [ + "objectiveSHA256", + "criteria", + "forbiddenShortcuts", + "literature", + "noveltyFloor" + ], + "additionalProperties": false + }, + "minReviewers": { + "type": "integer", + "minimum": 2, + "maximum": 5 + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "reviewer", + "scope", + "minReviewers", + "minConfidence" + ], + "additionalProperties": false + }, + "token": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "protocol", + "token" + ], + "additionalProperties": false + }, + "synthesis": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "scientific-synthesis-v1" + }, + "querySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceFactsSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceFactCount": { + "type": "integer", + "minimum": 1, + "maximum": 2048 + }, + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "tools": { + "minItems": 1, + "maxItems": 3, + "type": "array", + "items": { + "type": "string", + "enum": [ + "google_search", + "paper_search", + "web_browse" + ] + } + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "filterPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "maxToolEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "decomposer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "judges": { + "type": "object", + "properties": { + "precision": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "recall": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "precision", + "recall" + ], + "additionalProperties": false + }, + "minGeneratedFacts": { + "type": "integer", + "minimum": 1, + "maximum": 512 + }, + "minPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minRecall": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minF1": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "cleanRoomRequired": { + "type": "boolean", + "const": true + }, + "judgeFailurePolicy": { + "type": "string", + "const": "inconclusive" + } + }, + "required": [ + "protocolVersion", + "querySHA256", + "referenceSHA256", + "referenceFactsSHA256", + "referenceFactCount", + "cutoff", + "tools", + "traceSchemaSHA256", + "filterPolicySHA256", + "maxToolEvents", + "decomposer", + "judges", + "minGeneratedFacts", + "minPrecision", + "minRecall", + "minF1", + "cleanRoomRequired", + "judgeFailurePolicy" + ], + "additionalProperties": false + }, + "autonomy": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "human-ai-autonomy-v1" + }, + "claimedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "recorder": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "source": { + "type": "string", + "const": "evaluator_runtime" + } + }, + "required": [ + "name", + "version", + "artifactSHA256", + "source" + ], + "additionalProperties": false + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "classificationPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "maxEvents": { + "type": "integer", + "minimum": 2, + "maximum": 10000 + }, + "rawRetention": { + "type": "string", + "const": "required" + }, + "disclosure": { + "type": "string", + "enum": [ + "evaluator_retained", + "public_essential_after_release" + ] + }, + "completeTraceRequired": { + "type": "boolean", + "const": true + }, + "uncertaintyPolicy": { + "type": "string", + "const": "inconclusive" + } + }, + "required": [ + "protocolVersion", + "claimedLevel", + "recorder", + "traceSchemaSHA256", + "classificationPolicySHA256", + "maxEvents", + "rawRetention", + "disclosure", + "completeTraceRequired", + "uncertaintyPolicy" + ], + "additionalProperties": false + }, + "formalProof": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "formal-proof-v1" + }, + "language": { + "type": "string", + "const": "lean4" + }, + "tier": { + "type": "string", + "enum": [ + "kernel", + "fresh_recheck", + "external_crosscheck" + ] + }, + "relation": { + "type": "string", + "enum": [ + "exact_proof", + "exact_refutation", + "repaired_proof" + ] + }, + "challengeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "leanVersion": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "leanToolchainSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lakeManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "dependencyTreeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifiers": { + "minItems": 2, + "maxItems": 6, + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": [ + "lean_kernel", + "source_auditor", + "axiom_auditor", + "fresh_rechecker", + "sandbox_comparator", + "external_checker" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "role", + "name", + "version", + "artifactSHA256" + ], + "additionalProperties": false + } + }, + "sandboxImageSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "forbiddenConstructs": { + "minItems": 4, + "maxItems": 4, + "type": "array", + "items": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + } + }, + "allowedAxioms": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 6, + "maximum": 10000 + }, + "completeManifestRequired": { + "type": "boolean", + "const": true + }, + "warningPolicy": { + "type": "string", + "const": "fail" + }, + "semanticPolicy": { + "type": "string", + "const": "formal_statement_only" + }, + "blueprint": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "proof-blueprint-v1" + }, + "graphSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "compilerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchValidatorArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "nodePolicy": { + "type": "string", + "const": "and-or-monotone-v1" + }, + "failurePolicy": { + "type": "string", + "const": "preserve-and-refine" + }, + "memoization": { + "type": "string", + "const": "goal-sha256" + }, + "finalAuthority": { + "type": "string", + "const": "formal-proof-v1" + }, + "directAttemptFirst": { + "type": "boolean", + "const": true + }, + "verifiedSketchRequired": { + "type": "boolean", + "const": true + }, + "completeFailureHistoryRequired": { + "type": "boolean", + "const": true + }, + "maxNodes": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "maxDepth": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "maxParallel": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "maxAttemptsPerGoal": { + "type": "integer", + "minimum": 1, + "maximum": 16 + }, + "maxRefinementsPerGoal": { + "type": "integer", + "minimum": 0, + "maximum": 16 + }, + "leaseDurationMs": { + "type": "integer", + "minimum": 1000, + "maximum": 3600000 + } + }, + "required": [ + "protocolVersion", + "graphSchemaSHA256", + "compilerArtifactSHA256", + "sketchValidatorArtifactSHA256", + "reviewerArtifactSHA256", + "reviewerPromptSHA256", + "nodePolicy", + "failurePolicy", + "memoization", + "finalAuthority", + "directAttemptFirst", + "verifiedSketchRequired", + "completeFailureHistoryRequired", + "maxNodes", + "maxDepth", + "maxParallel", + "maxAttemptsPerGoal", + "maxRefinementsPerGoal", + "leaseDurationMs" + ], + "additionalProperties": false + } + }, + "required": [ + "protocolVersion", + "language", + "tier", + "relation", + "challengeSHA256", + "statementSHA256", + "declaration", + "module", + "leanVersion", + "leanToolchainSHA256", + "lakeManifestSHA256", + "dependencyTreeSHA256", + "verifiers", + "forbiddenConstructs", + "allowedAxioms", + "maxFiles", + "completeManifestRequired", + "warningPolicy", + "semanticPolicy" + ], + "additionalProperties": false + }, + "replication": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "replicated-evaluation-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sampling": { + "type": "object", + "properties": { + "design": { + "type": "string", + "const": "crossed-stratified-cluster-v1" + }, + "stratumKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "strata": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + }, + "clusters": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "design", + "stratumKind", + "clusterKind", + "strata", + "clusters" + ], + "additionalProperties": false + }, + "estimator": { + "type": "string", + "enum": [ + "mean", + "median", + "iqm", + "pass_rate" + ] + }, + "interval": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "stratified-bootstrap-percentile-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "resamples": { + "type": "integer", + "minimum": 1000, + "maximum": 50000 + }, + "seed": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + }, + "required": [ + "method", + "confidence", + "resamples", + "seed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "wilson-score-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + } + }, + "required": [ + "method", + "confidence" + ], + "additionalProperties": false + } + ] + }, + "decision": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "conservative-bound-v1" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + }, + "maxIntervalWidth": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "rule", + "direction", + "target" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "environmentSHA256", + "sampling", + "estimator", + "interval", + "decision", + "failurePolicy" + ], + "additionalProperties": false + }, + "confirmation": { + "type": "object", + "properties": { + "protocol": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "sealed-confirmation-v1" + }, + "optimization": { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "development", + "validation" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "split", + "manifestSHA256" + ], + "additionalProperties": false + }, + "claim": { + "type": "object", + "properties": { + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "format": "uri" + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "repository", + "revision" + ], + "additionalProperties": false + }, + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + }, + "target": { + "type": "number" + } + }, + "required": [ + "taskID", + "split", + "manifestSHA256", + "validatorSHA256", + "environmentSHA256", + "evaluator", + "metric", + "direction", + "target" + ], + "additionalProperties": false + }, + "selection": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "terminal-verified-best-v1" + }, + "subjects": { + "type": "number", + "const": 1 + } + }, + "required": [ + "rule", + "subjects" + ], + "additionalProperties": false + }, + "exposure": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "const": "terminal-receipt-only" + }, + "searchFeedback": { + "type": "boolean", + "const": false + }, + "memoryCapture": { + "type": "boolean", + "const": false + } + }, + "required": [ + "policy", + "searchFeedback", + "memoryCapture" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "optimization", + "claim", + "selection", + "exposure", + "failurePolicy" + ], + "additionalProperties": false + }, + "token": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "protocol", + "token" + ], + "additionalProperties": false + }, + "packs": { + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "statistics", + "biology", + "physics", + "pde", + "chemistry", + "ml", + "forecast", + "formal" + ] + } + }, + "metric": { + "default": { + "direction": "pass" + }, + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + } + }, + "required": [ + "direction" + ], + "additionalProperties": false + }, + "objectives": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "metric", + "direction" + ], + "additionalProperties": false + } + }, + "objectiveAudit": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "guardIDs": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + } + }, + "required": [ + "schemaVersion", + "planSHA256", + "validatorSHA256", + "contractSHA256", + "guardIDs" + ], + "additionalProperties": false + }, + "fidelities": { + "minItems": 2, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "final": { + "type": "boolean" + }, + "maxWallTimeMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxCostUSD": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "id", + "final" + ], + "additionalProperties": false + } + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "effort": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "provider", + "name" + ], + "additionalProperties": false + }, + "tools": { + "default": [], + "maxItems": 256, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "skills": { + "default": [], + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + }, + "budget": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "steps": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "candidates": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "cpuHours": { + "type": "number", + "minimum": 0 + }, + "gpuHours": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "intervention": { + "type": "string", + "enum": [ + "autonomous", + "human_reprompted" + ] + }, + "contamination": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "minLength": 1, + "maxLength": 2000 + }, + "hiddenTestsAccessible": { + "type": "boolean", + "const": false + }, + "publicDataCutoff": { + "type": "string", + "minLength": 1, + "maxLength": 120 + } + }, + "required": [ + "policy", + "hiddenTestsAccessible" + ], + "additionalProperties": false + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "benchmark", + "version", + "taskID", + "split", + "evaluator", + "objective", + "model", + "budget", + "seed", + "intervention", + "contamination", + "createdAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.bind({\n ...\n})" + } + ] + } + }, + "/harness/evaluations": { + "post": { + "operationId": "harness.evaluate", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Ingest an evaluator-authenticated result", + "description": "Records an immutable subject result, promotes a verified search candidate, and captures task-scoped hindsight.", + "responses": { + "200": { + "description": "Recorded external evaluation" + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "sessionID": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + }, + "candidateID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "stage": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "simulationReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "integrityReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "interventionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatorAuditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "semanticReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "replicationReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "failureDiscoveryReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "synthesisReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "autonomyReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "proofReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "metrics": { + "default": {}, + "type": "object", + "propertyNames": { + "type": "string", + "maxLength": 200 + }, + "additionalProperties": { + "type": "number" + } + }, + "checks": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "blocking": { + "type": "boolean" + }, + "score": { + "type": "number" + }, + "evidence": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "id", + "status", + "blocking" + ], + "additionalProperties": false + } + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "usage": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "number", + "minimum": 0 + }, + "costUSD": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "notes": { + "type": "string", + "maxLength": 8000 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "evaluatorToken", + "status", + "checks", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.evaluate({\n ...\n})" + } + ] + } + }, + "/harness/compare": { + "post": { + "operationId": "harness.compare", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Compare compatible scientific evaluation runs", + "description": "Reports direction-aware deltas and the quality-cost Pareto frontier.", + "responses": { + "200": { + "description": "Comparable run deltas" + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessionIDs": { + "minItems": 2, + "maxItems": 100, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "baselineRunID": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "sessionIDs", + "baselineRunID" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.compare({\n ...\n})" + } + ] + } + }, + "/harness/skills": { + "get": { + "operationId": "harness.skills", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "List quarantined learned skill proposals", + "responses": { + "200": { + "description": "Learned skill qualification manifests", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "name": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "contentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "origin": { + "type": "string", + "enum": [ + "conversation", + "rsi" + ] + }, + "source": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "pending", + "qualified", + "promoted", + "rejected" + ] + }, + "evidence": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "proposalSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "metric": { + "type": "string" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + } + }, + "required": [ + "name", + "version", + "taskID", + "split", + "direction" + ], + "additionalProperties": false + }, + "candidate": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256" + ], + "additionalProperties": false + }, + "control": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256" + ], + "additionalProperties": false + }, + "nonregressing": { + "type": "boolean" + }, + "improved": { + "type": "boolean" + }, + "trigger": { + "type": "object", + "properties": { + "datasetSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "split": { + "type": "string", + "const": "held_out" + }, + "examples": { + "type": "integer", + "minimum": 20, + "maximum": 9007199254740991 + }, + "truePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falsePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "trueNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falseNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "precision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "recall": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "datasetSHA256", + "split", + "examples", + "truePositive", + "falsePositive", + "trueNegative", + "falseNegative", + "precision", + "recall" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "name", + "version" + ], + "additionalProperties": false + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "proposalSHA256", + "benchmark", + "candidate", + "control", + "nonregressing", + "improved", + "trigger", + "evaluator", + "recordedAt" + ], + "additionalProperties": false + } + }, + "criteria": { + "type": "object", + "properties": { + "tasks": { + "type": "number", + "const": 3 + }, + "improvements": { + "type": "number", + "const": 2 + }, + "triggerPrecision": { + "type": "number", + "const": 0.8 + }, + "triggerRecall": { + "type": "number", + "const": 0.8 + } + }, + "required": [ + "tasks", + "improvements", + "triggerPrecision", + "triggerRecall" + ], + "additionalProperties": false + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "promotedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "name", + "description", + "contentSHA256", + "origin", + "source", + "status", + "evidence", + "criteria", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.skills({\n ...\n})" + } + ] + }, + "post": { + "operationId": "harness.skill.propose", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Create an inactive learned skill proposal", + "responses": { + "200": { + "description": "Quarantined proposal", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "name": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "contentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "origin": { + "type": "string", + "enum": [ + "conversation", + "rsi" + ] + }, + "source": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "pending", + "qualified", + "promoted", + "rejected" + ] + }, + "evidence": { + "maxItems": 100, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "proposalSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "metric": { + "type": "string" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + } + }, + "required": [ + "name", + "version", + "taskID", + "split", + "direction" + ], + "additionalProperties": false + }, + "candidate": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256" + ], + "additionalProperties": false + }, + "control": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "evaluationSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "sessionID", + "runID", + "status", + "evaluationSHA256" + ], + "additionalProperties": false + }, + "nonregressing": { + "type": "boolean" + }, + "improved": { + "type": "boolean" + }, + "trigger": { + "type": "object", + "properties": { + "datasetSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "split": { + "type": "string", + "const": "held_out" + }, + "examples": { + "type": "integer", + "minimum": 20, + "maximum": 9007199254740991 + }, + "truePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falsePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "trueNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falseNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "precision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "recall": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "datasetSHA256", + "split", + "examples", + "truePositive", + "falsePositive", + "trueNegative", + "falseNegative", + "precision", + "recall" + ], + "additionalProperties": false + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "name", + "version" + ], + "additionalProperties": false + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "id", + "proposalSHA256", + "benchmark", + "candidate", + "control", + "nonregressing", + "improved", + "trigger", + "evaluator", + "recordedAt" + ], + "additionalProperties": false + } + }, + "criteria": { + "type": "object", + "properties": { + "tasks": { + "type": "number", + "const": 3 + }, + "improvements": { + "type": "number", + "const": 2 + }, + "triggerPrecision": { + "type": "number", + "const": 0.8 + }, + "triggerRecall": { + "type": "number", + "const": 0.8 + } + }, + "required": [ + "tasks", + "improvements", + "triggerPrecision", + "triggerRecall" + ], + "additionalProperties": false + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "updatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "promotedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "name", + "description", + "contentSHA256", + "origin", + "source", + "status", + "evidence", + "criteria", + "createdAt", + "updatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 64000 + }, + "origin": { + "type": "string", + "enum": [ + "conversation", + "rsi" + ] + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "name", + "description", + "content", + "origin" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.skill.propose({\n ...\n})" + } + ] + } + }, + "/harness/skills/evidence": { + "post": { + "operationId": "harness.skill.attest", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + } + ], + "summary": "Attach paired held-out skill evidence", + "description": "Requires both evaluator capabilities and accepts only otherwise-identical candidate/control contracts.", + "responses": { + "200": { + "description": "Updated qualification state" + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" + }, + "candidate": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + }, + "control": { + "type": "object", + "properties": { + "sessionID": { + "type": "string", + "minLength": 1 + }, + "evaluatorToken": { + "type": "string", + "minLength": 32, + "maxLength": 1024 + } + }, + "required": [ + "sessionID", + "evaluatorToken" + ], + "additionalProperties": false + }, + "trigger": { + "type": "object", + "properties": { + "datasetSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "split": { + "type": "string", + "const": "held_out" + }, + "examples": { + "type": "integer", + "minimum": 20, + "maximum": 9007199254740991 + }, + "truePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falsePositive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "trueNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "falseNegative": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "datasetSHA256", + "split", + "examples", + "truePositive", + "falsePositive", + "trueNegative", + "falseNegative" + ], + "additionalProperties": false + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "name", + "candidate", + "control", + "trigger" + ], + "additionalProperties": false + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.skill.attest({\n ...\n})" + } + ] + } + }, + "/harness/skills/{name}/promotion": { + "post": { + "operationId": "harness.skill.promote", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "name", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Promote a qualified learned skill", + "description": "Copies only an unchanged proposal that has met every held-out qualification criterion.", + "responses": { + "200": { + "description": "Promoted skill" + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.skill.promote({\n ...\n})" + } + ] + } + }, + "/harness/runs/{sessionID}/contract": { + "get": { + "operationId": "harness.contract", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Read a bound harness contract", + "responses": { + "200": { + "description": "Harness contract", + "content": { + "application/json": { + "schema": { + "anyOf": [ + { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "objective": { + "type": "string", + "minLength": 1 + }, + "benchmark": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "title": { + "default": "Scientific evaluation", + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "family": { + "default": "custom", + "type": "string", + "enum": [ + "data", + "biology", + "physics", + "chemistry", + "ml", + "generalist", + "custom" + ] + }, + "task": { + "default": "Scientific evaluation task", + "type": "string", + "minLength": 1, + "maxLength": 4000 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "split": { + "type": "string", + "enum": [ + "development", + "validation", + "held_out", + "release" + ] + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "evaluatorVersion": { + "type": "string", + "minLength": 1 + }, + "evaluatorSource": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + }, + "fidelities": { + "minItems": 2, + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "final": { + "type": "boolean" + }, + "maxWallTimeMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxCostUSD": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "id", + "final" + ], + "additionalProperties": false + } + }, + "metric": { + "type": "string", + "minLength": 1 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + }, + "objectives": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "metric", + "direction" + ], + "additionalProperties": false + } + }, + "objectiveAudit": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "guardIDs": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + } + }, + "required": [ + "schemaVersion", + "planSHA256", + "validatorSHA256", + "contractSHA256", + "guardIDs" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "version", + "taskID", + "split", + "evaluator" + ], + "additionalProperties": false + }, + "profile": { + "type": "string", + "enum": [ + "react", + "optimize", + "reproduce", + "theory", + "numerical", + "training", + "forecast" + ] + }, + "orchestration": { + "type": "object", + "properties": { + "topology": { + "type": "string", + "enum": [ + "auto", + "solo", + "centralized", + "fork_join", + "tournament", + "evolution", + "verifier_loop" + ] + }, + "traits": { + "type": "object", + "properties": { + "decomposability": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "sequentiality": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "toolIntensity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "uncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "verificationRisk": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "novelty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "crossDomain": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "decomposability", + "sequentiality", + "toolIntensity", + "uncertainty", + "verificationRisk", + "novelty", + "crossDomain" + ], + "additionalProperties": false + }, + "maxWorkers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "maxRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "roles": { + "minItems": 1, + "maxItems": 10, + "type": "array", + "items": { + "type": "string", + "enum": [ + "generation", + "proximity", + "reflection", + "ranking", + "evolution", + "revision", + "verification", + "investigation", + "simulation", + "synthesis" + ] + } + }, + "minIndependentVerifiers": { + "type": "integer", + "minimum": 1, + "maximum": 2 + }, + "adaptive": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "marginal-utility-v1" + }, + "minRounds": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "patience": { + "type": "integer", + "minimum": 1, + "maximum": 7 + }, + "minUtilityGain": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetUtility": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "minRounds", + "patience", + "minUtilityGain", + "maxUncertainty" + ], + "additionalProperties": false + }, + "repair": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "verifier-routed-v1" + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "minConfidence" + ], + "additionalProperties": false + } + }, + "required": [ + "topology", + "maxWorkers", + "maxRounds", + "minIndependentVerifiers" + ], + "additionalProperties": false + }, + "search": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "adaptive-search-v1" + }, + "signal": { + "type": "object", + "properties": { + "source": { + "type": "string", + "const": "verified-final-evaluations" + }, + "decay": { + "type": "number", + "const": 0.9 + }, + "epsilon": { + "type": "number", + "const": 1e-8 + } + }, + "required": [ + "source", + "decay", + "epsilon" + ], + "additionalProperties": false + }, + "local": { + "type": "object", + "properties": { + "minIntensity": { + "type": "number", + "const": 0.15 + }, + "maxIntensity": { + "type": "number", + "const": 0.5 + } + }, + "required": [ + "minIntensity", + "maxIntensity" + ], + "additionalProperties": false + }, + "global": { + "type": "object", + "properties": { + "exploration": { + "type": "number", + "const": 1.4142135623730951 + }, + "minVisits": { + "type": "number", + "const": 2 + } + }, + "required": [ + "exploration", + "minVisits" + ], + "additionalProperties": false + }, + "stagnation": { + "type": "object", + "properties": { + "patience": { + "type": "number", + "const": 5 + }, + "maxSignal": { + "type": "number", + "const": 0.02 + } + }, + "required": [ + "patience", + "maxSignal" + ], + "additionalProperties": false + } + }, + "required": [ + "protocolVersion", + "signal", + "local", + "global", + "stagnation" + ], + "additionalProperties": false + }, + "audit": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "performance", + "failure", + "hybrid" + ] + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minSamples": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "noiseVariance": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 2 + }, + "lengthscale": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 100 + }, + "beta": { + "default": 1.96, + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "failureThreshold": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "tolerance": { + "default": 0.02, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "maxUncertainty": { + "default": 0.05, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + }, + "estimationWeight": { + "default": 0.5, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "diversityWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "coverageWeight": { + "default": 0.2, + "type": "number", + "minimum": 0, + "maximum": 0.5 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + }, + "transfer": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "score-history-prior-v1" + }, + "poolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sourceManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "selectionMethod": { + "type": "string", + "enum": [ + "pca-gmm-profile-v1", + "holdout-embedding-gmm-v1" + ] + }, + "sourceModels": { + "minItems": 3, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "calibrationSamples": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "maxCalibrationMAE": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "poolSHA256", + "sourceManifestSHA256", + "selectionSHA256", + "selectionMethod", + "sourceModels", + "calibrationSamples", + "maxCalibrationMAE" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean" + } + }, + "required": [ + "mode", + "budget", + "minSamples" + ], + "additionalProperties": false + }, + "failureDiscovery": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "topic-aware-failure-v1" + }, + "sourcePoolSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "topicModel": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "predefined", + "bertopic" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + }, + "topics": { + "minItems": 2, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,119}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "generator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "validators": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "correctness", + "topic", + "novelty" + ] + }, + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "identity" + ], + "additionalProperties": false + } + }, + "embedding": { + "type": "object", + "properties": { + "identity": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "dimensions": { + "type": "integer", + "minimum": 2, + "maximum": 64 + }, + "regularization": { + "default": 0.000001, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 0.01 + } + }, + "required": [ + "identity", + "dimensions" + ], + "additionalProperties": false + }, + "budget": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "anchorsPerAttempt": { + "type": "integer", + "minimum": 1, + "maximum": 8 + }, + "exploration": { + "default": 1.4142135623730951, + "type": "number", + "exclusiveMinimum": 0, + "maximum": 4 + }, + "failureThreshold": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "targetFailures": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 512 + } + }, + "required": [ + "protocolVersion", + "sourcePoolSHA256", + "topicModel", + "topics", + "generator", + "validators", + "embedding", + "budget", + "anchorsPerAttempt", + "failureThreshold" + ], + "additionalProperties": false + }, + "integrity": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "benchmark-integrity-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "minCoverage": { + "type": "number", + "minimum": 0.9, + "maximum": 1 + }, + "assignedModel": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "baseArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "baseArtifactSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "forbiddenModelArtifacts": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "policy": { + "type": "object", + "properties": { + "testItemDerivation": { + "type": "string", + "const": "forbidden" + }, + "unapprovedExternalModels": { + "type": "string", + "const": "forbidden" + }, + "benchmarkLookup": { + "type": "string", + "const": "forbidden" + } + }, + "required": [ + "testItemDerivation", + "unapprovedExternalModels", + "benchmarkLookup" + ], + "additionalProperties": false + }, + "auditors": { + "minItems": 3, + "maxItems": 3, + "type": "array", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "test_item_contamination", + "external_model_use", + "benchmark_lookup" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "name", + "version", + "promptSHA256" + ], + "additionalProperties": false + } + }, + "hiddenCanaryManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "minHiddenCanaries": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "traceSchemaSHA256", + "minEvents", + "minCoverage", + "assignedModel", + "policy", + "auditors", + "hiddenCanaryManifestSHA256", + "minHiddenCanaries" + ], + "additionalProperties": false + }, + "evolution": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evolution-trace-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lineAlgorithm": { + "type": "string", + "const": "sha256-exact-line-v1" + }, + "roots": { + "minItems": 1, + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "extensions": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "pattern": "^\\.[a-zA-Z0-9][a-zA-Z0-9._+-]{0,31}$" + } + }, + "exclude": { + "default": [], + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 1, + "maximum": 100000 + }, + "maxFileBytes": { + "type": "integer", + "minimum": 1, + "maximum": 1000000000 + }, + "maxTotalBytes": { + "type": "integer", + "minimum": 1, + "maximum": 10000000000 + }, + "maxSourceLines": { + "type": "integer", + "minimum": 1, + "maximum": 10000000 + }, + "maxChangedLines": { + "type": "integer", + "minimum": 1, + "maximum": 2000000 + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "manifestSchemaSHA256", + "lineAlgorithm", + "roots", + "extensions", + "maxFiles", + "maxFileBytes", + "maxTotalBytes", + "maxSourceLines", + "maxChangedLines" + ], + "additionalProperties": false + }, + "metaHarness": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "meta-harness-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "archiveSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "baseline": { + "type": "object", + "properties": { + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "artifactSHA256", + "manifestSHA256" + ], + "additionalProperties": false + }, + "mutable": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "root": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + }, + "component": { + "type": "string", + "enum": [ + "prompt", + "memory", + "skill", + "tool", + "middleware", + "subagent", + "scaffold" + ] + } + }, + "required": [ + "root", + "component" + ], + "additionalProperties": false + } + }, + "protected": { + "type": "object", + "properties": { + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "roots": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + } + }, + "required": [ + "manifestSHA256", + "roots" + ], + "additionalProperties": false + }, + "archive": { + "type": "object", + "properties": { + "contents": { + "type": "string", + "const": "full-source-scores-traces" + }, + "query": { + "type": "string", + "const": "filesystem" + }, + "summariesOnly": { + "type": "boolean", + "const": false + }, + "hiddenContent": { + "type": "string", + "const": "excluded" + }, + "evaluatorContent": { + "type": "string", + "const": "excluded" + } + }, + "required": [ + "contents", + "query", + "summariesOnly", + "hiddenContent", + "evaluatorContent" + ], + "additionalProperties": false + }, + "updater": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "judge": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "search": { + "type": "object", + "properties": { + "models": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "tasks": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "activationRequired": { + "type": "boolean" + } + }, + "required": [ + "id", + "commitment", + "activationRequired" + ], + "additionalProperties": false + } + } + }, + "required": [ + "models", + "tasks" + ], + "additionalProperties": false + }, + "heldout": { + "type": "object", + "properties": { + "models": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitment" + ], + "additionalProperties": false + } + }, + "tasks": { + "minItems": 1, + "maxItems": 256, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,199}$" + }, + "commitment": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "activationRequired": { + "type": "boolean" + } + }, + "required": [ + "id", + "commitment", + "activationRequired" + ], + "additionalProperties": false + } + } + }, + "required": [ + "models", + "tasks" + ], + "additionalProperties": false + }, + "thresholds": { + "type": "object", + "properties": { + "minSearchGain": { + "type": "number", + "minimum": 0 + }, + "minHeldoutGain": { + "type": "number", + "minimum": 0 + }, + "maxModelRegression": { + "type": "number", + "minimum": 0 + }, + "minActivationRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minRequiredAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minFinalAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxPhaseDrift": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minPredictionPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxRiskRegressions": { + "type": "integer", + "minimum": 0, + "maximum": 10000 + }, + "maxContextTokens": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "maxMeanContextIncrease": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "minSearchGain", + "minHeldoutGain", + "maxModelRegression", + "minActivationRate", + "minRequiredAdherence", + "minFinalAdherence", + "maxPhaseDrift", + "minPredictionPrecision", + "maxRiskRegressions", + "maxContextTokens", + "maxMeanContextIncrease" + ], + "additionalProperties": false + }, + "promotionRequired": { + "type": "boolean", + "const": true + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "archiveSchemaSHA256", + "traceSchemaSHA256", + "baseline", + "mutable", + "protected", + "archive", + "updater", + "judge", + "search", + "heldout", + "thresholds", + "promotionRequired" + ], + "additionalProperties": false + }, + "interventions": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "intervention-study-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "requiredForPromotion": { + "type": "boolean" + }, + "minPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxPairs": { + "type": "integer", + "minimum": 3, + "maximum": 32 + }, + "maxTotalPairs": { + "type": "integer", + "minimum": 3, + "maximum": 256 + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "required": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "replay", + "retune", + "ablation", + "repair", + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + } + }, + "rules": { + "minItems": 1, + "maxItems": 8, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "family": { + "type": "string", + "const": "replay" + }, + "mode": { + "type": "string", + "const": "max_absolute_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "retune", + "ablation", + "repair" + ] + }, + "mode": { + "type": "string", + "const": "min_effect" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "family": { + "type": "string", + "enum": [ + "model_transfer", + "context_transfer", + "evaluator_transfer", + "split_transfer" + ] + }, + "mode": { + "type": "string", + "const": "max_regression" + }, + "threshold": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "family", + "mode", + "threshold" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "requiredForPromotion", + "minPairs", + "maxPairs", + "maxTotalPairs", + "confidence", + "required", + "rules" + ], + "additionalProperties": false + }, + "simulation": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "ode", + "pde", + "cfd", + "materials", + "molecular", + "agentic" + ] + }, + "engine": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commandSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commandSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "problemSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reference": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "analytic", + "manufactured", + "benchmark", + "independent_solver", + "limiting_case" + ] + }, + "identity": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "kind", + "identity", + "sha256" + ], + "additionalProperties": false + }, + "validation": { + "type": "object", + "properties": { + "errorNorm": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "minLevels": { + "type": "integer", + "minimum": 3, + "maximum": 12 + }, + "maxLevels": { + "default": 12, + "type": "integer", + "minimum": 3, + "maximum": 24 + }, + "expectedOrder": { + "type": "number", + "exclusiveMinimum": 0, + "maximum": 20 + }, + "orderTolerance": { + "type": "number", + "minimum": 0, + "maximum": 10 + }, + "maxResidual": { + "type": "number", + "minimum": 0 + }, + "invariantTolerances": { + "type": "object", + "propertyNames": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "additionalProperties": { + "type": "number", + "minimum": 0 + } + }, + "requiredStressTests": { + "minItems": 1, + "maxItems": 7, + "type": "array", + "items": { + "type": "string", + "enum": [ + "timestep_sensitivity", + "solver_tolerance_sensitivity", + "reference_replay", + "independent_implementation", + "unit_convention", + "boundary_sensitivity", + "perturbation_stability" + ] + } + } + }, + "required": [ + "errorNorm", + "minLevels", + "expectedOrder", + "orderTolerance", + "maxResidual", + "invariantTolerances", + "requiredStressTests" + ], + "additionalProperties": false + } + }, + "required": [ + "kind", + "engine", + "problemSHA256", + "reference", + "validation" + ], + "additionalProperties": false + }, + "evaluatorAudit": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "evaluator-audit-v1" + }, + "auditor": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "suite": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "commitmentSHA256" + ], + "additionalProperties": false + }, + "minCleanCases": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "minCasesPerFault": { + "type": "integer", + "minimum": 1, + "maximum": 128 + }, + "requiredFaults": { + "minItems": 1, + "maxItems": 10, + "type": "array", + "items": { + "type": "string", + "enum": [ + "wrong_answer", + "unsupported_claim", + "missing_evidence", + "data_leakage", + "non_reproducible", + "reward_hacking", + "invalid_statistics", + "invalid_simulation", + "distribution_shift", + "evaluation_awareness" + ] + } + }, + "minSensitivity": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minSpecificity": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minBalancedAccuracy": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "minFaultRecall": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + }, + "maxBrierScore": { + "type": "number", + "minimum": 0, + "maximum": 0.5 + } + }, + "required": [ + "protocolVersion", + "auditor", + "suite", + "minCleanCases", + "minCasesPerFault", + "requiredFaults", + "minSensitivity", + "minSpecificity", + "minBalancedAccuracy", + "minFaultRecall", + "maxBrierScore" + ], + "additionalProperties": false + }, + "semanticAudit": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "semantic-audit-v1" + }, + "reviewer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "scope": { + "type": "object", + "properties": { + "objectiveSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "criteria": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "requirement": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "requirement" + ], + "additionalProperties": false + } + }, + "forbiddenShortcuts": { + "minItems": 1, + "maxItems": 24, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + }, + "required": [ + "id", + "description" + ], + "additionalProperties": false + } + }, + "literature": { + "type": "object", + "properties": { + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "corpusSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "cutoff", + "corpusSHA256" + ], + "additionalProperties": false + }, + "noveltyFloor": { + "type": "string", + "enum": [ + "not_required", + "known", + "rediscovery", + "minor", + "publication", + "major" + ] + } + }, + "required": [ + "objectiveSHA256", + "criteria", + "forbiddenShortcuts", + "literature", + "noveltyFloor" + ], + "additionalProperties": false + }, + "minReviewers": { + "type": "integer", + "minimum": 2, + "maximum": 5 + }, + "minConfidence": { + "type": "number", + "minimum": 0.5, + "maximum": 1 + } + }, + "required": [ + "protocolVersion", + "reviewer", + "scope", + "minReviewers", + "minConfidence" + ], + "additionalProperties": false + }, + "synthesis": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "scientific-synthesis-v1" + }, + "querySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceFactsSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "referenceFactCount": { + "type": "integer", + "minimum": 1, + "maximum": 2048 + }, + "cutoff": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "tools": { + "minItems": 1, + "maxItems": 3, + "type": "array", + "items": { + "type": "string", + "enum": [ + "google_search", + "paper_search", + "web_browse" + ] + } + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "filterPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "maxToolEvents": { + "type": "integer", + "minimum": 1, + "maximum": 10000 + }, + "decomposer": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "judges": { + "type": "object", + "properties": { + "precision": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + }, + "recall": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "promptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "configSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name", + "version", + "promptSHA256", + "configSHA256" + ], + "additionalProperties": false + } + }, + "required": [ + "precision", + "recall" + ], + "additionalProperties": false + }, + "minGeneratedFacts": { + "type": "integer", + "minimum": 1, + "maximum": 512 + }, + "minPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minRecall": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "minF1": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "cleanRoomRequired": { + "type": "boolean", + "const": true + }, + "judgeFailurePolicy": { + "type": "string", + "const": "inconclusive" + } + }, + "required": [ + "protocolVersion", + "querySHA256", + "referenceSHA256", + "referenceFactsSHA256", + "referenceFactCount", + "cutoff", + "tools", + "traceSchemaSHA256", + "filterPolicySHA256", + "maxToolEvents", + "decomposer", + "judges", + "minGeneratedFacts", + "minPrecision", + "minRecall", + "minF1", + "cleanRoomRequired", + "judgeFailurePolicy" + ], + "additionalProperties": false + }, + "autonomy": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "human-ai-autonomy-v1" + }, + "claimedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "recorder": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "source": { + "type": "string", + "const": "evaluator_runtime" + } + }, + "required": [ + "name", + "version", + "artifactSHA256", + "source" + ], + "additionalProperties": false + }, + "traceSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "classificationPolicySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "maxEvents": { + "type": "integer", + "minimum": 2, + "maximum": 10000 + }, + "rawRetention": { + "type": "string", + "const": "required" + }, + "disclosure": { + "type": "string", + "enum": [ + "evaluator_retained", + "public_essential_after_release" + ] + }, + "completeTraceRequired": { + "type": "boolean", + "const": true + }, + "uncertaintyPolicy": { + "type": "string", + "const": "inconclusive" + } + }, + "required": [ + "protocolVersion", + "claimedLevel", + "recorder", + "traceSchemaSHA256", + "classificationPolicySHA256", + "maxEvents", + "rawRetention", + "disclosure", + "completeTraceRequired", + "uncertaintyPolicy" + ], + "additionalProperties": false + }, + "formalProof": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "formal-proof-v1" + }, + "language": { + "type": "string", + "const": "lean4" + }, + "tier": { + "type": "string", + "enum": [ + "kernel", + "fresh_recheck", + "external_crosscheck" + ] + }, + "relation": { + "type": "string", + "enum": [ + "exact_proof", + "exact_refutation", + "repaired_proof" + ] + }, + "challengeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "statementSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "declaration": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "module": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "leanVersion": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "leanToolchainSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "lakeManifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "dependencyTreeSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "verifiers": { + "minItems": 2, + "maxItems": 6, + "type": "array", + "items": { + "type": "object", + "properties": { + "role": { + "type": "string", + "enum": [ + "lean_kernel", + "source_auditor", + "axiom_auditor", + "fresh_rechecker", + "sandbox_comparator", + "external_checker" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "artifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "role", + "name", + "version", + "artifactSHA256" + ], + "additionalProperties": false + } + }, + "sandboxImageSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "forbiddenConstructs": { + "minItems": 4, + "maxItems": 4, + "type": "array", + "items": { + "type": "string", + "enum": [ + "sorry", + "admit", + "debug.skipKernelTC", + "native_decide" + ] + } + }, + "allowedAxioms": { + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "maxFiles": { + "type": "integer", + "minimum": 6, + "maximum": 10000 + }, + "completeManifestRequired": { + "type": "boolean", + "const": true + }, + "warningPolicy": { + "type": "string", + "const": "fail" + }, + "semanticPolicy": { + "type": "string", + "const": "formal_statement_only" + }, + "blueprint": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "proof-blueprint-v1" + }, + "graphSchemaSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "compilerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sketchValidatorArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerArtifactSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "reviewerPromptSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "nodePolicy": { + "type": "string", + "const": "and-or-monotone-v1" + }, + "failurePolicy": { + "type": "string", + "const": "preserve-and-refine" + }, + "memoization": { + "type": "string", + "const": "goal-sha256" + }, + "finalAuthority": { + "type": "string", + "const": "formal-proof-v1" + }, + "directAttemptFirst": { + "type": "boolean", + "const": true + }, + "verifiedSketchRequired": { + "type": "boolean", + "const": true + }, + "completeFailureHistoryRequired": { + "type": "boolean", + "const": true + }, + "maxNodes": { + "type": "integer", + "minimum": 2, + "maximum": 512 + }, + "maxDepth": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "maxParallel": { + "type": "integer", + "minimum": 1, + "maximum": 32 + }, + "maxAttemptsPerGoal": { + "type": "integer", + "minimum": 1, + "maximum": 16 + }, + "maxRefinementsPerGoal": { + "type": "integer", + "minimum": 0, + "maximum": 16 + }, + "leaseDurationMs": { + "type": "integer", + "minimum": 1000, + "maximum": 3600000 + } + }, + "required": [ + "protocolVersion", + "graphSchemaSHA256", + "compilerArtifactSHA256", + "sketchValidatorArtifactSHA256", + "reviewerArtifactSHA256", + "reviewerPromptSHA256", + "nodePolicy", + "failurePolicy", + "memoization", + "finalAuthority", + "directAttemptFirst", + "verifiedSketchRequired", + "completeFailureHistoryRequired", + "maxNodes", + "maxDepth", + "maxParallel", + "maxAttemptsPerGoal", + "maxRefinementsPerGoal", + "leaseDurationMs" + ], + "additionalProperties": false + } + }, + "required": [ + "protocolVersion", + "language", + "tier", + "relation", + "challengeSHA256", + "statementSHA256", + "declaration", + "module", + "leanVersion", + "leanToolchainSHA256", + "lakeManifestSHA256", + "dependencyTreeSHA256", + "verifiers", + "forbiddenConstructs", + "allowedAxioms", + "maxFiles", + "completeManifestRequired", + "warningPolicy", + "semanticPolicy" + ], + "additionalProperties": false + }, + "replication": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "replicated-evaluation-v1" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "sampling": { + "type": "object", + "properties": { + "design": { + "type": "string", + "const": "crossed-stratified-cluster-v1" + }, + "stratumKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "clusterKind": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "strata": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + }, + "clusters": { + "minItems": 3, + "maxItems": 32, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 120 + }, + "commitmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "id", + "commitmentSHA256" + ], + "additionalProperties": false + } + } + }, + "required": [ + "design", + "stratumKind", + "clusterKind", + "strata", + "clusters" + ], + "additionalProperties": false + }, + "estimator": { + "type": "string", + "enum": [ + "mean", + "median", + "iqm", + "pass_rate" + ] + }, + "interval": { + "anyOf": [ + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "stratified-bootstrap-percentile-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + }, + "resamples": { + "type": "integer", + "minimum": 1000, + "maximum": 50000 + }, + "seed": { + "type": "integer", + "minimum": 0, + "maximum": 4294967295 + } + }, + "required": [ + "method", + "confidence", + "resamples", + "seed" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "method": { + "type": "string", + "const": "wilson-score-v1" + }, + "confidence": { + "type": "number", + "const": 0.95 + } + }, + "required": [ + "method", + "confidence" + ], + "additionalProperties": false + } + ] + }, + "decision": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "conservative-bound-v1" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "target": { + "type": "number" + }, + "maxIntervalWidth": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "rule", + "direction", + "target" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "validatorSHA256", + "environmentSHA256", + "sampling", + "estimator", + "interval", + "decision", + "failurePolicy" + ], + "additionalProperties": false + }, + "confirmation": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "sealed-confirmation-v1" + }, + "optimization": { + "type": "object", + "properties": { + "split": { + "type": "string", + "enum": [ + "development", + "validation" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "split", + "manifestSHA256" + ], + "additionalProperties": false + }, + "claim": { + "type": "object", + "properties": { + "taskID": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "split": { + "type": "string", + "enum": [ + "held_out", + "release" + ] + }, + "manifestSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "environmentSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "source": { + "type": "object", + "properties": { + "repository": { + "type": "string", + "format": "uri" + }, + "revision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + } + }, + "required": [ + "repository", + "revision" + ], + "additionalProperties": false + }, + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + }, + "target": { + "type": "number" + } + }, + "required": [ + "taskID", + "split", + "manifestSHA256", + "validatorSHA256", + "environmentSHA256", + "evaluator", + "metric", + "direction", + "target" + ], + "additionalProperties": false + }, + "selection": { + "type": "object", + "properties": { + "rule": { + "type": "string", + "const": "terminal-verified-best-v1" + }, + "subjects": { + "type": "number", + "const": 1 + } + }, + "required": [ + "rule", + "subjects" + ], + "additionalProperties": false + }, + "exposure": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "const": "terminal-receipt-only" + }, + "searchFeedback": { + "type": "boolean", + "const": false + }, + "memoryCapture": { + "type": "boolean", + "const": false + } + }, + "required": [ + "policy", + "searchFeedback", + "memoryCapture" + ], + "additionalProperties": false + }, + "failurePolicy": { + "type": "string", + "const": "fail-closed" + } + }, + "required": [ + "protocolVersion", + "optimization", + "claim", + "selection", + "exposure", + "failurePolicy" + ], + "additionalProperties": false + }, + "packs": { + "maxItems": 8, + "type": "array", + "items": { + "type": "string", + "enum": [ + "statistics", + "biology", + "physics", + "pde", + "chemistry", + "ml", + "forecast", + "formal" + ] + } + }, + "model": { + "type": "object", + "properties": { + "provider": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "effort": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "provider", + "name" + ], + "additionalProperties": false + }, + "tools": { + "default": [], + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "skills": { + "default": [], + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + }, + "budget": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "steps": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "candidates": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "tokens": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "costUSD": { + "type": "number", + "minimum": 0 + }, + "cpuHours": { + "type": "number", + "minimum": 0 + }, + "gpuHours": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "intervention": { + "type": "string", + "enum": [ + "autonomous", + "human_reprompted" + ] + }, + "contamination": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "minLength": 1 + }, + "hiddenTestsAccessible": { + "type": "boolean", + "const": false + }, + "publicDataCutoff": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "policy", + "hiddenTestsAccessible" + ], + "additionalProperties": false + }, + "createdAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "objective", + "benchmark", + "profile", + "model", + "budget", + "seed", + "intervention", + "contamination", + "createdAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.contract({\n ...\n})" + } + ] + } + }, + "/harness/runs/{sessionID}/evaluations": { + "get": { + "operationId": "harness.evaluations", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "List immutable harness evaluations", + "responses": { + "200": { + "description": "Evaluation journal", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "subject": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "run", + "candidate" + ] + }, + "id": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "type", + "id" + ], + "additionalProperties": false + }, + "fidelity": { + "type": "object", + "properties": { + "stage": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "final": { + "type": "boolean" + } + }, + "required": [ + "stage", + "final" + ], + "additionalProperties": false + }, + "simulationReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "integrityReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "interventionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatorAuditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "semanticReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "replicationReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "failureDiscoveryReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "synthesisReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "autonomyReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "proofReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluator": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "version": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "source": { + "type": "string", + "enum": [ + "benchmark", + "gate", + "human", + "external" + ] + } + }, + "required": [ + "name", + "version", + "source" + ], + "additionalProperties": false + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "score": { + "type": "number" + }, + "metrics": { + "default": {}, + "type": "object", + "propertyNames": { + "type": "string", + "maxLength": 200 + }, + "additionalProperties": { + "type": "number" + } + }, + "checks": { + "maxItems": 128, + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "blocking": { + "type": "boolean" + }, + "score": { + "type": "number" + }, + "evidence": { + "default": [], + "maxItems": 32, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "note": { + "type": "string", + "maxLength": 4000 + } + }, + "required": [ + "id", + "status", + "blocking" + ], + "additionalProperties": false + } + }, + "evidence": { + "minItems": 1, + "maxItems": 128, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 1000 + } + }, + "usage": { + "type": "object", + "properties": { + "wallTimeMs": { + "type": "number", + "minimum": 0 + }, + "costUSD": { + "type": "number", + "minimum": 0 + } + }, + "additionalProperties": false + }, + "evaluatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "recordedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "notes": { + "type": "string", + "maxLength": 8000 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "evaluator", + "status", + "checks", + "evidence", + "evaluatedAt" + ], + "additionalProperties": false + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.evaluations({\n ...\n})" + } + ] + } + }, + "/harness/runs/{sessionID}/report": { + "get": { + "operationId": "harness.report", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "sessionID", + "schema": { + "type": "string", + "minLength": 1 + }, + "required": true + } + ], + "summary": "Build an evaluation quality-cost report", + "responses": { + "200": { + "description": "Quality-cost report", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "runID": { + "type": "string", + "minLength": 1 + }, + "sessionID": { + "type": "string", + "minLength": 1 + }, + "contractFingerprint": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "comparisonKey": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "benchmark": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string", + "minLength": 1 + }, + "family": { + "type": "string", + "enum": [ + "data", + "biology", + "physics", + "chemistry", + "ml", + "generalist", + "custom" + ] + }, + "version": { + "type": "string", + "minLength": 1 + }, + "taskID": { + "type": "string", + "minLength": 1 + }, + "split": { + "type": "string", + "enum": [ + "development", + "validation", + "held_out", + "release" + ] + } + }, + "required": [ + "id", + "title", + "family", + "version", + "taskID", + "split" + ], + "additionalProperties": false + }, + "execution": { + "type": "object", + "properties": { + "profile": { + "type": "string", + "enum": [ + "react", + "optimize", + "reproduce", + "theory", + "numerical", + "training", + "forecast" + ] + }, + "packs": { + "type": "array", + "items": { + "type": "string" + } + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + }, + "effort": { + "type": "string" + }, + "intervention": { + "type": "string", + "enum": [ + "autonomous", + "human_reprompted" + ] + }, + "autonomy": { + "type": "object", + "properties": { + "claimedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "derivedLevel": { + "type": "string", + "enum": [ + "essentially_autonomous", + "human_ai_collaboration", + "primarily_human" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + } + }, + "required": [ + "claimedLevel" + ], + "additionalProperties": false + }, + "formal": { + "type": "object", + "properties": { + "tier": { + "type": "string", + "enum": [ + "kernel", + "fresh_recheck", + "external_crosscheck" + ] + }, + "relation": { + "type": "string", + "enum": [ + "exact_proof", + "exact_refutation", + "repaired_proof" + ] + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed" + ] + }, + "blueprint": { + "type": "object", + "properties": { + "blueprintID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "status": { + "type": "string", + "enum": [ + "open", + "proved", + "refuted", + "exhausted" + ] + }, + "goals": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "proved": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refuted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "exhausted": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "decompositions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "attempts": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rejected": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "refinements": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "openLeases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "blueprintID", + "status", + "goals", + "proved", + "refuted", + "exhausted", + "decompositions", + "attempts", + "rejected", + "refinements", + "openLeases", + "revision" + ], + "additionalProperties": false + } + }, + "required": [ + "tier", + "relation" + ], + "additionalProperties": false + }, + "seed": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + } + }, + "required": [ + "profile", + "packs", + "provider", + "model", + "intervention", + "seed" + ], + "additionalProperties": false + }, + "quality": { + "type": "object", + "properties": { + "source": { + "type": "string", + "enum": [ + "optimization", + "sealed_confirmation" + ] + }, + "provisional": { + "type": "boolean" + }, + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "metric": { + "type": "string" + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize", + "pass" + ] + }, + "score": { + "type": "number" + }, + "target": { + "type": "number" + }, + "targetReached": { + "type": "boolean" + }, + "evaluator": { + "type": "string", + "minLength": 1 + }, + "evaluatorVersion": { + "type": "string" + }, + "simulationReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "integrityReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evolutionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "interventionReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluatorAuditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "semanticReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "replicationReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "auditReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "failureDiscoveryReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "synthesisReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "autonomyReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "proofReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "metaReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "confirmationReceiptID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "evaluations": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "source", + "provisional", + "direction", + "targetReached", + "evaluator", + "evaluations" + ], + "additionalProperties": false + }, + "efficiency": { + "type": "object", + "properties": { + "costUSD": { + "type": "number", + "minimum": 0 + }, + "evaluatorCostUSD": { + "type": "number", + "minimum": 0 + }, + "tokens": { + "type": "object", + "properties": { + "input": { + "type": "number", + "minimum": 0 + }, + "output": { + "type": "number", + "minimum": 0 + }, + "reasoning": { + "type": "number", + "minimum": 0 + }, + "cacheRead": { + "type": "number", + "minimum": 0 + }, + "cacheWrite": { + "type": "number", + "minimum": 0 + }, + "total": { + "type": "number", + "minimum": 0 + } + }, + "required": [ + "input", + "output", + "reasoning", + "cacheRead", + "cacheWrite", + "total" + ], + "additionalProperties": false + }, + "wallTimeMs": { + "type": "number", + "minimum": 0 + }, + "evaluatorWallTimeMs": { + "type": "number", + "minimum": 0 + }, + "toolCalls": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "searches": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "dedupeHits": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "retries": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "failures": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "candidates": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "additionalProperties": false + }, + "search": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "active", + "completed" + ] + }, + "stopReason": { + "type": "string", + "enum": [ + "budget_exhausted", + "objective_met", + "no_improvement", + "user_cancelled", + "runtime_error" + ] + }, + "bestID": { + "type": "string" + }, + "candidates": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "verified": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "generations": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stalled": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "proposalPolicy": { + "type": "string", + "enum": [ + "advisory-v2", + "leased-v3", + "adaptive-v4" + ] + }, + "controller": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "adaptive-search-v1" + }, + "signal": { + "type": "object", + "properties": { + "source": { + "type": "string", + "const": "verified-final-evaluations" + }, + "decay": { + "type": "number", + "const": 0.9 + }, + "epsilon": { + "type": "number", + "const": 1e-8 + } + }, + "required": [ + "source", + "decay", + "epsilon" + ], + "additionalProperties": false + }, + "local": { + "type": "object", + "properties": { + "minIntensity": { + "type": "number", + "const": 0.15 + }, + "maxIntensity": { + "type": "number", + "const": 0.5 + } + }, + "required": [ + "minIntensity", + "maxIntensity" + ], + "additionalProperties": false + }, + "global": { + "type": "object", + "properties": { + "exploration": { + "type": "number", + "const": 1.4142135623730951 + }, + "minVisits": { + "type": "number", + "const": 2 + } + }, + "required": [ + "exploration", + "minVisits" + ], + "additionalProperties": false + }, + "stagnation": { + "type": "object", + "properties": { + "patience": { + "type": "number", + "const": 5 + }, + "maxSignal": { + "type": "number", + "const": 0.02 + } + }, + "required": [ + "patience", + "maxSignal" + ], + "additionalProperties": false + } + }, + "required": [ + "protocolVersion", + "signal", + "local", + "global", + "stagnation" + ], + "additionalProperties": false + }, + "adaptation": { + "type": "object", + "properties": { + "protocolVersion": { + "type": "string", + "const": "adaptive-search-v1" + }, + "policySHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "events": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "stalled": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "selectedIsland": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "globalStagnation": { + "type": "boolean" + }, + "islands": { + "minItems": 1, + "maxItems": 4, + "type": "array", + "items": { + "type": "object", + "properties": { + "island": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "visits": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "decayedVisits": { + "type": "number", + "minimum": 0 + }, + "improvements": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "accumulatedImprovement": { + "type": "number", + "minimum": 0 + }, + "decayedReward": { + "type": "number", + "minimum": 0 + }, + "rewardMean": { + "type": "number", + "minimum": 0 + }, + "intensity": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "ucb": { + "type": "number", + "minimum": 0 + }, + "bestID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "bestFitness": { + "type": "number" + } + }, + "required": [ + "island", + "visits", + "decayedVisits", + "improvements", + "accumulatedImprovement", + "decayedReward", + "rewardMean", + "intensity", + "ucb" + ], + "additionalProperties": false + } + } + }, + "required": [ + "protocolVersion", + "policySHA256", + "events", + "stalled", + "globalStagnation", + "islands" + ], + "additionalProperties": false + }, + "objectives": { + "maxItems": 8, + "type": "array", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "direction": { + "type": "string", + "enum": [ + "maximize", + "minimize" + ] + } + }, + "required": [ + "metric", + "direction" + ], + "additionalProperties": false + } + }, + "objectiveAudit": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "const": 1 + }, + "planSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "validatorSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "contractSHA256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "guardIDs": { + "minItems": 1, + "maxItems": 64, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 200 + } + } + }, + "required": [ + "schemaVersion", + "planSHA256", + "validatorSHA256", + "contractSHA256", + "guardIDs" + ], + "additionalProperties": false + }, + "archive": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "status", + "candidates", + "verified", + "generations", + "stalled", + "proposalPolicy", + "objectives", + "archive" + ], + "additionalProperties": false + }, + "metaHarness": { + "type": "object", + "properties": { + "status": { + "type": "string", + "enum": [ + "passed", + "failed", + "inconclusive" + ] + }, + "selectionID": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "diagnostics": { + "type": "object", + "properties": { + "updaterGain": { + "type": "number" + }, + "beneficiaryGain": { + "type": "number" + }, + "worstHeldoutModelGain": { + "type": "number" + }, + "activationRate": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "requiredAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "finalAdherence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "maxPhaseDrift": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "predictionPrecision": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "riskRegressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "maxContextTokens": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "meanContextIncrease": { + "type": "number" + }, + "loadedBenefit": { + "type": "number" + }, + "searchPairs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "heldoutPairs": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "activationRate", + "predictionPrecision", + "riskRegressions", + "maxContextTokens", + "meanContextIncrease", + "searchPairs", + "heldoutPairs" + ], + "additionalProperties": false + }, + "failures": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "status", + "selectionID", + "diagnostics", + "failures" + ], + "additionalProperties": false + }, + "generatedAt": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "schemaVersion", + "runID", + "sessionID", + "contractFingerprint", + "comparisonKey", + "benchmark", + "execution", + "quality", + "efficiency", + "generatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.harness.report({\n ...\n})" + } + ] + } + }, + "/search": { + "get": { + "operationId": "search.query", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "q", + "schema": { + "type": "string", + "minLength": 2, + "maxLength": 200 + }, + "required": true + } + ], + "summary": "Search sessions, messages, and artifacts", + "description": "Case-insensitive plain-text search across session titles, recent conversation text, and artifact files in the project.", + "responses": { + "200": { + "description": "Grouped plain-text matches", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "sessions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": "string" + } + }, + "required": [ + "id", + "title" + ] + } + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "sessionID": { + "type": "string" + }, + "messageID": { + "type": "string" + }, + "role": { + "type": "string" + }, + "snippet": { + "type": "string" + } + }, + "required": [ + "sessionID", + "messageID", + "role", + "snippet" + ] + } + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "name": { + "type": "string" + }, + "kind": { + "type": "string" + } + }, + "required": [ + "path", + "name", + "kind" + ] + } + } + }, + "required": [ + "sessions", + "messages", + "artifacts" + ] + } + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.search.query({\n ...\n})" + } + ] + } + }, + "/permission/{requestID}/reply": { + "post": { + "operationId": "permission.reply", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "requestID", + "schema": { + "type": "string" + }, + "required": true + } + ], + "summary": "Respond to permission request", + "description": "Approve or deny a permission request from the AI assistant.", + "responses": { + "200": { + "description": "Permission processed successfully", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + }, + "404": { + "description": "Not found", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NotFoundError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "reply": { + "type": "string", + "enum": [ + "once", + "session", + "project", + "always", + "reject" + ] + }, + "message": { + "type": "string" + } + }, + "required": [ + "reply" + ] + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.permission.reply({\n ...\n})" + } + ] + } + }, + "/permission": { + "get": { + "operationId": "permission.list", + "parameters": [ + { + "in": "query", + "name": "directory", "schema": { "type": "string" } @@ -23695,94 +69263,6 @@ ] } }, - "/file/starters": { - "post": { - "operationId": "file.starter", - "parameters": [ - { - "in": "query", - "name": "directory", - "schema": { - "type": "string" - } - } - ], - "summary": "Create a local scientific starter project", - "description": "Materialize a valid notebook, sample data, and README without external downloads.", - "responses": { - "200": { - "description": "Created starter files", - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "template": { - "type": "string", - "enum": [ - "single-cell", - "dose-response", - "protein-structure" - ] - }, - "directory": { - "type": "string" - }, - "files": { - "type": "array", - "items": { - "type": "string" - } - }, - "notebook": { - "type": "string" - }, - "readme": { - "type": "string" - } - }, - "required": [ - "template", - "directory", - "files", - "notebook", - "readme" - ] - } - } - } - } - }, - "requestBody": { - "content": { - "application/json": { - "schema": { - "type": "object", - "properties": { - "template": { - "type": "string", - "enum": [ - "single-cell", - "dose-response", - "protein-structure" - ] - } - }, - "required": [ - "template" - ] - } - } - } - }, - "x-codeSamples": [ - { - "lang": "js", - "source": "import { createOpenScienceClient } from \"@synsci/sdk\"\n\nconst client = createOpenScienceClient()\nawait client.file.starter({\n ...\n})" - } - ] - } - }, "/file/publication/capabilities": { "get": { "operationId": "file.publicationCapabilities",