diff --git a/.github/scripts/build-marketplace-fixture.test.mjs b/.github/scripts/build-marketplace-fixture.test.mjs index 46a46692c..2b64d48c7 100644 --- a/.github/scripts/build-marketplace-fixture.test.mjs +++ b/.github/scripts/build-marketplace-fixture.test.mjs @@ -57,7 +57,19 @@ test("the fixture catalog carries the fields the resolver requires", () => { const [plugin] = catalog.plugins; assert.deepEqual(Object.keys(plugin).sort(), PLUGIN_KEYS); assert.equal(plugin.name, "codestory"); + assert.deepEqual(plugin.policy, { + installation: "AVAILABLE", + authentication: "ON_INSTALL", + }); + assert.equal(plugin.category, "Developer Tools"); assert.deepEqual(Object.keys(plugin.source).sort(), SOURCE_KEYS); + // Key sets alone do not pin a delivery: a wrong source kind or repository + // URL resolves a different (or no) plugin while passing every key check. + assert.equal(plugin.source.source, "git-subdir"); + assert.equal( + plugin.source.url, + "https://github.com/TheGreenCedar/CodeStory.git", + ); assert.equal(plugin.source.sha, commit); assert.equal(plugin.source.path, "plugins/codestory"); } finally { @@ -96,8 +108,29 @@ test("the fixture identifies itself and the commit it pins", () => { assert.equal(marker.schema_version, 1); assert.equal(marker.purpose, "codestory-candidate-pinned-marketplace-fixture"); assert.equal(marker.pinned_commit, commit); + // The consumer requires the marker's version to be the pinned commit's own + // plugin version; compare it, do not only require the key to exist. + const pinnedManifest = JSON.parse( + execFileSync( + "git", + [ + "-C", + repositoryRoot, + "show", + `${commit}:plugins/codestory/.codex-plugin/plugin.json`, + ], + { encoding: "utf8" }, + ), + ); + assert.equal(marker.plugin_version, pinnedManifest.version); // The marker must be committed, or a clean-tree check would pass over a fixture that had - // been re-marked after the fact. + // been re-marked after the fact. ls-files --error-unmatch proves the marker is in the + // committed tree -- a gitignored marker leaves status clean while never being delivered. + execFileSync( + "git", + ["-C", out, "ls-files", "--error-unmatch", ".codestory-marketplace-fixture.json"], + { encoding: "utf8" }, + ); assert.equal( execFileSync("git", ["-C", out, "status", "--porcelain"], { encoding: "utf8" }).trim(), "", diff --git a/.github/scripts/cargo-build-artifacts.test.mjs b/.github/scripts/cargo-build-artifacts.test.mjs index b860fedf1..538b5db4e 100644 --- a/.github/scripts/cargo-build-artifacts.test.mjs +++ b/.github/scripts/cargo-build-artifacts.test.mjs @@ -684,6 +684,22 @@ test("rejects debug-profile output even when the target name matches", () => { assert.throws(() => build(input), /not built with the release profile/u); }); +// Each release-profile predicate is an independent admission check: a row that +// trips several at once cannot tell whether a dropped check would still fail. +for (const [field, value] of [ + ["opt_level", "0"], + ["debug_assertions", true], + ["overflow_checks", true], +]) { + test(`rejects a release-directory executable whose only debug marker is profile.${field}`, () => { + const input = fixture(); + input.messages[1].profile[field] = value; + input.jsonLines = input.messages.map((message) => JSON.stringify(message)).join("\n"); + + assert.throws(() => build(input), /not built with the release profile/u); + }); +} + test("rejects a production binary actually built with the test profile", () => { const input = fixture(); input.messages[1].profile.test = true; @@ -824,6 +840,29 @@ test("rejects bytes changed after Cargo emitted the authenticated executable", ( ); }); +test("rejects a same-length rewrite of the authenticated executable", () => { + const { input, manifest } = build(); + // Appending drifts size and digest together, so a size check alone would + // still catch it. Flip bytes in place: size, inode and link count are + // unchanged and only the digest can expose the rewrite. + const executable = input.artifacts[0].executable; + const bytes = fs.readFileSync(executable); + bytes[0] = bytes[0] === 0 ? 1 : 0; + fs.writeFileSync(executable, bytes); + + assert.throws( + () => + verifyCargoArtifactManifest({ + exactSha: SOURCE_SHA, + exactTree: SOURCE_TREE, + manifest, + rustTarget: RUST_TARGET, + workspaceRoot: input.root, + }), + /no longer matches its authenticated build output/u, + ); +}); + test("rejects a hardlink added after Cargo artifact selection", () => { const { input, manifest } = build(); const alias = path.join(input.root, "cross-graph-cli.exe"); diff --git a/.github/scripts/check-workflow-policy.mjs b/.github/scripts/check-workflow-policy.mjs index 015dcf1e6..8690997a3 100644 --- a/.github/scripts/check-workflow-policy.mjs +++ b/.github/scripts/check-workflow-policy.mjs @@ -498,14 +498,16 @@ export function retrievalGeneralizationSuitePolicyViolations( const expectedFilesystemMemberCounts = { existsSync: 1, mkdirSync: 7, - mkdtempSync: 1, + // The hostile matrix owns one tree; the ranker counterexample owns a + // second. Both are mkdtempSync under os.tmpdir(), removed on exit. + mkdtempSync: 2, // Two of these read the shipped pending inventory and the registry document it points at, // so the claim-profile ratchet is checked against the tree that ships, not a synthetic one. - readFileSync: 6, + readFileSync: 5, readdirSync: 4, readlinkSync: 1, - rmSync: 1, - writeFileSync: 1, + rmSync: 2, + writeFileSync: 2, }; const filesystemMemberCounts = new Map(); for (const name of filesystemMemberReferences) { @@ -515,8 +517,8 @@ export function retrievalGeneralizationSuitePolicyViolations( ); } const fixtureFilesystemShapeIsExact = - fsReferenceCount === 24 - && filesystemMemberReferences.length === 22 + fsReferenceCount === 26 + && filesystemMemberReferences.length === 24 && Object.entries(expectedFilesystemMemberCounts).every( ([name, count]) => (filesystemMemberCounts.get(name) ?? 0) === count, ) @@ -550,7 +552,7 @@ export function retrievalGeneralizationSuitePolicyViolations( && writeFirstArguments.length === writeReferenceCount && writeFirstArguments.every((root) => registeredWriteRoots.has(root)); const fixturePathReferenceShapeIsExact = - repositoryRootReferenceCount === 14 + repositoryRootReferenceCount === 15 && fixtureRootReferenceCount === 10 && productionRepositoryRootReferenceCount === 5; const protectedRetrievalWorkflow = `.github/workflows/${retrievalFile}`; @@ -605,8 +607,8 @@ export function retrievalGeneralizationSuitePolicyViolations( ); add( violations, - invocationCount === 1 && lintReferenceCount === 2, - `${retrievalGeneralizationSuiteFile} must execute the hostile fixture matrix through one in-process lint invocation`, + invocationCount === 2 && lintReferenceCount === 3, + `${retrievalGeneralizationSuiteFile} must execute the hostile matrix and the ranker probes through in-process lint invocations only`, ); add( violations, @@ -635,8 +637,11 @@ export function retrievalGeneralizationSuitePolicyViolations( source.includes( 'fs.mkdtempSync(path.join(os.tmpdir(), "codestory-generalization-"))', ) - && temporaryRootCount === 1 - && temporaryTreeCount === 1 + && source.includes( + 'fs.mkdtempSync(path.join(os.tmpdir(), "ranker-literal-"))', + ) + && temporaryRootCount === 2 + && temporaryTreeCount === 2 && source.includes( 'assert.ok(\n path.relative(repositoryRoot, fixtureRoot).startsWith(".."),', ) @@ -644,7 +649,7 @@ export function retrievalGeneralizationSuitePolicyViolations( && source.includes("const extraRustRoot = path.join(fixtureRoot,") && source.includes("const nonRustRoot = path.join(fixtureRoot,") && source.includes("const taskRoot = path.join(fixtureRoot,"), - `${retrievalGeneralizationSuiteFile} must keep every mutable hostile fixture under one temporary tree outside the checkout`, + `${retrievalGeneralizationSuiteFile} must keep every mutable hostile fixture under its declared temporary trees outside the checkout`, ); add( violations, @@ -667,8 +672,9 @@ export function retrievalGeneralizationSuitePolicyViolations( ); add( violations, - source.includes("fs.rmSync(fixtureRoot, { recursive: true, force: true });"), - `${retrievalGeneralizationSuiteFile} must remove its isolated fixture tree after the matrix`, + source.includes("fs.rmSync(fixtureRoot, { recursive: true, force: true });") + && source.includes("fs.rmSync(rankerRoot, { recursive: true, force: true });"), + `${retrievalGeneralizationSuiteFile} must remove its isolated fixture trees after the matrix`, ); return violations; } diff --git a/.github/scripts/check-workflow-policy.test.mjs b/.github/scripts/check-workflow-policy.test.mjs index b8f051d29..526a8d906 100644 --- a/.github/scripts/check-workflow-policy.test.mjs +++ b/.github/scripts/check-workflow-policy.test.mjs @@ -6045,7 +6045,7 @@ await import("node:worker_threads"); `, {}, /must not create subprocesses, workers, or clusters/u], ["matrix calls the lint twice", `${source} runRetrievalGeneralizationLint({}); -`, {}, /through one in-process lint invocation/u], +`, {}, /through in-process lint invocations/u], ["matrix aliases the lint for a second invocation", source.replace( " const result = runRetrievalGeneralizationLint({", [ @@ -6053,7 +6053,7 @@ runRetrievalGeneralizationLint({}); " invokeAgain({});", " const result = runRetrievalGeneralizationLint({", ].join("\n"), - ), {}, /through one in-process lint invocation/u], + ), {}, /through in-process lint invocations/u], ["global file lock returns", `${source} fs.openSync(path.join(os.tmpdir(), "retrieval-generalization.lock"), "wx"); `, {}, /must not restore a global or cross-process fixture lock/u], @@ -6069,15 +6069,15 @@ fs.openSync(path.join(os.tmpdir(), "retrieval-generalization.lock"), "wx"); ["second global temporary root returns", `${source} const sharedRoot = fs.mkdtempSync(path.join(os.tmpdir(), "shared-suite-")); fs.mkdirSync(path.join(sharedRoot, "sentinel")); -`, {}, /under one temporary tree outside the checkout/u], +`, {}, /declared temporary trees outside the checkout/u], ["second sibling temporary root returns", `${source} const sharedRoot = fs.mkdtempSync(path.join(path.dirname(fixtureRoot), "shared-suite-")); fs.mkdirSync(path.join(sharedRoot, "sentinel")); -`, {}, /under one temporary tree outside the checkout/u], +`, {}, /declared temporary trees outside the checkout/u], ["fixtures move into the checkout", source.replace( 'fs.mkdtempSync(path.join(os.tmpdir(), "codestory-generalization-"))', 'fs.mkdtempSync(path.join(repositoryRoot, "codestory-generalization-"))', - ), {}, /under one temporary tree outside the checkout/u], + ), {}, /declared temporary trees outside the checkout/u], ["checkout read-only comparison is removed", source.replace( "const checkoutBefore = treeDigest(repositoryRoot);", "const checkoutBefore = null;", @@ -6110,7 +6110,7 @@ fs.mkdirSync(path.join(sharedRoot, "sentinel")); ["fixture cleanup is removed", source.replace( "fs.rmSync(fixtureRoot, { recursive: true, force: true });", "", - ), {}, /remove its isolated fixture tree/u], + ), {}, /remove its isolated fixture trees/u], ]; for (const [name, candidate, options, expectedReason] of mutations) { diff --git a/.github/scripts/install-codestory-marketplace-proof.test.mjs b/.github/scripts/install-codestory-marketplace-proof.test.mjs index 6a7917255..3f9252caa 100644 --- a/.github/scripts/install-codestory-marketplace-proof.test.mjs +++ b/.github/scripts/install-codestory-marketplace-proof.test.mjs @@ -5,6 +5,7 @@ import { mkdtempSync, mkdirSync, readFileSync, + readdirSync, realpathSync, rmSync, writeFileSync, @@ -89,12 +90,21 @@ test("Windows executes a command shim whose path and arguments contain spaces", } }); -function run(executable, args, options = {}) { - const result = spawnSync(executable, args, { +// Spawning through a shell breaks on spaced executable paths (node.exe under +// "C:\Program Files\nodejs" dies as 'C:\Program' is not recognized). Route +// every child through the production planner: real executables spawn +// directly, only .cmd/.bat shims go through comspec. +function spawnPlan(executable, args, options = {}) { + const { command, commandArgs, spawnOptions = {} } = commandPlan(executable, args); + return spawnSync(command, commandArgs, { ...options, + ...spawnOptions, encoding: "utf8", - shell: process.platform === "win32", }); +} + +function run(executable, args, options = {}) { + const result = spawnPlan(executable, args, options); assert.equal( result.status, 0, @@ -153,11 +163,8 @@ function proofArgs({ ]; } -function assertFailedProof(args, message) { - const result = spawnSync(process.execPath, args, { - encoding: "utf8", - shell: process.platform === "win32", - }); +function assertFailedProof(args, message, options = {}) { + const result = spawnPlan(process.execPath, args, options); assert.notEqual(result.status, 0); assert.match(result.stderr, message); } @@ -165,8 +172,25 @@ function assertFailedProof(args, message) { test("pinned Codex installs a local marketplace fixture into the attested cache", () => { const root = mkdtempSync(path.join(tmpdir(), "codestory-marketplace-proof-")); try { + // Every child of this test runs against owned state: an npm cache and a + // HOME under the fixture root. Inheriting ambient npm config/cache or the + // developer's HOME makes the outcome depend on machine state. const packageRoot = path.join(root, "codex-package"); - const npm = process.platform === "win32" ? "npm.cmd" : "npm"; + const npmCache = path.join(root, "npm-cache"); + const personalHome = path.join(root, "personal-home"); + const childEnv = { + ...process.env, + HOME: personalHome, + USERPROFILE: personalHome, + npm_config_cache: npmCache, + }; + // Resolve npm.cmd to an absolute path: commandPlan quotes the command + // name, and a quoted bare name leaves cmd's %0 unexpanded, so npm.cmd + // would compute %~dp0 (its install prefix) from the cwd. + const npm = process.platform === "win32" + ? spawnSync("where.exe", ["npm.cmd"], { encoding: "utf8" }).stdout.trim() + .split(/\r?\n/u)[0] + : "npm"; run(npm, [ "install", "--prefix", @@ -174,7 +198,11 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" "--no-audit", "--no-fund", `@openai/codex@${codexVersion}`, - ]); + ], { env: childEnv }); + assert.ok( + readdirSync(npmCache).length > 0, + "npm install did not use the fixture-owned cache", + ); const marketplaceRoot = path.join(root, "marketplace"); const pluginSourceRoot = path.join(root, "plugin-source"); @@ -209,7 +237,6 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" const marketplaceRevision = commitFixture(marketplaceRoot, "fixture", { init: true, }); - const personalHome = path.join(root, "personal-home"); mkdirSync(path.join(personalHome, ".agents", "plugins"), { recursive: true }); writeFileSync( path.join(personalHome, ".agents", "plugins", "marketplace.json"), @@ -245,12 +272,7 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" marketplaceRevision, expectedVersion: pluginManifest.version, sourceRepository: pluginSourceRoot, - }), { - env: { - ...process.env, - HOME: personalHome, - }, - }); + }), { env: childEnv }); const attestation = JSON.parse(readFileSync(attestationPath)); const expectedPluginRoot = path.join( @@ -320,9 +342,7 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" "--installation-source", "codex_marketplace_restored_fixture", ); - run(process.execPath, restoredArgs, { - env: { ...process.env, HOME: personalHome }, - }); + run(process.execPath, restoredArgs, { env: childEnv }); const restoredAttestation = JSON.parse( readFileSync(path.join(restoredRoot, "attestation.json")), ); @@ -349,6 +369,7 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" sourceRepository: pluginSourceRoot, }), /installed plugin bytes do not match the checked-out CodeStory package/u, + { env: childEnv }, ); commitFixture(pluginSourceRoot, "change release tree"); assertFailedProof( @@ -361,6 +382,7 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" sourceRepository: pluginSourceRoot, }), /pinned marketplace plugin source does not match the release source tree/u, + { env: childEnv }, ); const catalogPath = path.join( @@ -386,6 +408,7 @@ test("pinned Codex installs a local marketplace fixture into the attested cache" sourceRepository: pluginSourceRoot, }), /marketplace plugin source is not pinned to one immutable commit/u, + { env: childEnv }, ); } finally { rmSync(root, { recursive: true, force: true }); diff --git a/.github/scripts/packaged_agent_proof/self_test_cli.py b/.github/scripts/packaged_agent_proof/self_test_cli.py index b95c13320..af6fda15c 100644 --- a/.github/scripts/packaged_agent_proof/self_test_cli.py +++ b/.github/scripts/packaged_agent_proof/self_test_cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import os import tempfile from pathlib import Path @@ -47,6 +48,26 @@ def run_cli_self_tests() -> None: calibration_producer_run_id=None, calibration_producer_artifact=None, ) + relative_args = argparse.Namespace( + qualification_evidence=None, + qualification_driver=None, + publication_fault_evidence=None, + calibration_bundle=None, + constant_calibration_output_dir=None, + installed_plugin_attestation=Path("candidate-attestation.json"), + installed_plugin_data=None, + ) + previous_cwd = os.getcwd() + os.chdir(root) + try: + _resolve_optional_paths(relative_args) + finally: + os.chdir(previous_cwd) + require( + relative_args.installed_plugin_attestation == attestation.resolve() + and relative_args.installed_plugin_attestation.is_absolute(), + "an optional CLI path kept its ambient working-directory dependence", + ) _resolve_optional_paths(args) require( args.installed_plugin_attestation == attestation.resolve(), @@ -155,3 +176,41 @@ def run_cli_self_tests() -> None: pass else: raise ProofFailure("constant calibration accepted CPU execution") + calibration_args.engine_policy = "accelerated" + calibration_args.expected_backend = "metal" + + for field, hostile_value in ( + ("proof_tier", "installed_runtime"), + ("version_only", True), + ("constant_calibration_output_dir", None), + ("qualification_driver", None), + ("offline", False), + ("project", root / "project"), + ("additional_project", [root / "other"]), + ("additional_query", ["q"]), + ("qualification_evidence", attestation), + ("publication_fault_evidence", attestation), + ("calibration_bundle", attestation), + ): + hostile = argparse.Namespace(**vars(calibration_args)) + setattr(hostile, field, hostile_value) + try: + _validate_calibration_mode(hostile) + except ProofFailure: + pass + else: + raise ProofFailure( + f"constant calibration accepted {field}={hostile_value!r}" + ) + + # The retained runs and the proof output must not nest. + nested = argparse.Namespace(**vars(calibration_args)) + nested.constant_calibration_output_dir = root / "proof" / "constant-runs" + try: + _validate_calibration_mode(nested) + except ProofFailure: + pass + else: + raise ProofFailure( + "constant calibration accepted runs nested inside proof output" + ) diff --git a/.github/scripts/packaged_agent_proof/self_test_contract_scope.py b/.github/scripts/packaged_agent_proof/self_test_contract_scope.py index 3d4662463..1b7202c3e 100644 --- a/.github/scripts/packaged_agent_proof/self_test_contract_scope.py +++ b/.github/scripts/packaged_agent_proof/self_test_contract_scope.py @@ -256,14 +256,58 @@ def _publication_identity_tests() -> None: } publication_identity = publication_identity_from_status(publication_status) require_sha256(publication_identity, "publication identity self-test") - hostile_publication_status = json.loads(json.dumps(publication_status)) - hostile_publication_status["manifest"]["sidecar_generation"] = "stale-generation" - try: - publication_identity_from_status(hostile_publication_status) - except ProofFailure: - pass - else: - raise ProofFailure("manifest report/contract drift was accepted") + + # Every contract->manifest pairing is bound one field at a time: dropping + # any one equality check must turn that field's drift invisible, so each + # divergence is an independent negative. + for contract_field, manifest_field in ( + ("project_id", "project_id"), + ("generation", "sidecar_generation"), + ("input_hash", "sidecar_input_hash"), + ("schema_version", "sidecar_schema_version"), + ("graph_hash", "graph_artifact_hash"), + ): + hostile = json.loads(json.dumps(publication_status)) + original = hostile["manifest"][manifest_field] + hostile["manifest"][manifest_field] = ( + "stale-generation" if isinstance(original, str) + else original + 1 + ) + try: + publication_identity_from_status(hostile) + except ProofFailure: + pass + else: + raise ProofFailure( + f"manifest report/contract drift in {manifest_field} was accepted" + ) + + # Each identity-payload field must change the digest when it changes on + # both sides; a field dropped from the canonical payload stops binding it. + for contract_field, manifest_field, hostile_value in ( + ("project_id", "project_id", "repo-v2-other-project"), + ("generation", "sidecar_generation", "repo-v2-other-generation"), + ("input_hash", "sidecar_input_hash", "9" * 64), + ("schema_version", "sidecar_schema_version", 7), + ("graph_hash", "graph_artifact_hash", "8" * 64), + (None, "lexical_version", "sqlite-fts5-v2"), + (None, "semantic_generation", "semantic-other"), + (None, "scip_revision", "graph-other"), + ): + consistent = json.loads(json.dumps(publication_status)) + consistent["manifest"][manifest_field] = hostile_value + if contract_field is not None: + consistent["manifest_contract"][contract_field] = hostile_value + try: + shifted = publication_identity_from_status(consistent) + except ProofFailure as error: + raise ProofFailure( + f"a consistent {manifest_field} change was rejected: {error}" + ) from error + require( + shifted != publication_identity, + f"the publication identity does not bind {manifest_field}", + ) def run_contract_scope_self_tests() -> None: diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py index a700cd2dc..2e50512c5 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_external.py @@ -161,20 +161,89 @@ def _publication_fault_hostile( external_package, external_contracts = _external_contracts(fixture) publication_path = publication.path publication_payload = publication.payload - hostile_publication = json.loads(json.dumps(publication_payload)) - hostile_publication["assertions"] = {"lost_publication_lease_blocks_commit": True} - write_json(publication_path, hostile_publication) - try: - verify_publication_fault_raw_evidence( - publication_path, - source=manifest["source"], - package=external_package, - contracts=external_contracts, - ) - except ProofFailure: - pass - else: - raise ProofFailure("self-declared publication assertions were accepted") + + def expect_rejected(label: str, mutate) -> None: + hostile = json.loads(json.dumps(publication_payload)) + mutate(hostile) + write_json(publication_path, hostile) + try: + verify_publication_fault_raw_evidence( + publication_path, + source=manifest["source"], + package=external_package, + contracts=external_contracts, + ) + except ProofFailure: + return + raise ProofFailure(f"hostile publication evidence was accepted: {label}") + + expect_rejected( + "self-declared assertions", + lambda payload: payload.__setitem__( + "assertions", {"lost_publication_lease_blocks_commit": True} + ), + ) + # Lease: a revalidation that succeeded must not certify a blocked commit. + expect_rejected( + "lease revalidation reported as held", + lambda payload: payload["publication_hook_events"][2].__setitem__( + "status", "succeeded" + ), + ) + # Fence: a committed manifest must not certify a blocked publication. + expect_rejected( + "manifest commit reported as committed", + lambda payload: payload["publication_hook_events"][3].__setitem__( + "status", "committed" + ), + ) + # Fence: the lease event cannot be silently dropped from the sequence. + expect_rejected( + "missing lease revalidation event", + lambda payload: payload["publication_hook_events"].pop(2), + ) + # Fence: hook clocks cannot move backwards within the commit. + expect_rejected( + "non-monotonic hook clock", + lambda payload: payload["publication_hook_events"][3]["clock"].__setitem__( + "elapsed_ns", 0 + ), + ) + # The candidate must have failed; a successful commit is not a blocked one. + expect_rejected( + "candidate commit succeeded", + lambda payload: payload["candidate_observation"].__setitem__("exit_code", 0), + ) + # Product preservation: post-fault reads must pin the previous publication. + expect_rejected( + "ordinary observation used a different publication", + lambda payload: payload["ordinary_product_observations"][1].__setitem__( + "publication_identity_sha256", + hashlib.sha256(b"new-publication").hexdigest(), + ), + ) + # Product preservation: an ordinary product call failing after the fault + # proves the previous publication did not stay usable. + expect_rejected( + "ordinary search failed after the fault", + lambda payload: payload["ordinary_product_observations"][1].__setitem__( + "exit_code", 1 + ), + ) + # The replacement must be a different server instance, not the crashed one. + expect_rejected( + "server was not actually replaced", + lambda payload: payload["server_observations"][1].update( + { + "server_instance_id": payload["server_observations"][0][ + "server_instance_id" + ], + "process_start_id": payload["server_observations"][0][ + "process_start_id" + ], + } + ), + ) write_json(publication_path, publication_payload) @@ -246,9 +315,15 @@ def _server_crash_scenario_tests() -> None: "scenario assertion self-test did not derive exact raw claims", ) hostile_scenario = json.loads(json.dumps(scenario_observations)) - hostile_scenario["query_replayed"][0]["values"]["wire_attempts"][1]["outcome"] = ( - "server_loss" - ) + hostile_scenario["query_replayed"][0]["values"]["wire_attempts"][1][ + "outcome" + ] = "server_loss" + # Keep the failed replay structurally valid so the rejection proves the + # semantic contract -- the replacement attempt did not complete -- rather + # than failing at loss_code shape validation first. + hostile_scenario["query_replayed"][0]["values"]["wire_attempts"][1][ + "loss_code" + ] = "embedding_server_connection_lost" try: derive_scenario_assertions( "server_crash", @@ -258,8 +333,13 @@ def _server_crash_scenario_tests() -> None: same_account={}, materialization={}, ) - except ProofFailure: - pass + except ProofFailure as error: + require( + "do not bind the old loss and exact replacement completion" + in str(error), + f"a replay that itself lost its server was rejected for its shape" + f" instead of its semantics: {error}", + ) else: raise ProofFailure("named scenario transitions with false values were accepted") misclassified_scenario = json.loads(json.dumps(scenario_observations)) diff --git a/.github/scripts/packaged_agent_proof/self_test_full_stack_manifest.py b/.github/scripts/packaged_agent_proof/self_test_full_stack_manifest.py index 014d7a16a..1352dbeae 100644 --- a/.github/scripts/packaged_agent_proof/self_test_full_stack_manifest.py +++ b/.github/scripts/packaged_agent_proof/self_test_full_stack_manifest.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +from unittest.mock import patch +from . import native_manifest from .contract_primitives import write_json -from .foundation import NATIVE_MANIFEST_FILE, ProofFailure +from .foundation import NATIVE_MANIFEST_FILE, ProofFailure, require from .native_manifest import load_native_manifest from .self_test_full_stack_types import FullStackFixture @@ -29,13 +31,27 @@ def run_hostile_manifest_self_tests(fixture: FullStackFixture) -> None: else: raise ProofFailure("binary/manifest digest mismatch was accepted") + # No supported asset pairs the fixture's Mach-O format with a foreign + # arch, so name one: the wrong target must be rejected by the architecture + # verifier itself, not earlier for being an unknown target. + x64_macos = dict(native_manifest.TARGET_CONTRACTS["macos-arm64"]) + x64_macos["target_arch"] = "x86_64" + x64_macos["target_triple"] = "x86_64-apple-darwin" wrong_target = json.loads(json.dumps(valid_manifest)) wrong_target["asset_target"] = "macos-x64" write_json(hostile_root / NATIVE_MANIFEST_FILE, wrong_target) try: - load_native_manifest(hostile_root, hostile_cli, "0.0.0") - except ProofFailure: - pass + with patch.dict(native_manifest.TARGET_CONTRACTS, {"macos-x64": x64_macos}): + load_native_manifest(hostile_root, hostile_cli, "0.0.0") + except ProofFailure as error: + # Either architecture check counts: the runtime-binary inventory runs + # before the launcher descriptor, and both reject the same mismatch. + require( + "architecture does not match" in str(error) + and "target" in str(error), + "the asset-target negative was rejected before any architecture" + f" check: {error}", + ) else: raise ProofFailure("asset target/binary architecture mismatch was accepted") diff --git a/.github/scripts/packaged_agent_proof/self_test_managed_layout.py b/.github/scripts/packaged_agent_proof/self_test_managed_layout.py index e060a23e7..6452e63d5 100644 --- a/.github/scripts/packaged_agent_proof/self_test_managed_layout.py +++ b/.github/scripts/packaged_agent_proof/self_test_managed_layout.py @@ -87,8 +87,9 @@ def _flat_staging_test(root: Path) -> None: require( ( version_root / "native-generations" / "generation-1" / "runtime-module.dll" - ).is_file(), - "staging did not copy the package root contents into the version root", + ).read_bytes() + == b"runtime-bytes", + "staging did not copy the native module bytes into the version root", ) require( verify_flat_managed_layout(plugin_data, _VERSION, _TARGET) == launcher, diff --git a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py index 6f7f81cee..5b4e5f76b 100644 --- a/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py +++ b/.github/scripts/packaged_agent_proof/self_test_marketplace_delivery.py @@ -18,6 +18,7 @@ import argparse import copy import json +import os import shutil import subprocess import tempfile @@ -60,12 +61,46 @@ def _run_directory_contract_ordering_self_test() -> None: ) +# A self-test git child must not see the developer's git configuration, hooks, or +# credential helpers: ambient config (init.defaultBranch, commit hooks, signing +# rules, GIT_* variables) changes what init/commit/rev-parse do on a machine. +# Isolate the child: every GIT_* variable is scrubbed, global/system config and +# hooks point at the null device, and HOME/XDG resolve under an owned directory. +_GIT_CHILD_HOME: Path | None = None + + def _git(repository: Path, *arguments: str) -> str: + global _GIT_CHILD_HOME + if _GIT_CHILD_HOME is None: + _GIT_CHILD_HOME = Path(tempfile.mkdtemp(prefix="codestory-git-home-")) + null_device = "NUL" if os.name == "nt" else "/dev/null" + environment = { + key: value for key, value in os.environ.items() if not key.startswith("GIT_") + } + environment.update( + { + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": null_device, + "GIT_CONFIG_SYSTEM": null_device, + "GIT_TERMINAL_PROMPT": "0", + "HOME": str(_GIT_CHILD_HOME), + "USERPROFILE": str(_GIT_CHILD_HOME), + "XDG_CONFIG_HOME": str(_GIT_CHILD_HOME / "xdg-config"), + } + ) completed = subprocess.run( - ["git", "-C", str(repository), *arguments], + [ + "git", + "-c", + f"core.hooksPath={null_device}", + "-C", + str(repository), + *arguments, + ], text=True, capture_output=True, timeout=60, + env=environment, ) require( completed.returncode == 0, @@ -605,9 +640,27 @@ def _run_shared_identity_self_tests() -> None: which is precisely the failure this whole path was repaired for, and it would surface only after the tag was already pushed. """ - source = ( - REPOSITORY_ROOT / ".github" / "scripts" / "marketplace-delivery-identity.mjs" - ).read_text(encoding="utf-8") + # Import the module and read its live exports. A source-text match can be + # satisfied by a comment or an unused declaration; only the imported binding + # is what a producer child process actually resolves. + module = REPOSITORY_ROOT / ".github" / "scripts" / "marketplace-delivery-identity.mjs" + program = ( + f"import({json.dumps(module.resolve().as_uri())})" + ".then((exports) => console.log(JSON.stringify(exports)));" + ) + node = shutil.which("node") + require(node is not None, "node is required for the delivery identity self-test") + completed = subprocess.run( + [node, "--input-type=module", "--eval", program], + text=True, + capture_output=True, + timeout=60, + ) + require( + completed.returncode == 0, + f"marketplace delivery identity module did not evaluate: {completed.stderr.strip()}", + ) + exports = json.loads(completed.stdout) for name, value in ( ("LIVE_INSTALLATION_SOURCE", LIVE_INSTALLATION_SOURCE), ("DEFERRED_INSTALLATION_SOURCE", DEFERRED_INSTALLATION_SOURCE), @@ -619,7 +672,7 @@ def _run_shared_identity_self_tests() -> None: ("FIXTURE_MARKER_PURPOSE", _MARKER_PURPOSE), ): require( - f'export const {name} = "{value}";' in source, + exports.get(name) == value, f"marketplace delivery identity {name} differs between the producer and the verifier", ) diff --git a/.github/scripts/packaged_agent_proof/self_test_process_deadline.py b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py index 6447b0e27..188dd31d8 100644 --- a/.github/scripts/packaged_agent_proof/self_test_process_deadline.py +++ b/.github/scripts/packaged_agent_proof/self_test_process_deadline.py @@ -117,10 +117,20 @@ def _run_shared_deadline_leg() -> None: pass else: raise ProofFailure("search_until_ready did not fail on a host that never converged") + # Equality, not just the bound: an early failure would pass "<=" while never + # exercising the shared deadline, and a nested per-poll budget overshoots it. require( - clock.now <= _TIMEOUT_SECS, + clock.now == _TIMEOUT_SECS, f"search_until_ready waited {clock.now}s against its {_TIMEOUT_SECS}s bound", ) + # The degraded answer must reach the second poll under the same deadline. If + # a poll minted a fresh budget, the first poll would burn the whole window + # and this request id would never exist. + require( + host.tool_attempt_counts.get("search-degraded-2") == 2, + "a nested per-poll deadline never reached the post-degradation poll:" + f" {host.tool_attempt_counts}", + ) def _run_transport_deadline_leg() -> None: diff --git a/.github/scripts/packaged_agent_proof/self_test_process_exit.py b/.github/scripts/packaged_agent_proof/self_test_process_exit.py index e3062c2ea..e2d749368 100644 --- a/.github/scripts/packaged_agent_proof/self_test_process_exit.py +++ b/.github/scripts/packaged_agent_proof/self_test_process_exit.py @@ -4,6 +4,7 @@ import json import os +import re import shutil import subprocess import sys @@ -42,6 +43,40 @@ def _temporary_boundary_ordering_test() -> None: "runtime proof no longer fences the exact native server before" " returning to the temporary-directory boundary", ) + # Order alone is not enough: a ``return`` between the two calls, or a + # conditional guarding the wait, would preserve the substring order above + # while bypassing the fence. + lines = body.splitlines() + proof_line = next( + i + for i, line in enumerate(lines) + if "runtime = prove_runtime(" in line or "prove_single_project_runtime(" in line + ) + cleanup_line = next( + i + for i, line in enumerate(lines) + if "cleanup = wait_for_final_temporary_package_server(" in line + ) + require( + not any( + re.search(r"\breturn\b", line) + for line in lines[proof_line + 1 : cleanup_line] + ), + "run_runtime_proof can return between proving the runtime and fencing" + " the exact native server", + ) + cleanup_indent = len(lines[cleanup_line]) - len(lines[cleanup_line].lstrip()) + enclosing_header = next( + line.strip() + for line in reversed(lines[:cleanup_line]) + if (len(line) - len(line.lstrip())) < cleanup_indent + and line.rstrip().endswith(":") + ) + require( + enclosing_header == "try:", + "the crash-fence wait is guarded by something other than its own" + f" error capture: {enclosing_header}", + ) def _target_os() -> str: diff --git a/.github/scripts/packaged_agent_proof/self_test_process_identity.py b/.github/scripts/packaged_agent_proof/self_test_process_identity.py index d11dfc7aa..b59df1858 100644 --- a/.github/scripts/packaged_agent_proof/self_test_process_identity.py +++ b/.github/scripts/packaged_agent_proof/self_test_process_identity.py @@ -10,7 +10,7 @@ from . import process_identity -from .foundation import ProofFailure +from .foundation import ProofFailure, require from .process_identity import ( live_process_executable_sha256, process_start_identity, @@ -32,6 +32,20 @@ def run_process_identity_self_tests() -> None: pid = os.getpid() start_id = process_start_identity(pid) live_digest = live_process_executable_sha256(pid, start_id, target_os) + # The live digest must equal an independent read of the on-disk executable; + # using the same implementation for both sides would accept any hash bug. + import hashlib + from pathlib import Path + + executable_digests = { + hashlib.sha256(Path(candidate).read_bytes()).hexdigest() + for candidate in {sys.executable, getattr(sys, "_base_executable", None)} + if candidate and Path(candidate).is_file() + } + require( + executable_digests and live_digest in executable_digests, + "live executable hashing disagreed with an independent file read", + ) verified_live_executable( pid=pid, process_start_id=start_id, @@ -190,6 +204,23 @@ def _macos_terminal_lifecycle_test() -> None: ) assert waiter.exited() waiter.close() + # A waiter pinned to a different start identity must not accept the + # same terminal record as exit evidence: without the identity check + # in the terminal-record path, this classifies the zombie as gone. + wrong_identity = identity[:-1] + ("0" if identity[-1] != "0" else "1") + wrong_waiter = process_identity.ExactProcessExitWaiter( + child.pid, wrong_identity, "macos", allow_already_exited=True, + ) + try: + wrong_waiter.exited() + except ProofFailure: + pass + else: + raise ProofFailure( + "a terminal record carrying a different start identity was" + " admitted as exit evidence" + ) + wrong_waiter.close() assert child.wait(timeout=5) == 0 assert process_identity.macos_terminal_process_observation(child.pid) is None finally: diff --git a/.github/scripts/packaged_agent_proof/self_test_producer_liveness.py b/.github/scripts/packaged_agent_proof/self_test_producer_liveness.py index c0e16d524..87a9378f1 100644 --- a/.github/scripts/packaged_agent_proof/self_test_producer_liveness.py +++ b/.github/scripts/packaged_agent_proof/self_test_producer_liveness.py @@ -19,6 +19,7 @@ from .event_producer_liveness import ( ChildProcessProducer, + EventProducer, NativeProcessProducer, ObservationalProducer, ProducerGroup, @@ -139,6 +140,31 @@ def _dead_producer_fails_fast_instead_of_timing_out() -> None: ) +class _AppendingProducer(EventProducer): + """A producer whose exit probe lands just after its final append. + + Its ``exited()`` is not side-effect free like a real producer's contract + requires; it exists to place the awaited record between the wait's poll + read and the exit probe, which is the exact race the drain read exists for. + """ + + def __init__(self, log_path: Path, record: dict) -> None: + super().__init__( + "the self-test producer", + "finishing its last record", + ) + self._log_path = log_path + self._record = record + + def exited(self) -> bool: + with self._log_path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(self._record, sort_keys=True) + "\n") + return True + + def termination(self) -> str: + return "exited after appending its last record" + + def _a_record_written_just_before_exit_still_completes() -> None: """Exit must not beat a record the producer already appended.""" with tempfile.TemporaryDirectory( @@ -174,6 +200,37 @@ def _a_record_written_just_before_exit_still_completes() -> None: "an exited producer lost a record it had already written", ) + with tempfile.TemporaryDirectory( + prefix="codestory-producer-liveness-self-test-" + ) as raw: + # The awaited record lands after the poll read but before the exit + # probe completes: only the drain read after `exited()` can see it. + # This drives wait_for_jsonl_event directly rather than a control so + # the scripted producer stays out of the residency audit's producer + # allowlist; the control path delegates to the same wait. + directory = Path(raw) + log_path = directory / "late.events.jsonl" + record = { + "schema_version": 1, + "sequence": 1, + "action": "snapshot", + "status": "completed", + } + event = wait_for_jsonl_event( + log_path, + lambda candidate: ( + candidate.get("sequence") == 1 + and candidate.get("action") == "snapshot" + ), + timeout=_UNREACHED_TIMEOUT_SECS, + awaited="the late self-test record", + producer=_AppendingProducer(log_path, record), + ) + require( + event.get("sequence") == 1 and event.get("action") == "snapshot", + "a producer exit lost a record appended before the exit probe", + ) + def _timeout_names_the_wait_the_log_and_the_producer() -> None: with tempfile.TemporaryDirectory( diff --git a/.github/scripts/packaged_agent_proof/self_test_publication_crash_exit.py b/.github/scripts/packaged_agent_proof/self_test_publication_crash_exit.py index a945669ef..e10ab6c2c 100644 --- a/.github/scripts/packaged_agent_proof/self_test_publication_crash_exit.py +++ b/.github/scripts/packaged_agent_proof/self_test_publication_crash_exit.py @@ -269,8 +269,14 @@ def _never_exit_fails_by_identity_without_replacement() -> None: def _candidate_exit_fails_before_the_predecessor_probe() -> None: predecessor = _ScriptedPredecessor([False]) candidate = _ScriptedCandidate(exit_on_probe=1) + invocations: list[tuple[float, int | None]] = [] try: - _drive_replacement(predecessor, candidate, allowance_secs=1.0) + _drive_replacement( + predecessor, + candidate, + allowance_secs=1.0, + invocations=invocations, + ) except ProofFailure as error: message = str(error) require( @@ -286,6 +292,11 @@ def _candidate_exit_fails_before_the_predecessor_probe() -> None: predecessor.probes == 0, "the crash fence probed the predecessor before noticing candidate exit", ) + require( + not invocations, + "a dead paused candidate still ran the replacement worker:" + f" {invocations}", + ) def _candidate_exit_wins_the_same_poll_as_predecessor_exit() -> None: diff --git a/.github/scripts/packaged_agent_proof/self_test_server_identity.py b/.github/scripts/packaged_agent_proof/self_test_server_identity.py index 1a5428316..0b67c928a 100644 --- a/.github/scripts/packaged_agent_proof/self_test_server_identity.py +++ b/.github/scripts/packaged_agent_proof/self_test_server_identity.py @@ -129,8 +129,16 @@ def _shared_snapshot_test( } } first_snapshot = server_snapshot(snapshot_payload, manifest, require_resident=True) + # The second host observes the same server at a different moment: every + # identity field the shared proof pins is equal, while the volatile + # observation fields are not -- comparing a clone against itself could + # never show the comparator reads either side. + second_payload = json.loads(json.dumps(snapshot_payload)) + second_payload["embedding_server"]["event_sequence"] = 31 + second_payload["embedding_server"]["scheduler"]["connection_count"] = 5 + second_payload["embedding_server"]["engine"]["successful_encode_count"] = 9 second_snapshot = server_snapshot( - json.loads(json.dumps(snapshot_payload)), + second_payload, manifest, require_resident=True, ) @@ -146,6 +154,23 @@ def _shared_snapshot_test( raise ProofFailure("launcher digest was accepted as the runtime process") shared = shared_server_identity(first_snapshot, second_snapshot) require(shared["model_load_count"] == 1, "shared server identity self-test failed") + second_host = json.loads(json.dumps(second_snapshot)) + second_host["process"]["server_instance_id"] = "server-2" + try: + shared_server_identity(first_snapshot, second_host) + except ProofFailure as error: + require( + str(error) + == "independent plugin hosts observed different" + " process.server_instance_id", + f"a second host's distinct server identity lost its own" + f" attribution: {error}", + ) + else: + raise ProofFailure( + "independent plugin hosts with different server identities were" + " accepted as observing one server" + ) return snapshot_payload, first_snapshot, shared diff --git a/.github/scripts/packaged_agent_proof/self_test_server_idle.py b/.github/scripts/packaged_agent_proof/self_test_server_idle.py index d084f38c5..4d8bff069 100644 --- a/.github/scripts/packaged_agent_proof/self_test_server_idle.py +++ b/.github/scripts/packaged_agent_proof/self_test_server_idle.py @@ -985,6 +985,15 @@ def visit(node: ast.AST, *, conditional: bool) -> None: marks.append( (node.lineno, node.col_offset, "control", node, conditional) ) + if isinstance(node, ast.BoolOp): + # `and`/`or` short-circuit: every operand after the first runs + # only on the paths where the earlier ones did not already decide + # the result. `False and ensure_resident_qualification_server(...)` + # never establishes anything. + visit(node.values[0], conditional=conditional) + for operand in node.values[1:]: + visit(operand, conditional=True) + return branching = _CONDITIONAL_FIELDS.get(type(node), frozenset()) for field, value in ast.iter_fields(node): children = value if isinstance(value, list) else [value] @@ -1019,13 +1028,24 @@ def _establishes_its_own_residency(call: ast.Call) -> bool: The argument has to name something that establishes residency. Merely being present and non-``None`` would let a callable that does no embedding work -- ``lambda: None`` is the honest example -- satisfy the one guard standing - between this harness and the deadlock it just removed. + between this harness and the deadlock it just removed. And an establishing + name that only occurs in the argument without being executed -- a + ``lambda: (ensure_resident_qualification_server, None)[1]`` -- establishes + nothing either, so the name has to be in call position or be the callable + passed by reference. """ establish = _keyword(call, "establish") if establish is None: return False + # Handing the establisher itself to the sender by reference qualifies on + # its own: the sender invokes it for each attempt. + if isinstance(establish, ast.Name): + return establish.id in _RESIDENCY_ESTABLISHING + if isinstance(establish, ast.Attribute): + return establish.attr in _RESIDENCY_ESTABLISHING return any( - isinstance(node, ast.Name) and node.id in _RESIDENCY_ESTABLISHING + isinstance(node, ast.Call) + and _call_name(node) in _RESIDENCY_ESTABLISHING for node in ast.walk(establish) ) @@ -1040,12 +1060,15 @@ def _pins_a_live_producer(call: ast.Call) -> bool: as a control holding none. Naming the two producers that do pin a process keeps the next non-pinning producer out by default, instead of admitting it until someone remembers to exclude it here. + + The pinning producer has to be constructed, not merely named: a class + object passed as data pins no process, so only a call counts. """ producer = _keyword(call, "producer") if producer is None: return False return any( - isinstance(node, ast.Name) and node.id in _PINNING_PRODUCERS + isinstance(node, ast.Call) and _call_name(node) in _PINNING_PRODUCERS for node in ast.walk(producer) ) @@ -1058,7 +1081,10 @@ def _varies_with_its_attempt(call: ast.Call) -> bool: evidence identically therefore cannot both run: the replacement dies on its own output before it starts, and the run reports that instead of the server it lost. So the attempt number has to reach the evidence, not just the - signature. + signature: it has to land in the ``label`` argument of the establishing + call, because that label names the worker's request and output files. An + attempt spent anywhere else -- a log line, an unused computation -- leaves + the label constant and the collision in place. Only an inline lambda is decided here, which is the shape the production call site uses; a named callable is a self-test stand-in that asserts the @@ -1070,10 +1096,21 @@ def _varies_with_its_attempt(call: ast.Call) -> bool: parameters = [argument.arg for argument in establish.args.args] if len(parameters) != 1: return False - return any( - isinstance(node, ast.Name) and node.id == parameters[0] - for node in ast.walk(establish.body) - ) + attempt = parameters[0] + for node in ast.walk(establish.body): + if not isinstance(node, ast.Call): + continue + if _call_name(node) not in _RESIDENCY_ESTABLISHING: + continue + label = _keyword(node, "label") + if label is None: + continue + if any( + isinstance(leaf, ast.Name) and leaf.id == attempt + for leaf in ast.walk(label) + ): + return True + return False def _every_control_is_issued_to_a_server_proven_resident() -> None: diff --git a/.github/scripts/windows-link-timing.mjs b/.github/scripts/windows-link-timing.mjs index 08f13ffd1..65fc6adda 100644 --- a/.github/scripts/windows-link-timing.mjs +++ b/.github/scripts/windows-link-timing.mjs @@ -47,6 +47,12 @@ function fail(message) { } function requireNonNegativeInteger(value, label) { + // A CLI field must be exactly a decimal integer: parseInt silently truncates + // "12abc", "1.5", and "0x10", which would let a malformed duration invent a + // receipt record. + if (typeof value !== "number" && !/^[0-9]+$/u.test(String(value))) { + fail(`${label} must be a non-negative integer`); + } const parsed = typeof value === "number" ? value : Number.parseInt(String(value), 10); if (!Number.isSafeInteger(parsed) || parsed < 0) { fail(`${label} must be a non-negative integer`); diff --git a/.github/scripts/windows-link-timing.test.mjs b/.github/scripts/windows-link-timing.test.mjs index 231e7b608..26d02305b 100644 --- a/.github/scripts/windows-link-timing.test.mjs +++ b/.github/scripts/windows-link-timing.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -269,11 +269,24 @@ test("a malformed selector invocation fails loudly instead of inventing a record [["select", "--input", log, "--out", out], /missing --build-elapsed-ms/u], [["select", "--input", log, "--out", out, "--build-elapsed-ms", "-1"], /must be a non-negative integer/u], [["select", "--input", log, "--out", out, "--build-elapsed-ms", "later"], /must be a non-negative integer/u], + // Permissive parses that Number.parseInt would silently truncate must still + // fail rather than invent a truncated duration. + [["select", "--input", log, "--out", out, "--build-elapsed-ms", "1.5"], /must be a non-negative integer/u], + [["select", "--input", log, "--out", out, "--build-elapsed-ms", "12abc"], /must be a non-negative integer/u], + [["select", "--input", log, "--out", out, "--build-elapsed-ms", "0x10"], /must be a non-negative integer/u], + [["select", "--input", log, "--out", out, "--build-elapsed-ms", "1e3"], /must be a non-negative integer/u], + [["select", "--input", log, "--out", out, "--build-elapsed-ms", "+42"], /must be a non-negative integer/u], + [["select", "--input", log, "--out", out, "--build-elapsed-ms", " 42"], /must be a non-negative integer/u], [["select", "--input", log, "--out", out, "--build-elapsed-ms", "1", "--input", log], /--input may be supplied only once/u], ]; for (const [args, expected] of invocations) { const result = runSelector(args); assert.equal(result.status, 1, `expected ${JSON.stringify(args)} to fail`); assert.match(result.stderr, expected); + assert.equal( + existsSync(out), + false, + `a rejected invocation left a receipt behind: ${JSON.stringify(args)}`, + ); } }); diff --git a/CHANGELOG.md b/CHANGELOG.md index f93bd65ba..98a7bb5a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,40 @@ ## Unreleased +## 0.17.7 + +CodeStory keeps source reads tied to the indexed bytes, waits for refreshes across sessions, and makes indexing failures easier to diagnose and recover from. + +### Read from a coherent index + +- Snippets and source-backed context, search and packet results verify source against the indexed content hash. Changed or missing files receive a typed refusal instead of returning unrelated current text. +- During a refresh, source-backed reads wait for a fresh complete index within the call deadline. Graph navigation may use the retained publication, with its freshness labelled `historical` in the response. Waiting across preparation stages shares one deadline. +- A second session waits for a project already being refreshed and adopts the completed publication. Cancelling the waiting call leaves the writer running. +- Call-path verification reports when its search budget is exhausted, so an incomplete investigation cannot look complete. + +### Index and recover with clearer diagnostics + +- Incremental deletion uses indexed proof-fact lookups, avoiding stalls caused by scanning the surviving graph. +- Files with trailing whitespace in their extensions no longer abort indexing. +- Obsolete index images are reclaimed after publication finishes, including work deferred while another session was publishing. Cleanup preserves foreign entries and verifies the retired directory's identity before removing it. +- A write-lock timeout identifies the holding process and provides guidance to wait for or stop it. A failed freshness probe names the failed stage instead of reporting a misleading zero file count. +- Markdown errors include their cause chain and recovery commands. `doctor --support-bundle ` writes a local redacted diagnostic bundle, and `doctor` lists other cached projects with incompatible schemas and their recovery commands. +- Indexing warns when a repository exceeds the 25,000-file freshness-scan bound; `doctor` reports the same limitation as a warning. + +### Windows and MCP hosts + +- Linked Git worktrees open with the repository identity shared by their sibling worktrees. +- Search-index repair and republishing can replace read-only components, including files shared with earlier generations. Failed publication restores their immutable permissions. +- Rewritten or replaced search-index files are revalidated even when their size and timestamps match the previous file. +- Plugin setup briefly retries a managed CLI folder publication blocked by another program. +- The user guide covers other MCP hosts, the `CODESTORY_CLI` override and recovery from a managed-CLI containment failure. + +### Upgrading + +The core cache moves to schema 36. Existing 0.17.6 indexes are rebuilt into a new generation on the next product call, with a one-time indexing cost; status and diagnostic reads remain observational. An older cache supplies the explicit recovery command `codestory-cli index --project --refresh full`. A cache written by a newer binary is refused with guidance to reset derived state and rebuild. + +Publication stamps move to schema 4 to add freshness metadata and remain compatible with schema 3 clients. Install matching CLI and plugin versions and restart the MCP host. + ## 0.17.6 ### A rebuilt evidence engine for coding agents diff --git a/Cargo.lock b/Cargo.lock index 17471c25a..ae668a956 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -522,7 +522,7 @@ dependencies = [ [[package]] name = "codestory-agent" -version = "0.17.6" +version = "0.17.7" dependencies = [ "codestory-contracts", "serde", @@ -533,7 +533,7 @@ dependencies = [ [[package]] name = "codestory-bench" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "clap", @@ -557,7 +557,7 @@ dependencies = [ [[package]] name = "codestory-cli" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "clap", @@ -589,7 +589,7 @@ dependencies = [ [[package]] name = "codestory-contracts" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "crossbeam-channel", @@ -603,11 +603,12 @@ dependencies = [ "thiserror 2.0.18", "tracing", "uuid", + "windows-sys 0.59.0", ] [[package]] name = "codestory-indexer" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "codestory-contracts", @@ -652,7 +653,7 @@ dependencies = [ [[package]] name = "codestory-llama-sys" -version = "0.17.6" +version = "0.17.7" dependencies = [ "codestory-contracts", "crossbeam-channel", @@ -667,7 +668,7 @@ dependencies = [ [[package]] name = "codestory-retrieval" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "chrono", @@ -692,7 +693,7 @@ dependencies = [ [[package]] name = "codestory-runtime" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "codestory-agent", @@ -714,13 +715,12 @@ dependencies = [ "tantivy", "tempfile", "tracing", - "ureq", "uuid", ] [[package]] name = "codestory-store" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "codestory-contracts", @@ -740,7 +740,7 @@ dependencies = [ [[package]] name = "codestory-workspace" -version = "0.17.6" +version = "0.17.7" dependencies = [ "anyhow", "codestory-contracts", diff --git a/crates/codestory-agent/Cargo.toml b/crates/codestory-agent/Cargo.toml index b5b129f3e..e92172808 100644 --- a/crates/codestory-agent/Cargo.toml +++ b/crates/codestory-agent/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-agent" -version = "0.17.6" +version = "0.17.7" edition = "2024" [features] diff --git a/crates/codestory-bench/Cargo.toml b/crates/codestory-bench/Cargo.toml index 798d3df88..2e55db8af 100644 --- a/crates/codestory-bench/Cargo.toml +++ b/crates/codestory-bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-bench" -version = "0.17.6" +version = "0.17.7" edition = "2024" publish = false diff --git a/crates/codestory-bench/src/bin/codestory_proof_availability/materialize.rs b/crates/codestory-bench/src/bin/codestory_proof_availability/materialize.rs index 7d75b6ad1..05a958416 100644 --- a/crates/codestory-bench/src/bin/codestory_proof_availability/materialize.rs +++ b/crates/codestory-bench/src/bin/codestory_proof_availability/materialize.rs @@ -2276,8 +2276,15 @@ mod tests { ); } + /// Named lane `proof-availability-frozen-checkouts` + /// (docs/contributors/testing-matrix.md): the pinned cohort trees are + /// external checkouts fetched by the source-only `materialize + /// --verify-only` form, so the test is `#[ignore]`d out of the default + /// gate. The default suite covers the same receipt-file binding through + /// the synthetic fixture in + /// `oracle_source_verification_binds_steps_to_the_exact_full_file_bytes`. #[test] - #[ignore = "requires the source-only frozen CodeStory checkout"] + #[ignore = "proof-availability-frozen-checkouts lane: requires fetched oracle-workspaces checkouts"] fn frozen_source_receipts_match_the_exact_pinned_checkouts() { let repository_root = Path::new(env!("CARGO_MANIFEST_DIR")) .parent() @@ -2380,17 +2387,38 @@ mod tests { #[cfg(unix)] #[test] fn destination_overlap_detects_platform_root_aliases() { - let root = tempfile::tempdir().unwrap(); - let spelled_root = root.path().to_path_buf(); - let canonical_root = spelled_root.canonicalize().unwrap(); - if spelled_root == canonical_root { - return; - } - - let workspace = spelled_root.join("workspace"); - let out = spelled_root.join("source-environment.json"); + // The overlap contract must hold whenever a destination is reachable + // through a whitelisted platform root alias (a top-level symlink such + // as macOS /var -> private/var or merged-usr /bin -> usr/bin). + // Discover one dynamically instead of relying on the tempdir spelling: + // on hosts whose tempdir is already canonical the implicit pair never + // exists and the assertions below would pass vacuously. + let (alias_root, canonical_root) = fs::read_dir("/") + .expect("root directory listing") + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + fs::symlink_metadata(path) + .map(|metadata| metadata.file_type().is_symlink()) + .unwrap_or(false) + }) + .find_map(|alias| { + alias + .canonicalize() + .ok() + .filter(|canonical| *canonical != alias) + .map(|canonical| (alias, canonical)) + }) + .expect("a platform root alias must exist on every supported unix host"); + + // No directory is created through the alias: observe_destination only + // walks metadata, so a unique missing leaf keeps the ancestor chain at + // exactly the symlink being exercised. + let nonce = format!("codestory-alias-probe-{}", std::process::id()); + let workspace = alias_root.join(&nonce).join("workspace"); + let out = alias_root.join(&nonce).join("source-environment.json"); let arguments = MaterializeArgs { - corpus: spelled_root.join("unused-corpus.json"), + corpus: alias_root.join(&nonce).join("unused-corpus.json"), workspace: workspace.clone(), cache_root: canonical_root, out: out.clone(), diff --git a/crates/codestory-cli/Cargo.toml b/crates/codestory-cli/Cargo.toml index 1ef222764..d4d0b1b52 100644 --- a/crates/codestory-cli/Cargo.toml +++ b/crates/codestory-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-cli" -version = "0.17.6" +version = "0.17.7" edition = "2024" description = "Local repository evidence and grounding CLI for source-backed coding workflows." license = "Apache-2.0" diff --git a/crates/codestory-cli/src/app.rs b/crates/codestory-cli/src/app.rs index 16a48146f..54ecb9b60 100644 --- a/crates/codestory-cli/src/app.rs +++ b/crates/codestory-cli/src/app.rs @@ -33,8 +33,9 @@ use crate::stdio_catalog::{ }; pub(crate) use artifacts::preflight_output_file; use resolution::{ - StructuredCommandFailure, command_failure_envelope, command_failure_message, - emit_command_failure, generic_command_failure, json_output_requested, requested_output_file, + StructuredCommandFailure, command_failure_details_markdown, command_failure_envelope, + command_failure_message, emit_command_failure, generic_command_failure, json_output_requested, + render_command_failure_markdown, requested_output_file, }; const MAX_DRILL_JOBS: usize = 8; @@ -163,29 +164,34 @@ pub async fn run() -> ExitCode { Err(error) => { crate::diagnostics::record_command_failure(&error); let structured = error.downcast_ref::(); + let envelope = structured + .map(|failure| failure.envelope.clone()) + .or_else(|| { + runtime::api_error_in_chain(&error) + .cloned() + .map(CommandFailureEnvelope::new) + }) + .unwrap_or_else(|| generic_command_failure(&error)); + let output_file = structured + .and_then(|failure| failure.output_file.as_deref()) + .or_else(|| requested_output_file(&raw_args)); if json { - let envelope = structured - .map(|failure| failure.envelope.clone()) - .or_else(|| { - runtime::api_error_in_chain(&error) - .cloned() - .map(CommandFailureEnvelope::new) - }) - .unwrap_or_else(|| generic_command_failure(&error)); - let output_file = structured - .and_then(|failure| failure.output_file.as_deref()) - .or_else(|| requested_output_file(&raw_args)); emit_command_failure(&envelope, output_file); } else { - if let Some(failure) = structured - && let (Some(path), Some(markdown)) = - (failure.output_file.as_deref(), failure.markdown.as_deref()) - && let Err(write_error) = fs::write(path, markdown) + let markdown = structured + .and_then(|failure| failure.markdown.as_deref()) + .map(str::to_owned) + .unwrap_or_else(|| render_command_failure_markdown(&envelope)); + if let Some(path) = output_file + && let Err(write_error) = fs::write(path, &markdown) { eprintln!("Error: failed to write {}: {write_error}", path.display()); return ExitCode::FAILURE; } eprintln!("Error: {}", command_failure_message(&error)); + // The default output owed the same evidence the JSON envelope + // carries: the causes chain and the typed next action. + eprint!("{}", command_failure_details_markdown(&envelope)); } ExitCode::FAILURE } diff --git a/crates/codestory-cli/src/app/agent_context/packet.rs b/crates/codestory-cli/src/app/agent_context/packet.rs index 958088a24..23f22cfec 100644 --- a/crates/codestory-cli/src/app/agent_context/packet.rs +++ b/crates/codestory-cli/src/app/agent_context/packet.rs @@ -110,6 +110,7 @@ pub(in crate::app) fn run_packet(cmd: PacketCommand) -> Result<()> { retrieval_publication: operation.retrieval_publication.clone(), operation_id: operation.operation_id.clone(), attempt: operation.attempt, + freshness: operation.freshness, }; codestory_runtime::finalize_packet_projection_v3_for_representation( &mut operation.value.projection, @@ -270,6 +271,7 @@ pub(in crate::app) fn enforce_packet_cli_json_output_budget( retrieval_publication: operation.retrieval_publication.clone(), operation_id: operation.operation_id.clone(), attempt: operation.attempt, + freshness: operation.freshness, }; let _ = render_public_operation_json_content(&envelope, &operation.value)?; codestory_runtime::enforce_packet_output_budget_for_representation( diff --git a/crates/codestory-cli/src/app/diagnostics/doctor.rs b/crates/codestory-cli/src/app/diagnostics/doctor.rs index 86425f031..2b783da5c 100644 --- a/crates/codestory-cli/src/app/diagnostics/doctor.rs +++ b/crates/codestory-cli/src/app/diagnostics/doctor.rs @@ -3,7 +3,7 @@ use super::super::readiness_commands::doctor_sidecar_status_is_live_ready; use super::super::resolution::quote_command_path; use super::readiness::{agent_readiness_status, build_readiness_lanes_for_runtime}; use super::sidecar::{build_summary_readiness, doctor_sidecar_status}; -use crate::args::{DoctorCheckOutput, DoctorOutput, RetrievalStatusOutput}; +use crate::args::{DoctorCheckOutput, DoctorOutput, DoctorStaleCachedCore, RetrievalStatusOutput}; use crate::display; use crate::embedding_config; use crate::readiness; @@ -94,6 +94,39 @@ pub(in crate::app) fn build_doctor_output( checks.push(index_freshness_check(freshness)); } + // Enumerate sibling project caches under this runtime's own process cache + // root. The scan is strictly observational: each cache is opened through + // the non-mutating schema reader, never read-write, so reporting a stale + // core can never migrate it. + let stale_cached_cores = codestory_runtime::observe_stale_cached_cores( + runtime.sidecar.process_cache_root(), + &runtime.cache_root, + ) + .into_iter() + .map(|entry| { + let project_root = entry + .project_root + .map(|root| display::clean_path_string(&root.to_string_lossy())); + DoctorStaleCachedCore { + cache_dir: display::clean_path_string(&entry.cache_dir.to_string_lossy()), + project_root, + found_schema: entry.found_schema, + required_schema: entry.required_schema, + next_action: entry.next_action, + } + }) + .collect::>(); + if !stale_cached_cores.is_empty() { + checks.push(doctor_check( + "cached_cores", + "warn", + format!( + "{} other cached project(s) have an incompatible core schema; follow each cache's next_action, or use a matching CodeStory version when its project root is unavailable.", + stale_cached_cores.len() + ), + )); + } + // Reported settings are observed through the registry so a secret-marked // value can never reach a doctor line, whatever this list grows to hold. // The list itself lives in the registry too: naming these identities here @@ -129,6 +162,7 @@ pub(in crate::app) fn build_doctor_output( readiness_lanes, checks, next_commands, + stale_cached_cores, environment, } } @@ -257,14 +291,19 @@ pub(in crate::app::diagnostics) fn index_freshness_check( freshness.duration_ms ), ), - IndexFreshnessStatusDto::NotChecked => doctor_check( - "index_freshness", - "info", - format!( - "Index freshness was not checked: {}.", - freshness.reason.as_deref().unwrap_or("no reason reported") - ), - ), + IndexFreshnessStatusDto::NotChecked => { + if let Some(warning) = readiness::bounded_inventory_freshness_warning(freshness) { + return doctor_check("index_freshness", "warn", warning); + } + doctor_check( + "index_freshness", + "info", + format!( + "Index freshness was not checked: {}.", + freshness.reason.as_deref().unwrap_or("no reason reported") + ), + ) + } } } @@ -654,3 +693,57 @@ mod rollback_recommendation_tests { ); } } + +#[cfg(test)] +mod freshness_check_tests { + use super::*; + use codestory_contracts::api::IndexFreshnessNotCheckedCauseDto; + + fn not_checked(cause: IndexFreshnessNotCheckedCauseDto) -> IndexFreshnessDto { + IndexFreshnessDto { + status: IndexFreshnessStatusDto::NotChecked, + changed_file_count: 0, + new_file_count: 0, + removed_file_count: 0, + checked_file_count: 0, + indexed_file_count: 30_000, + duration_ms: 0, + reason: Some( + "indexed file inventory exceeds bounded freshness cap (30000 > 25000)".to_string(), + ), + not_checked_cause: Some(cause), + samples: Vec::new(), + } + } + + #[test] + fn a_bounded_inventory_is_a_warn_not_an_info_line() { + // Repositories past the scale envelope are a supported-but-bounded + // state the operator should see, not a trivia footnote. + let check = index_freshness_check(¬_checked( + IndexFreshnessNotCheckedCauseDto::BoundedInventory, + )); + + assert_eq!(check.name, "index_freshness"); + assert_eq!(check.status, "warn"); + assert!( + check.message.contains("30000 > 25000"), + "the warning must carry the observed bound, not a bare label: {}", + check.message + ); + assert!( + check.message.contains("remains usable"), + "the warning must say the index stays usable: {}", + check.message + ); + } + + #[test] + fn an_unavailable_inventory_stays_informational() { + let check = index_freshness_check(¬_checked( + IndexFreshnessNotCheckedCauseDto::InventoryUnavailable, + )); + + assert_eq!(check.status, "info"); + } +} diff --git a/crates/codestory-cli/src/app/ground_smoke.rs b/crates/codestory-cli/src/app/ground_smoke.rs index 32dd63d68..52c357c5c 100644 --- a/crates/codestory-cli/src/app/ground_smoke.rs +++ b/crates/codestory-cli/src/app/ground_smoke.rs @@ -87,6 +87,7 @@ pub(super) fn run_smoke(cmd: SmokeCommand) -> Result<()> { embedding_retry: None, disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, }, )) .with_context(serde_json::to_value(&output).context("serialize smoke failure context")?); diff --git a/crates/codestory-cli/src/app/index_command.rs b/crates/codestory-cli/src/app/index_command.rs index 37e628c2f..5ebd12539 100644 --- a/crates/codestory-cli/src/app/index_command.rs +++ b/crates/codestory-cli/src/app/index_command.rs @@ -143,6 +143,13 @@ fn run_index_once(cmd: &IndexCommand) -> Result<()> { &sidecar_retrieval, ); let next_commands = readiness::compatibility_next_commands(&readiness); + let warnings = opened + .summary + .freshness + .as_ref() + .and_then(readiness::bounded_inventory_freshness_warning) + .into_iter() + .collect(); let output = IndexOutput { project: &opened.summary.root, storage_path: &storage_path, @@ -152,6 +159,7 @@ fn run_index_once(cmd: &IndexCommand) -> Result<()> { retrieval, phase_timings: opened.phase_timings.as_ref(), summary_generation: summary_generation.as_ref(), + warnings, readiness, next_commands, }; @@ -299,7 +307,10 @@ mod tests { let published_mode = || { observer .project - .complete_index_publication_at(std::path::Path::new(storage_path)) + .complete_index_publication_at( + &observer.project_root, + std::path::Path::new(storage_path), + ) .expect("read command publication") .expect("complete command publication") .mode diff --git a/crates/codestory-cli/src/app/readiness_commands/doctor.rs b/crates/codestory-cli/src/app/readiness_commands/doctor.rs index 541b7d636..e8e097d57 100644 --- a/crates/codestory-cli/src/app/readiness_commands/doctor.rs +++ b/crates/codestory-cli/src/app/readiness_commands/doctor.rs @@ -12,15 +12,16 @@ use crate::args::{ use crate::display::quote_command_path; use crate::output::{emit, render_doctor_markdown, render_ready_markdown}; use crate::runtime::{RuntimeContext, api_error_in_chain}; -use anyhow::Result; +use anyhow::{Context, Result}; use codestory_contracts::api::{ - ProjectSummary, ReadinessStatusDto, ReadinessVerdictDto, StorageStatsDto, + ApiErrorDetails, ProjectSummary, ReadinessStatusDto, ReadinessVerdictDto, StorageStatsDto, }; struct ObservedCore { summary: ProjectSummary, status: Option, reason: Option, + recovery: Option>, } fn observe_core(runtime: &RuntimeContext) -> Result { @@ -29,23 +30,33 @@ fn observe_core(runtime: &RuntimeContext) -> Result { summary, status: None, reason: None, + recovery: None, }), Ok(None) => Ok(ObservedCore { summary: unavailable_summary(runtime), status: Some(DiagnosticCoreStatus::Unavailable), reason: Some("No core cache database is available for this project.".to_string()), + recovery: None, }), Err(error) => { let Some(api_error) = api_error_in_chain(&error) else { return Err(error); }; - if api_error.code != "core_schema_upgrade_required" { + if !matches!( + api_error.code.as_str(), + "core_schema_upgrade_required" | "core_schema_too_new" + ) { return Err(error); } Ok(ObservedCore { summary: unavailable_summary(runtime), - status: Some(DiagnosticCoreStatus::UpgradeRequired), + status: Some(if api_error.code == "core_schema_too_new" { + DiagnosticCoreStatus::NewerSchema + } else { + DiagnosticCoreStatus::UpgradeRequired + }), reason: Some(api_error.message.clone()), + recovery: api_error.details.clone(), }) } } @@ -72,6 +83,7 @@ fn mark_unavailable_verdicts( runtime: &RuntimeContext, verdicts: &mut [ReadinessVerdictDto], reason: &str, + recovery: Option<&ApiErrorDetails>, ) { let project = quote_command_path(&runtime.project_root); let index_command = format!("codestory-cli index --project {project} --refresh full"); @@ -79,8 +91,13 @@ fn mark_unavailable_verdicts( for verdict in verdicts { verdict.status = ReadinessStatusDto::RepairIndex; verdict.summary = reason.to_string(); - verdict.minimum_next = vec![index_command.clone()]; - verdict.full_repair = vec![index_command.clone(), doctor_command.clone()]; + if let Some(recovery) = recovery { + verdict.minimum_next = recovery.minimum_next.clone(); + verdict.full_repair = recovery.full_repair.clone(); + } else { + verdict.minimum_next = vec![index_command.clone()]; + verdict.full_repair = vec![index_command.clone(), doctor_command.clone()]; + } } } @@ -89,9 +106,10 @@ fn mark_unavailable_doctor( output: &mut DoctorOutput, status: DiagnosticCoreStatus, reason: &str, + recovery: Option<&ApiErrorDetails>, ) { output.core_status = Some(status); - mark_unavailable_verdicts(runtime, &mut output.readiness, reason); + mark_unavailable_verdicts(runtime, &mut output.readiness, reason, recovery); output.readiness_lanes = build_readiness_lanes_for_runtime(runtime, &output.readiness, None, None); output.next_commands = crate::readiness::compatibility_next_commands(&output.readiness); @@ -106,11 +124,23 @@ fn mark_unavailable_doctor( pub(in crate::app) fn run_doctor(cmd: DoctorCommand) -> Result<()> { ensure_dot_only_for_trail(cmd.format, "doctor")?; preflight_output_file(cmd.output_file.as_deref())?; + preflight_output_file(cmd.support_bundle.as_deref())?; let runtime = RuntimeContext::new_inspect_only(&cmd.project)?; let observed = observe_core(&runtime)?; let mut output = build_doctor_output(&runtime, &observed.summary); if let (Some(status), Some(reason)) = (observed.status, observed.reason.as_deref()) { - mark_unavailable_doctor(&runtime, &mut output, status, reason); + mark_unavailable_doctor( + &runtime, + &mut output, + status, + reason, + observed.recovery.as_deref(), + ); + } + if let Some(path) = cmd.support_bundle.as_deref() { + let report = + serde_json::to_value(&output).context("serialize doctor report for support bundle")?; + crate::diagnostics::write_support_bundle(path, &report)?; } let markdown = render_doctor_markdown(&output); emit(cmd.format, &output, markdown, cmd.output_file.as_deref()) @@ -127,12 +157,18 @@ pub(in crate::app) fn run_ready(cmd: ReadyCommand) -> Result<()> { fn build_ready_output(cmd: &ReadyCommand) -> Result { let runtime = RuntimeContext::new_inspect_only(&cmd.project)?; let agent_run_id = cmd.run_id.as_deref(); - let (summary, local_refresh, core_status, core_reason) = if cmd.wait_fresh { + let (summary, local_refresh, core_status, core_reason, core_recovery) = if cmd.wait_fresh { let (summary, local_refresh) = wait_for_local_freshness(&cmd.project, &runtime)?; - (summary, local_refresh, None, None) + (summary, local_refresh, None, None, None) } else { let observed = observe_core(&runtime)?; - (observed.summary, None, observed.status, observed.reason) + ( + observed.summary, + None, + observed.status, + observed.reason, + observed.recovery, + ) }; let readiness_sidecar = if matches!(cmd.goal, None | Some(args::ReadyGoal::Agent)) { agent_readiness_status(&runtime, agent_run_id) @@ -151,7 +187,7 @@ fn build_ready_output(cmd: &ReadyCommand) -> Result { &readiness_sidecar, ); if let Some(reason) = core_reason.as_deref() { - mark_unavailable_verdicts(&runtime, &mut verdicts, reason); + mark_unavailable_verdicts(&runtime, &mut verdicts, reason, core_recovery.as_deref()); } let readiness_lanes = build_readiness_lanes_for_runtime( &runtime, diff --git a/crates/codestory-cli/src/app/resolution.rs b/crates/codestory-cli/src/app/resolution.rs index b18e8a72a..fdc40969b 100644 --- a/crates/codestory-cli/src/app/resolution.rs +++ b/crates/codestory-cli/src/app/resolution.rs @@ -2,9 +2,10 @@ mod failure; mod target; pub(super) use failure::{ - StructuredCommandFailure, command_failure_envelope, command_failure_message, - emit_command_failure, generic_command_failure, json_output_requested, - quote_command_argument_value, quote_command_path, quote_command_value, requested_output_file, + StructuredCommandFailure, command_failure_details_markdown, command_failure_envelope, + command_failure_message, emit_command_failure, generic_command_failure, json_output_requested, + quote_command_argument_value, quote_command_path, quote_command_value, + render_command_failure_markdown, requested_output_file, }; pub(crate) use target::{build_ambiguous_target_error_output, resolve_target_or_emit_ambiguity}; pub(super) use target::{ diff --git a/crates/codestory-cli/src/app/resolution/failure.rs b/crates/codestory-cli/src/app/resolution/failure.rs index 84920c001..35141ac6a 100644 --- a/crates/codestory-cli/src/app/resolution/failure.rs +++ b/crates/codestory-cli/src/app/resolution/failure.rs @@ -2,6 +2,7 @@ use crate::{args::SearchHitOutput, display, runtime}; use codestory_contracts::api::{ApiError, ApiErrorDetails, CommandFailureEnvelope}; use std::{ ffi::{OsStr, OsString}, + fmt::Write as _, fs, path::{Path, PathBuf}, }; @@ -42,6 +43,7 @@ pub(in crate::app) fn command_failure_envelope( embedding_retry: None, disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, }, )) .with_context(context) @@ -106,6 +108,64 @@ pub(in crate::app) fn emit_command_failure( println!("{json}"); } +/// The evidence lines a failing command owes a human reading the default +/// output: the same `context.causes` chain and `next_action`/`next_commands` +/// guidance the JSON envelope already carries. +pub(in crate::app) fn command_failure_details_markdown( + envelope: &CommandFailureEnvelope, +) -> String { + let mut markdown = String::new(); + let error = &envelope.error; + let _ = writeln!(markdown, "code: {}", error.code); + if let Some(details) = error.details.as_deref() + && let Some(layer) = details.failed_layer.as_deref() + { + let _ = writeln!(markdown, "failed_layer: {layer}"); + } + if let Some(causes) = envelope + .context + .as_ref() + .and_then(|context| context.get("causes")) + .and_then(serde_json::Value::as_array) + && !causes.is_empty() + { + let _ = writeln!(markdown, "causes:"); + for cause in causes { + let cause = cause + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| cause.to_string()); + let _ = writeln!(markdown, "- {cause}"); + } + } + if let Some(details) = error.details.as_deref() { + for action in &details.minimum_next { + let _ = writeln!(markdown, "next_action: {action}"); + } + if details.next_commands.len() > details.minimum_next.len() { + let _ = writeln!(markdown, "next_commands:"); + for command in details + .next_commands + .iter() + .skip(details.minimum_next.len()) + { + let _ = writeln!(markdown, "- `{command}`"); + } + } + } + markdown +} + +/// The complete markdown document for a failed command, written to +/// `--output-file` when the run did not ask for JSON. +pub(in crate::app) fn render_command_failure_markdown(envelope: &CommandFailureEnvelope) -> String { + let mut markdown = String::new(); + let _ = writeln!(markdown, "# Command Error"); + let _ = writeln!(markdown, "message: {}", envelope.error.message); + markdown.push_str(&command_failure_details_markdown(envelope)); + markdown +} + pub(in crate::app) fn quote_command_path(path: &Path) -> String { display::quote_command_path(path) } diff --git a/crates/codestory-cli/src/app/resolution/target.rs b/crates/codestory-cli/src/app/resolution/target.rs index 5138e309d..a4979d83f 100644 --- a/crates/codestory-cli/src/app/resolution/target.rs +++ b/crates/codestory-cli/src/app/resolution/target.rs @@ -101,6 +101,7 @@ pub(in crate::app) fn ambiguous_command_failure( embedding_retry: None, disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, }, )) .with_context(serde_json::json!({ diff --git a/crates/codestory-cli/src/app/source_commands/source_read.rs b/crates/codestory-cli/src/app/source_commands/source_read.rs index 0a610cca9..228f76126 100644 --- a/crates/codestory-cli/src/app/source_commands/source_read.rs +++ b/crates/codestory-cli/src/app/source_commands/source_read.rs @@ -27,7 +27,7 @@ pub(in crate::app) fn run_snippet(cmd: SnippetCommand) -> Result<()> { let operation = if cmd.target.query.is_some() { "graph_assisted" } else { - "graph" + "source_snippet" }; let colorize = cmd.format == args::OutputFormat::Markdown && cmd.output_file.is_none() diff --git a/crates/codestory-cli/src/app/tests/drill.rs b/crates/codestory-cli/src/app/tests/drill.rs index 182d1ceb0..b437af44d 100644 --- a/crates/codestory-cli/src/app/tests/drill.rs +++ b/crates/codestory-cli/src/app/tests/drill.rs @@ -133,6 +133,7 @@ fn legacy_supported_cannot_change_public_v3_drill_decisions() { retrieval_publication: None, operation_id: "legacy-authority-regression".to_string(), attempt: 1, + freshness: codestory_runtime::OperationFreshness::Fresh, }; let supported_dir = tempdir().expect("supported output dir"); write_drill_outputs( @@ -176,6 +177,7 @@ fn legacy_supported_cannot_change_public_v3_drill_decisions() { retrieval_publication: None, operation_id: "legacy-authority-regression".to_string(), attempt: 1, + freshness: codestory_runtime::OperationFreshness::Fresh, }; let mutated_dir = tempdir().expect("mutated output dir"); write_drill_outputs( @@ -327,6 +329,7 @@ fn drill_retained_fields_match_pre_adapter_fixture() { retrieval_publication: None, operation_id: "test-drill".to_string(), attempt: 1, + freshness: codestory_runtime::OperationFreshness::Fresh, }; write_drill_outputs(args::OutputFormat::Json, output_dir.path(), &operation) .expect("write drill fixtures"); diff --git a/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs b/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs index bda39c100..a3fdeefa1 100644 --- a/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs +++ b/crates/codestory-cli/src/app/tests/lifecycle/packet_diagnostics.rs @@ -82,6 +82,7 @@ fn packet_cli_json_budget_measures_publication_metadata_and_newline() { retrieval_publication: None, operation_id: "public-packet-budget".to_string(), attempt: 1, + freshness: codestory_runtime::OperationFreshness::Fresh, }; enforce_packet_cli_json_output_budget( Path::new("/workspace/project"), @@ -198,6 +199,7 @@ fn packet_cli_sixteen_row_identity_envelope_stays_complete() { }), operation_id: "public-operation-123456789".to_owned(), attempt: 2, + freshness: codestory_runtime::OperationFreshness::Fresh, }; let measured = codestory_runtime::finalize_packet_projection_v3_for_representation( diff --git a/crates/codestory-cli/src/app/tests/rendering/index.rs b/crates/codestory-cli/src/app/tests/rendering/index.rs index 0b0f1b61f..58fd869ca 100644 --- a/crates/codestory-cli/src/app/tests/rendering/index.rs +++ b/crates/codestory-cli/src/app/tests/rendering/index.rs @@ -77,6 +77,7 @@ fn render_index_markdown_includes_rich_timing_breakdown_when_available() { retrieval: &retrieval, phase_timings: Some(&timings), summary_generation: None, + warnings: Vec::new(), readiness: Vec::new(), next_commands: Vec::new(), }; @@ -165,3 +166,65 @@ fn render_index_markdown_includes_rich_timing_breakdown_when_available() { "resolution_semantic_requests: call_rows=36 call_unique=37 call_skipped=38 import_rows=39 import_unique=40 import_skipped=41" )); } + +#[test] +fn index_output_carries_the_bounded_inventory_warning_in_json_and_markdown() { + // #2531#6: a repository past the freshness-scan envelope must surface a + // visible warning, not silently succeed. The warning text comes from the + // shared readiness helper so markdown, JSON and doctor cannot diverge. + let mut summary = summary_with_files(3); + summary.freshness = Some(codestory_contracts::api::IndexFreshnessDto { + status: codestory_contracts::api::IndexFreshnessStatusDto::NotChecked, + changed_file_count: 0, + new_file_count: 0, + removed_file_count: 0, + checked_file_count: 0, + indexed_file_count: 30_000, + duration_ms: 0, + reason: Some( + "indexed file inventory exceeds bounded freshness cap (30000 > 25000)".to_string(), + ), + not_checked_cause: Some( + codestory_contracts::api::IndexFreshnessNotCheckedCauseDto::BoundedInventory, + ), + samples: Vec::new(), + }); + let retrieval = sample_retrieval(); + let warnings = summary + .freshness + .as_ref() + .and_then(crate::readiness::bounded_inventory_freshness_warning) + .into_iter() + .collect(); + let output = IndexOutput { + project: &summary.root, + storage_path: "C:/repo/.cache/index.sqlite", + refresh: "full", + refresh_reason: None, + summary: &summary, + retrieval: &retrieval, + phase_timings: None, + summary_generation: None, + warnings, + readiness: Vec::new(), + next_commands: Vec::new(), + }; + + let json = serde_json::to_value(&output).expect("serialize index output"); + let json_warnings = json["warnings"].as_array().expect("warnings array"); + assert_eq!(json_warnings.len(), 1); + assert!( + json_warnings[0] + .as_str() + .expect("warning text") + .contains("freshness-scan envelope"), + "{json_warnings:?}" + ); + + let markdown = render_index_markdown(&output); + assert!( + markdown.contains("warning: The indexed file inventory exceeds"), + "{markdown}" + ); + assert!(markdown.contains("30000 > 25000"), "{markdown}"); +} diff --git a/crates/codestory-cli/src/app/tests/test_support.rs b/crates/codestory-cli/src/app/tests/test_support.rs index c71a02aaf..a5832e0be 100644 --- a/crates/codestory-cli/src/app/tests/test_support.rs +++ b/crates/codestory-cli/src/app/tests/test_support.rs @@ -403,14 +403,16 @@ pub(super) fn sample_phase_timings() -> IndexingPhaseTimings { }), incremental_plan_probe: Some(IncrementalPlanProbeTimings { outcome: IncrementalPlanProbeOutcomeDto::PlanNotEmpty, + probe_unavailable_stage: None, probe_ms: 4, - files_to_index: 2, - files_to_remove: 1, + files_to_index: Some(2), + files_to_remove: Some(1), live_database_file_bytes: 4_096, skipped_database_copies: 0, skipped_database_copy_bytes: 0, skipped_search_state_rebuild: false, }), + core_retention: None, setup_existing_projection_ids_ms: Some(11), setup_seed_symbol_table_ms: Some(12), flush_files_ms: Some(13), diff --git a/crates/codestory-cli/src/args.rs b/crates/codestory-cli/src/args.rs index 1bd1fb37a..63a12bf81 100644 --- a/crates/codestory-cli/src/args.rs +++ b/crates/codestory-cli/src/args.rs @@ -667,6 +667,12 @@ pub(crate) struct DoctorCommand { help = "Write command output to this file instead of stdout. The parent directory must already exist." )] pub(crate) output_file: Option, + #[arg( + long, + value_name = "PATH", + help = "Also write a support bundle to this file: the doctor report plus this process's redacted diagnostics records. The parent directory must already exist." + )] + pub(crate) support_bundle: Option, } #[derive(Args, Debug)] @@ -1672,6 +1678,8 @@ pub(crate) struct IndexOutput<'a> { #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) summary_generation: Option<&'a SummaryGenerationDto>, #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) warnings: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) readiness: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub(crate) next_commands: Vec, @@ -1682,6 +1690,7 @@ pub(crate) struct IndexOutput<'a> { pub(crate) enum DiagnosticCoreStatus { Unavailable, UpgradeRequired, + NewerSchema, } #[derive(Debug, Serialize)] @@ -2526,9 +2535,24 @@ pub(crate) struct DoctorOutput { pub(crate) readiness_lanes: BTreeMap, pub(crate) checks: Vec, pub(crate) next_commands: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub(crate) stale_cached_cores: Vec, pub(crate) environment: Vec, } +/// Another project cache under the process cache root whose core schema this +/// binary cannot serve until it is re-indexed. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct DoctorStaleCachedCore { + pub(crate) cache_dir: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) project_root: Option, + pub(crate) found_schema: u32, + pub(crate) required_schema: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) next_action: Option, +} + #[derive(Debug, Clone)] pub(crate) enum TargetSelection { Id(NodeId), diff --git a/crates/codestory-cli/src/config.rs b/crates/codestory-cli/src/config.rs index 392ff4f45..59623bc63 100644 --- a/crates/codestory-cli/src/config.rs +++ b/crates/codestory-cli/src/config.rs @@ -380,6 +380,7 @@ pub(crate) fn config_env_test_lock() -> std::sync::MutexGuard<'static, ()> { mod tests { use super::*; use anyhow::Result; + use serde_json::{Value, json}; use std::ffi::OsString; use tempfile::tempdir; @@ -728,13 +729,74 @@ summary_model = "trusted/model" let mut startup = isolated_startup(); startup.user_home = Some(home.path().to_path_buf()); - let (_, warnings) = load_config_report(project.path(), &startup)?; + let (config, warnings) = load_config_report(project.path(), &startup)?; assert!( warnings.is_empty(), "registered keys must not be reported as unknown: {warnings:?}" ); + // Registry completeness alone is not enough: a registered key that no + // CliConfig field claims produces no warning, so each key must prove + // the literal nondefault value it wrote actually landed. + for entry in codestory_contracts::config_registry::CONFIG_FILE_KEYS { + let written = match entry.kind { + codestory_contracts::config_registry::SettingKind::Boolean => json!(true), + codestory_contracts::config_registry::SettingKind::Integer => json!(1), + _ => json!("value"), + }; + let loaded = match entry.key { + codestory_contracts::config_registry::CONFIG_SCHEMA_VERSION_KEY => { + // Consumed by the version gate itself, not a CliConfig field. + Some(written.clone()) + } + "cache_dir" => config + .cache_dir + .as_ref() + .map(|path| json!(path.to_string_lossy())), + "hybrid_retrieval_enabled" => config.hybrid_retrieval_enabled.map(Value::from), + "semantic_doc_alias_mode" => config + .semantic_doc_alias_mode + .as_ref() + .map(|value| json!(value)), + "semantic_doc_scope" => { + config.semantic_doc_scope.as_ref().map(|value| json!(value)) + } + "summary_endpoint" => config.summary_endpoint.as_ref().map(|value| json!(value)), + "summary_model" => config.summary_model.as_ref().map(|value| json!(value)), + other => panic!("registry key {other} has no proven CliConfig mapping"), + }; + assert_eq!( + loaded.as_ref(), + Some(&written), + "registered key {} did not load its written value", + entry.key + ); + } + + // The runtime overrides carry the retrieval-facing subset verbatim. + let overrides = config.runtime_overrides(); + assert_eq!( + overrides.hybrid_retrieval_enabled, + config.hybrid_retrieval_enabled + ); + assert_eq!( + overrides.semantic_doc_scope.as_deref(), + config.semantic_doc_scope.as_deref() + ); + assert_eq!( + overrides.semantic_doc_alias_mode.as_deref(), + config.semantic_doc_alias_mode.as_deref() + ); + assert_eq!( + overrides.summary_endpoint.as_deref(), + config.summary_endpoint.as_deref() + ); + assert_eq!( + overrides.summary_model.as_deref(), + config.summary_model.as_deref() + ); + Ok(()) } diff --git a/crates/codestory-cli/src/diagnostics.rs b/crates/codestory-cli/src/diagnostics.rs index 31a4d9d94..f45e9ca3e 100644 --- a/crates/codestory-cli/src/diagnostics.rs +++ b/crates/codestory-cli/src/diagnostics.rs @@ -106,6 +106,99 @@ pub(crate) fn record_command_failure(error: &anyhow::Error) { let _ = process_diagnostics().write_record(record); } +/// A support bundle is the user-facing triage artifact: the command's own +/// report plus every diagnostics record this process cache root already holds. +/// The records carry only what was safe at capture time (panic site +/// file:line:column, payload sizes, event classes), so bundling them adds no +/// new disclosure and there is no unredacted mode. +const SUPPORT_BUNDLE_SCHEMA_VERSION: u32 = 1; +const SUPPORT_BUNDLE_MAX_DIAGNOSTIC_BYTES: u64 = 4 * 1024 * 1024; + +pub(crate) fn write_support_bundle(path: &Path, report: &Value) -> Result<()> { + let sink = process_diagnostics(); + let (diagnostics, diagnostics_truncated) = collect_diagnostic_records(&sink); + let bundle = json!({ + "schema_version": SUPPORT_BUNDLE_SCHEMA_VERSION, + "generated_at_unix_ms": unix_timestamp_ms(), + "correlation_id": sink.correlation_id.clone(), + "report": report, + "diagnostics": diagnostics, + "diagnostics_truncated": diagnostics_truncated, + }); + let mut encoded = serde_json::to_vec_pretty(&bundle).context("encode support bundle")?; + encoded.push(b'\n'); + refuse_symlink(path)?; + let mut file = open_private_truncated_file(path) + .with_context(|| format!("write support bundle {}", path.display()))?; + file.write_all(&encoded) + .with_context(|| format!("write support bundle {}", path.display()))?; + file.sync_all() + .with_context(|| format!("sync support bundle {}", path.display()))?; + Ok(()) +} + +/// Read the records the sink already persisted. Reading must stay +/// observational: a missing or unreadable log contributes nothing, and the +/// reader never creates the diagnostics directory or touches the record lock. +fn collect_diagnostic_records(sink: &DiagnosticSink) -> (Vec, bool) { + let mut records = Vec::new(); + let mut bytes: u64 = 0; + let mut truncated = false; + let mut read_file = |path: &Path, records: &mut Vec| { + let Ok(content) = fs::read(path) else { + return; + }; + for line in content.split(|byte| *byte == b'\n') { + if line.is_empty() { + continue; + } + if bytes.saturating_add(line.len() as u64) > SUPPORT_BUNDLE_MAX_DIAGNOSTIC_BYTES { + truncated = true; + return; + } + match serde_json::from_slice::(line) { + Ok(record) => { + bytes += line.len() as u64; + records.push(record); + } + Err(_) => { + records.push(json!({ + "event": "unparseable_diagnostic_record", + "bytes": line.len(), + })); + } + } + } + }; + let directory = sink.diagnostics_dir(); + read_file(&directory.join(LOG_FILE), &mut records); + for index in 1..=RETAINED_LOGS { + read_file( + &rotated_path(&directory.join(LOG_FILE), index), + &mut records, + ); + } + for namespace in ["emergency-", "fail-stop-"] { + let Ok(entries) = fs::read_dir(&directory) else { + continue; + }; + let mut candidates = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(namespace)) + }) + .collect::>(); + candidates.sort(); + for candidate in candidates { + read_file(&candidate, &mut records); + } + } + (records, truncated) +} + fn command_failure_record(error: &anyhow::Error) -> Value { let chain_count = error.chain().take(MAX_ERROR_CHAIN_COUNT + 1).count(); json!({ @@ -848,16 +941,29 @@ mod tests { query = "must stay redacted", "packet entry observation" ); + // Hostile guard: an allowlisted field carrying arbitrary private + // text must still be redacted — the allowlist admits digest lists + // only. + tracing::warn!( + raf_ranked_identity_digests = "unlabeled private query payload", + "hostile digest field" + ); }); let rows = read_jsonl(&sink.log_path())?; - assert_eq!(rows.len(), 1); + assert_eq!(rows.len(), 2); let fields = &rows[0]["fields"]; assert_eq!(fields["raf_ranked_identity_digests"], ranked); assert_eq!(fields["raf_admitted_identity_digests"], admitted); assert_eq!(fields["raf_final_identity_digests"], final_support); assert_eq!(fields["query"], "[redacted]"); assert_eq!(fields["message"], REDACTED); + let hostile = serde_json::to_string(&rows[1])?; + assert_eq!( + rows[1]["fields"]["raf_ranked_identity_digests"], "[redacted]", + "a non-digest payload in an allowlisted field must not survive: {hostile}" + ); + assert!(!hostile.contains("unlabeled private query payload")); Ok(()) } @@ -1070,6 +1176,30 @@ mod tests { bounded_locks::release(&lock)?; let files = evidence_files(&sink.diagnostics_dir(), "emergency-", ".jsonl")?; + // Positive controls first: a fallback writer that emitted nothing at + // all must not satisfy this test. + assert_eq!( + files.len(), + EMERGENCY_LOG_SLOTS, + "every contended write must reach a bounded emergency slot" + ); + let mut survived = Vec::new(); + for path in &files { + let row: Value = serde_json::from_slice(&fs::read(path)?) + .expect("each emergency slot holds one JSON record"); + assert_eq!(row["event"], json!("lock_contention")); + assert_eq!(row["correlation_id"], json!("emergency-test")); + survived.push(row["index"].as_u64().expect("emergency row index")); + } + survived.sort_unstable(); + let expected = ((EMERGENCY_LOG_SLOTS * 2)..(EMERGENCY_LOG_SLOTS * 3)) + .map(|index| index as u64) + .collect::>(); + assert_eq!( + survived, expected, + "each slot must retain its newest write, so the surviving rows are \ + the last EMERGENCY_LOG_SLOTS indexes" + ); assert!(files.len() <= EMERGENCY_LOG_SLOTS); assert!( evidence_namespace_count(&sink.diagnostics_dir(), "emergency-")? diff --git a/crates/codestory-cli/src/local_refresh_status.rs b/crates/codestory-cli/src/local_refresh_status.rs index 845711b09..bceeb8c92 100644 --- a/crates/codestory-cli/src/local_refresh_status.rs +++ b/crates/codestory-cli/src/local_refresh_status.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use codestory_contracts::bounded_locks::{ self, DEFAULT_LOCK_WAIT, FileLockKind, LockDeadline, acquire_with_deadline, }; @@ -558,7 +558,13 @@ fn acquire_local_refresh_state_guard(cache_root: &Path) -> Result) -> String { } append_readiness_verdicts(&mut markdown, &output.readiness); append_index_summary_generation(&mut markdown, output); + for warning in &output.warnings { + let _ = writeln!(markdown, "warning: {warning}"); + } append_next_commands(&mut markdown, &output.next_commands); markdown } @@ -287,6 +290,7 @@ fn diagnostic_core_status_label(status: crate::args::DiagnosticCoreStatus) -> &' match status { crate::args::DiagnosticCoreStatus::Unavailable => "unavailable", crate::args::DiagnosticCoreStatus::UpgradeRequired => "upgrade_required", + crate::args::DiagnosticCoreStatus::NewerSchema => "newer_schema", } } @@ -2761,6 +2765,27 @@ pub(crate) fn render_doctor_markdown(output: &DoctorOutput) -> String { compact_doctor_check_message(check) ); } + if !output.stale_cached_cores.is_empty() { + let _ = writeln!(markdown, "stale_cached_cores:"); + for core in &output.stale_cached_cores { + match (core.project_root.as_ref(), core.next_action.as_ref()) { + (Some(root), Some(action)) => { + let _ = writeln!( + markdown, + "- `{}` schema={} required={} next: `{action}`", + root, core.found_schema, core.required_schema + ); + } + _ => { + let _ = writeln!( + markdown, + "- `{}` schema={} required={}", + core.cache_dir, core.found_schema, core.required_schema + ); + } + } + } + } let _ = writeln!(markdown, "environment:"); for item in &output.environment { let _ = writeln!( @@ -4308,6 +4333,7 @@ mod tests { retrieval_publication: None, operation_id: "public-1".to_string(), attempt: 1, + freshness: codestory_runtime::OperationFreshness::Fresh, }; emit_public_operation(OutputFormat::Markdown, operation(), Some(&markdown_path)) @@ -4337,6 +4363,7 @@ mod tests { retrieval_publication: None, operation_id: "public-2".to_string(), attempt: 1, + freshness: codestory_runtime::OperationFreshness::Fresh, }, Some(&graph_path), ) @@ -4479,6 +4506,7 @@ mod tests { readiness_lanes: std::collections::BTreeMap::new(), checks: Vec::new(), next_commands: Vec::new(), + stale_cached_cores: Vec::new(), environment: Vec::new(), } } @@ -6315,83 +6343,6 @@ legend: assert!(!markdown.contains("[edge-2]")); } - #[test] - fn trail_story_reports_side_effects_and_test_scope() { - let included = sample_trail_story(true); - assert!( - included - .side_effects - .iter() - .any(|item| item.contains("write_audit_log")), - "story should name likely side-effect calls: {included:#?}" - ); - assert!( - included - .test_scope - .iter() - .any(|item| item.contains("tests and benches included")), - "include-tests story should say tests are included: {included:#?}" - ); - assert!( - included - .test_scope - .iter() - .any(|item| item.contains("test_request_flow")), - "include-tests story should name rendered test-like nodes: {included:#?}" - ); - - let excluded = sample_trail_story(false); - assert!( - excluded - .test_scope - .iter() - .any(|item| item.contains("tests and benches excluded")), - "production-scope story should say tests are excluded: {excluded:#?}" - ); - } - - #[test] - fn trail_story_handles_single_node_without_edges() { - let story = TrailStoryDto { - summary: "Story trail around `A` found 1 node and 0 edges; mode=neighborhood direction=both tests=excluded utility_calls=hidden truncated=false.".to_string(), - entry_points: vec![ - "focus: A [function]".to_string(), - "no graph entry edges were returned for this focus".to_string(), - ], - core_flow: Vec::new(), - runtime_flow: Vec::new(), - data_flow: Vec::new(), - type_structure: Vec::new(), - utility_calls: Vec::new(), - side_effects: vec![ - "none detected from conservative edge-kind and target-name heuristics; inspect snippets for runtime effects".to_string(), - ], - uncertainty: vec!["no rendered trail edges to evaluate for certainty".to_string()], - test_scope: vec![ - "tests and benches excluded by default production-only scope; pass --include-tests to include them".to_string(), - "no test-like nodes are present in the rendered trail".to_string(), - ], - limits: vec![ - "trail not truncated; max_nodes=24 omitted_edge_count=0".to_string(), - "no edges were returned, so core flow is limited to the focus node".to_string(), - ], - }; - - assert!(story.core_flow.is_empty()); - assert!( - story - .entry_points - .iter() - .any(|item| item.contains("no graph entry edges")) - ); - assert!( - story - .limits - .iter() - .any(|item| item.contains("no edges were returned")) - ); - } - #[test] fn render_trail_dot_emits_graphviz_nodes_and_edges() { let context = TrailContextDto { diff --git a/crates/codestory-cli/src/readiness.rs b/crates/codestory-cli/src/readiness.rs index ead037ac2..2e4b48ed1 100644 --- a/crates/codestory-cli/src/readiness.rs +++ b/crates/codestory-cli/src/readiness.rs @@ -182,6 +182,30 @@ pub(crate) fn freshness_requires_refresh(freshness: &IndexFreshnessDto) -> bool } } +/// User-facing guidance when the freshness scan stopped at the bounded +/// inventory cap. A bounded scan is a scale signal, not a defect: the +/// publication stays usable, but drift detection did not cover the whole +/// inventory. Every surface that reports index state uses this one message so +/// `index` and `doctor` cannot drift apart. +pub(crate) fn bounded_inventory_freshness_warning(freshness: &IndexFreshnessDto) -> Option { + if freshness.status != IndexFreshnessStatusDto::NotChecked + || freshness.not_checked_cause != Some(IndexFreshnessNotCheckedCauseDto::BoundedInventory) + { + return None; + } + let detail = freshness + .reason + .as_deref() + .map(|reason| format!(" ({reason})")) + .unwrap_or_default(); + Some(format!( + "The indexed file inventory exceeds the supported freshness-scan \ + envelope{detail}. The index remains usable, but freshness drift is \ + not checked beyond that bound; narrow the indexed scope or accept \ + bounded freshness checks." + )) +} + pub(crate) fn status_label(status: ReadinessStatusDto) -> &'static str { match status { ReadinessStatusDto::Ready => "ready", @@ -1220,4 +1244,25 @@ mod tests { ); } } + + #[test] + fn bounded_inventory_warning_fires_only_on_the_scale_cap_cause() { + let mut observed = freshness(IndexFreshnessStatusDto::NotChecked); + observed.not_checked_cause = Some(IndexFreshnessNotCheckedCauseDto::BoundedInventory); + observed.reason = Some( + "indexed file inventory exceeds bounded freshness cap (30000 > 25000)".to_string(), + ); + + let warning = bounded_inventory_freshness_warning(&observed) + .expect("a bounded inventory must carry scale-envelope guidance"); + assert!(warning.contains("30000 > 25000"), "{warning}"); + assert!(warning.contains("remains usable"), "{warning}"); + + observed.not_checked_cause = Some(IndexFreshnessNotCheckedCauseDto::InventoryUnavailable); + assert_eq!(bounded_inventory_freshness_warning(&observed), None); + + observed.status = IndexFreshnessStatusDto::Stale; + observed.not_checked_cause = Some(IndexFreshnessNotCheckedCauseDto::BoundedInventory); + assert_eq!(bounded_inventory_freshness_warning(&observed), None); + } } diff --git a/crates/codestory-cli/src/retrieval.rs b/crates/codestory-cli/src/retrieval.rs index 964377820..daba07c9b 100644 --- a/crates/codestory-cli/src/retrieval.rs +++ b/crates/codestory-cli/src/retrieval.rs @@ -415,10 +415,13 @@ fn retrieval_index_should_retry_full_refresh( } fn map_retrieval_finalize_error(error: anyhow::Error) -> anyhow::Error { - match codestory_runtime::insufficient_space_api_error(&error) { - Some(refusal) => map_api_error(refusal), - None => error, + if let Some(refusal) = codestory_runtime::insufficient_space_api_error(&error) { + return map_api_error(refusal); } + if let Some(refusal) = codestory_runtime::peer_writer_api_error(&error) { + return map_api_error(refusal); + } + error } fn error_chain_contains(error: &anyhow::Error, needle: &str) -> bool { @@ -1102,6 +1105,19 @@ mod tests { output_bytes: None, attested_bytes: None, }, + // Single-cause rejection: an allowlisted component carrying an + // unallowlisted mode must be refused on the mode alone. + FinalizeComponentWork { + component: "vectors".into(), + mode: "/private/mode".into(), + retained: None, + inserted: None, + removed: None, + reordered: None, + predecessor_bytes: None, + output_bytes: None, + attested_bytes: None, + }, ]; let safe_phases = safe_retrieval_phase_timings(&phases); diff --git a/crates/codestory-cli/src/runtime.rs b/crates/codestory-cli/src/runtime.rs index cdb2fae23..4b4a34231 100644 --- a/crates/codestory-cli/src/runtime.rs +++ b/crates/codestory-cli/src/runtime.rs @@ -628,25 +628,49 @@ fn publication_env_nonempty(name: &str) -> Option { } /// Build the canonical publication/contract stamp shared by CLI JSON, HTTP, and stdio. +/// +/// `freshness` is the runtime's own admission decision for the operation that +/// produced the response, never a status-cache guess: `Historical` exactly +/// when the runtime admitted the read from a retained publication during a +/// refresh. `served_from` derives from it. pub(crate) fn codestory_publication_meta( core_publication: Option, retrieval_publication: Option, operation_id: Option<&str>, attempt: Option, - refreshing: bool, + freshness: Option, ) -> serde_json::Value { let served_from = if core_publication.is_none() && retrieval_publication.is_none() { "contract_only" - } else if refreshing { + } else if matches!( + freshness, + Some(codestory_runtime::OperationFreshness::Historical(_)) + ) { "last_complete_publication" } else { "complete_publication" }; + let served_generation = core_publication + .as_ref() + .and_then(|publication| publication.pointer("/generation_id")) + .cloned(); + let freshness = match freshness { + Some(codestory_runtime::OperationFreshness::Historical(reason)) => serde_json::json!({ + "state": "historical", + "reason": reason, + "served_generation": served_generation, + }), + _ => serde_json::json!({ + "state": "fresh", + "served_generation": served_generation, + }), + }; serde_json::json!({ "schema_version": CODESTORY_PUBLICATION_META_SCHEMA_VERSION, "minimum_compatible_schema_version": CODESTORY_PUBLICATION_META_MINIMUM_COMPATIBLE_SCHEMA_VERSION, "served_from": served_from, + "freshness": freshness, "publication": core_publication, "core_publication": core_publication, "retrieval_publication": retrieval_publication, @@ -697,7 +721,7 @@ pub(crate) fn public_operation_json_value( retrieval_publication, Some(&operation.operation_id), Some(operation.attempt), - false, + Some(operation.freshness), ), ); Ok(value) @@ -713,6 +737,7 @@ pub(crate) fn map_public_operation( retrieval_publication: operation.retrieval_publication, operation_id: operation.operation_id, attempt: operation.attempt, + freshness: operation.freshness, } } @@ -1118,7 +1143,8 @@ fn is_cache_busy_text(text: &str) -> bool { } fn api_error_is_schema_too_new(error: &ApiError) -> bool { - is_schema_too_new_text(&format!("{} {}", error.code, error.message)) + error.code == "core_schema_too_new" + || is_schema_too_new_text(&format!("{} {}", error.code, error.message)) } fn is_schema_too_new_text(text: &str) -> bool { @@ -1190,6 +1216,7 @@ mod tests { embedding_retry: None, disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, }, ) } @@ -1381,6 +1408,7 @@ mod tests { retrieval_publication: None, operation_id: "public-7".to_string(), attempt: 2, + freshness: codestory_runtime::OperationFreshness::Fresh, }; let response = serde_json::json!({ "result": "ok", @@ -1406,7 +1434,7 @@ mod tests { // must see the bump rather than a self-referential comparison. assert_eq!( value.pointer("/_meta/codestory_publication/schema_version"), - Some(&serde_json::json!(3)) + Some(&serde_json::json!(4)) ); assert_eq!( value.pointer("/_meta/codestory_publication/minimum_compatible_schema_version"), @@ -1440,6 +1468,19 @@ mod tests { } Self { values } } + + /// Set `name` to `value` for the guard's lifetime, restoring the prior + /// value (or absence) on drop. Callers must hold the config env test + /// lock so no concurrent capture observes the forced value. + fn set(name: &'static str, value: &str) -> Self { + let prior = env::var_os(name); + unsafe { + env::set_var(name, value); + } + Self { + values: vec![(name, prior)], + } + } } impl Drop for EnvSnapshot { @@ -1835,9 +1876,7 @@ mod tests { let _env_lock = crate::config::config_env_test_lock(); let _managed_env = EnvSnapshot::clear(MANAGED_ENV_VARS); let _home_env = EnvSnapshot::clear(HOME_ENV_VARS); - unsafe { - env::set_var("CODESTORY_TEST_EMBED_ALLOW_CPU", "1"); - } + let _embed_env = EnvSnapshot::set("CODESTORY_TEST_EMBED_ALLOW_CPU", "1"); let temp = tempdir().expect("temp dir"); let project = temp.path().join("project"); let cache = temp.path().join("cache"); @@ -1885,7 +1924,7 @@ mod tests { .value; let current_generation = publisher .project - .complete_index_publication_at(&publisher.storage_path) + .complete_index_publication_at(&publisher.project_root, &publisher.storage_path) .expect("read current publication") .expect("current publication") .generation; @@ -1899,9 +1938,7 @@ mod tests { let _env_lock = crate::config::config_env_test_lock(); let _managed_env = EnvSnapshot::clear(MANAGED_ENV_VARS); let _home_env = EnvSnapshot::clear(HOME_ENV_VARS); - unsafe { - env::set_var("CODESTORY_TEST_EMBED_ALLOW_CPU", "1"); - } + let _embed_env = EnvSnapshot::set("CODESTORY_TEST_EMBED_ALLOW_CPU", "1"); let temp = tempdir().expect("temp dir"); let project = temp.path().join("project"); let cache = temp.path().join("cache"); @@ -1950,7 +1987,7 @@ mod tests { .expect("observational response should retry one core replacement"); let current_generation = publisher .project - .complete_index_publication_at(&publisher.storage_path) + .complete_index_publication_at(&publisher.project_root, &publisher.storage_path) .expect("read current publication") .expect("current publication") .generation; @@ -2006,9 +2043,7 @@ mod tests { let _env_lock = crate::config::config_env_test_lock(); let _managed_env = EnvSnapshot::clear(MANAGED_ENV_VARS); let _home_env = EnvSnapshot::clear(HOME_ENV_VARS); - unsafe { - env::set_var("CODESTORY_TEST_EMBED_ALLOW_CPU", "1"); - } + let _embed_env = EnvSnapshot::set("CODESTORY_TEST_EMBED_ALLOW_CPU", "1"); let temp = tempdir().expect("temp dir"); let project = temp.path().join("project"); let cache = temp.path().join("cache"); @@ -2046,7 +2081,7 @@ mod tests { ); let generation_a = reader .project - .complete_index_publication_at(&reader.storage_path) + .complete_index_publication_at(&reader.project_root, &reader.storage_path) .expect("read generation A") .expect("generation A exists"); diff --git a/crates/codestory-cli/src/stdio_arguments.rs b/crates/codestory-cli/src/stdio_arguments.rs index c4b7e54fe..78f7c2dbb 100644 --- a/crates/codestory-cli/src/stdio_arguments.rs +++ b/crates/codestory-cli/src/stdio_arguments.rs @@ -599,6 +599,44 @@ mod tests { } } + /// Collect every keyword a published schema uses. `value` is already in + /// schema position, so each member key is a keyword regardless of whether + /// the object carries a recognized keyword; heuristic detection is only a + /// fallback for literal-valued members (a `const`/`default` payload, for + /// instance, can itself be a JSON object). + fn collect_schema_keywords(schema: &Value, found: &mut std::collections::BTreeSet) { + let Value::Object(members) = schema else { + return; + }; + for (key, value) in members { + found.insert(key.clone()); + match key.as_str() { + "properties" => { + if let Some(properties) = value.as_object() { + for property in properties.values() { + collect_schema_keywords(property, found); + } + } + } + "items" | "additionalProperties" | "not" => { + collect_schema_keywords(value, found); + } + "oneOf" | "anyOf" | "allOf" => { + if let Some(variants) = value.as_array() { + for variant in variants { + collect_schema_keywords(variant, found); + } + } + } + _ => collect_keywords(value, found), + } + } + } + + /// Heuristic schema detection for values that are not already in schema + /// position. A schema-shaped literal keeps coverage it had before; a + /// keyword-only-unsupported object that reaches this path stays uncollected + /// because nothing positions it as a schema. fn collect_keywords(schema: &Value, found: &mut std::collections::BTreeSet) { match schema { Value::Object(members) => { @@ -612,18 +650,11 @@ mod tests { || members.contains_key("items") || members.contains_key("enum") || members.contains_key("const"); - for (key, value) in members { - if is_schema { - found.insert(key.clone()); - } - if is_schema && key == "properties" { - if let Some(properties) = value.as_object() { - for property in properties.values() { - collect_keywords(property, found); - } - } - continue; - } + if is_schema { + collect_schema_keywords(schema, found); + return; + } + for value in members.values() { collect_keywords(value, found); } } @@ -796,6 +827,15 @@ mod tests { codes_from_output(&schema, json!({"kind":"wrong","value":"forbidden"})), vec!["invalid_const_value", "forbidden_combination"] ); + // Each of these satisfies no `anyOf` variant; a validator that dropped + // union enforcement would admit all three. + for value in [json!(false), json!(""), json!(0)] { + let codes = codes_from_output(&schema, json!({"kind":"tagged","value":value})); + assert!( + codes.contains(&"unsatisfied_any_of"), + "{value} satisfies no anyOf branch: {codes:?}" + ); + } } #[test] diff --git a/crates/codestory-cli/src/stdio_catalog.rs b/crates/codestory-cli/src/stdio_catalog.rs index 99cee7ce0..daa3aef9d 100644 --- a/crates/codestory-cli/src/stdio_catalog.rs +++ b/crates/codestory-cli/src/stdio_catalog.rs @@ -2995,9 +2995,29 @@ mod tests { observed_size: Some(2), byte_cap: Some(1), }]; + answer.freshness = Some(codestory_contracts::api::IndexFreshnessDto { + status: codestory_contracts::api::IndexFreshnessStatusDto::Stale, + changed_file_count: 2, + new_file_count: 1, + removed_file_count: 0, + checked_file_count: 7, + indexed_file_count: 4, + duration_ms: 3, + reason: None, + not_checked_cause: None, + samples: Vec::new(), + }); let packet = crate::output::context_packet_json(&answer); let emitted = packet.as_object().expect("packet object"); + // The optional fields must actually reach the wire: an empty fixture + // proves nothing about whether an emitted key is declared. + for field in ["freshness", "source_coverage"] { + assert!( + emitted.contains_key(field), + "populated {field} must reach the packet wire so its schema declaration is audited: {emitted:?}" + ); + } let undeclared = emitted .keys() .filter(|key| !declared.contains(&key.as_str())) diff --git a/crates/codestory-cli/src/stdio_transport.rs b/crates/codestory-cli/src/stdio_transport.rs index a2dd9fc79..41467ec2e 100644 --- a/crates/codestory-cli/src/stdio_transport.rs +++ b/crates/codestory-cli/src/stdio_transport.rs @@ -827,6 +827,9 @@ fn stdio_activation_stage_message(stage: codestory_runtime::ActivationStage) -> use codestory_runtime::ActivationStage; match stage { ActivationStage::Discovery => "CodeStory is checking project files", + ActivationStage::WaitingForPeerWriter => { + "CodeStory is waiting for another session to finish indexing" + } ActivationStage::CoreFreshness => "CodeStory is updating the code index", ActivationStage::SearchPreparation => "CodeStory is preparing search", ActivationStage::DensePreparation => "CodeStory is preparing semantic search", @@ -1718,7 +1721,7 @@ fn handle_stdio_request( ); let mut result = session.protocol_v3.initialize_result(); result["_meta"]["codestory_publication"] = - crate::runtime::codestory_publication_meta(None, None, None, None, false); + crate::runtime::codestory_publication_meta(None, None, None, None, None); return Some(stdio_jsonrpc_success(id, result)); } "tools/list" => serde_json::json!({ @@ -2084,7 +2087,7 @@ fn handle_stdio_request( } if runtime.activation.snapshot().is_some_and(|snapshot| { snapshot.state == codestory_runtime::ActivationState::Ready - && !snapshot.allows_operation(name) + && !snapshot.allows_operation(public_operation) }) { return Some(stdio_jsonrpc_success( id, @@ -2099,9 +2102,21 @@ fn handle_stdio_request( codestory_runtime::observe_packet_entry_phase( codestory_runtime::PacketEntryObservationPhase::ActivationStarted, ); - let goal = if observes_complete_core { + // PinnedObserver and source-backed reads need only the + // complete core, never the retrieval sidecars. A CoreOnly run + // stops at the core publication; allows_operation still + // requires a fresh Ready core for them, so Retained no longer + // admits. + let goal = if matches!(public_operation, "exact_search" | "source_snippet") + || codestory_runtime::operation_read_class(public_operation) + == codestory_runtime::OperationReadClass::PinnedObserver + { codestory_runtime::ActivationGoal::CoreOnly } else { + // Retrieval-class operations (`packet`, `search`, + // `context`, `drill`, `resolution`, `graph_assisted`) need + // managed retrieval preparation, and allows_operation + // requires `broad_search` Ready for them. codestory_runtime::ActivationGoal::Full }; let first_slice = preparation_deadline @@ -2133,11 +2148,12 @@ fn handle_stdio_request( } else { runtime .activation - .activate_project_with_foreground_budget( + .activate_project_with_foreground_budget_and_goal( &runtime.project_root, &runtime.storage_path, Arc::clone(cancelled), first_slice, + goal, ) .map(|_| ()) }; @@ -2163,7 +2179,11 @@ fn handle_stdio_request( )); break; } - if !observes_complete_core && snapshot.allows_operation(name) { + // Only a graph-only read may leave the wait early on a + // Retained snapshot: its answer is labelled historical. + // SourceBacked and PinnedObserver keep waiting because + // allows_operation requires a fresh Ready core for them. + if !observes_complete_core && snapshot.allows_operation(public_operation) { break; } waiting_operation_id = Some(snapshot.operation_id.clone()); @@ -2208,7 +2228,7 @@ fn handle_stdio_request( let allowed = !observes_complete_core && operation .as_ref() - .is_some_and(|snapshot| snapshot.allows_operation(name)); + .is_some_and(|snapshot| snapshot.allows_operation(public_operation)); if matches!(error.code.as_str(), "cancelled" | "publication_changed") || !allowed { @@ -2471,6 +2491,7 @@ fn handle_stdio_request( }), Some(&operation.operation_id), Some(operation.attempt), + Some(operation.freshness), )); execution } @@ -2490,6 +2511,7 @@ fn handle_stdio_request( stdio_response_retrieval_publication(&execution.response), None, None, + None, )); execution } @@ -3243,15 +3265,33 @@ fn stdio_search_repo_text_mode(request: &serde_json::Value) -> SearchRepoTextMod } } +fn stdio_tool_read_class( + name: &str, + request: &serde_json::Value, +) -> codestory_runtime::OperationReadClass { + codestory_runtime::operation_read_class(stdio_public_operation_name(name, request)) +} + fn stdio_tool_observes_complete_core(name: &str, request: &serde_json::Value) -> bool { - name == "affected" - || crate::prove_call_path::is_proof_tool_name(name) - || (name == "search" && stdio_search_repo_text_mode(request) == SearchRepoTextMode::Off) + stdio_tool_read_class(name, request) == codestory_runtime::OperationReadClass::PinnedObserver } fn stdio_public_operation_name<'a>(name: &'a str, request: &serde_json::Value) -> &'a str { if name == "search" { codestory_runtime::search_operation_name(stdio_search_repo_text_mode(request)) + } else if name == "snippet" { + // An id-selected snippet serves verified source bytes, so it is a + // source-backed read that waits for a fresh complete core. A snippet + // requested by query text is a graph-assisted selection instead. + if request + .pointer("/params/arguments/query") + .and_then(serde_json::Value::as_str) + .is_some_and(|query| !query.trim().is_empty()) + { + "graph_assisted" + } else { + "source_snippet" + } } else if matches!( name, "symbol" @@ -3264,7 +3304,6 @@ fn stdio_public_operation_name<'a>(name: &'a str, request: &serde_json::Value) - | "query_subgraph" | "definition" | "references" - | "snippet" ) { if request .pointer("/params/arguments/query") @@ -3275,6 +3314,11 @@ fn stdio_public_operation_name<'a>(name: &'a str, request: &serde_json::Value) - } else { "graph" } + } else if crate::prove_call_path::is_proof_tool_name(name) { + // Proof tools keep their own observed-operation internals; the + // admission loop only needs the pinned-observer class, so they share + // one runtime-visible operation name. + codestory_runtime::PROOF_DOMAIN } else { name } @@ -3286,7 +3330,11 @@ fn stdio_served_publication_meta( retrieval_publication: Option<&serde_json::Value>, operation_id: Option<&str>, attempt: Option, + freshness: Option, ) -> serde_json::Value { + // `freshness`/`served_from` come from the runtime's admission decision on + // the PublicOperation, not this status cache. The cache still feeds the + // `refresh` block, which is diagnostic detail about the live refresh only. let status = state.status_cache.as_ref().map(|cached| &cached.value); let refreshing = status .and_then(|status| status.pointer("/local_refresh/state")) @@ -3299,7 +3347,7 @@ fn stdio_served_publication_meta( retrieval_publication.cloned(), operation_id, attempt, - refreshing, + freshness, ); if refreshing { meta["refresh"] = serde_json::json!({ @@ -4055,7 +4103,7 @@ fn stdio_initialize_result_json(request: &serde_json::Value) -> serde_json::Valu None, None, None, - false, + None, ), "codestory_protocol": negotiation, } @@ -4388,7 +4436,9 @@ fn handle_stdio_tool_call( match name { "status" => read_stdio_status_resource_cached(runtime, state) .map(|status| serde_json::json!({"result": compact_stdio_status(runtime, &status)})) - .unwrap_or_else(|error| serde_json::json!({"error": error.to_string()})), + .unwrap_or_else( + |error| serde_json::json!({"error": stdio_typed_error_value(runtime, &error)}), + ), "packet" => handle_stdio_packet(runtime, state, request), "search" => handle_stdio_search(runtime, state, request, query), "ground" => handle_stdio_ground(runtime, request), @@ -6057,8 +6107,8 @@ fn stdio_target_selection(request: &serde_json::Value) -> args::TargetSelection /// boundary without a typed cause; `architecture_contracts` forbids adding new /// stringified paths beside it. fn stdio_typed_error_value(runtime: &RuntimeContext, error: &anyhow::Error) -> serde_json::Value { - if let Some(ambiguous) = error.downcast_ref::() { - return serde_json::to_value(build_ambiguous_target_error_output( + let mut value = if let Some(ambiguous) = error.downcast_ref::() { + serde_json::to_value(build_ambiguous_target_error_output( &runtime.project_root, ambiguous, )) @@ -6069,14 +6119,22 @@ fn stdio_typed_error_value(runtime: &RuntimeContext, error: &anyhow::Error) -> s "ambiguous_target", ambiguous.to_string(), )) - }); - } - if let Some(api_error) = crate::runtime::api_error_in_chain(error) { - return stdio_api_error_value(api_error.clone()); + }) + } else if let Some(api_error) = crate::runtime::api_error_in_chain(error) { + stdio_api_error_value(api_error.clone()) + } else { + stdio_api_error_value(codestory_contracts::api::ApiError::internal( + error.to_string(), + )) + }; + // The legacy `{"error": string}` shape surfaced the rendered message + // (code prefix plus recovery commands) as the JSON-RPC message text. Keep + // that text so message-only readers see the same words; the typed code and + // details now ride alongside it in the object. + if let Some(object) = value.as_object_mut() { + object.insert("message".to_string(), serde_json::json!(error.to_string())); } - stdio_api_error_value(codestory_contracts::api::ApiError::internal( - error.to_string(), - )) + value } fn read_stdio_resource( @@ -6097,7 +6155,9 @@ fn read_stdio_resource( }; result .map(|value| serde_json::json!({"result": {"contents": [{"uri": uri, "mimeType": "application/json", "text": value.to_string()}]}})) - .unwrap_or_else(|error| serde_json::json!({"error": error.to_string()})) + .unwrap_or_else(|error| { + serde_json::json!({"error": stdio_typed_error_value(runtime, &error)}) + }) } fn read_stdio_static_resource(resource: &ParsedStdioResource) -> serde_json::Value { @@ -6192,7 +6252,7 @@ fn read_stdio_status_resource_base_cached( for attempt in 1..=STDIO_STATUS_PUBLICATION_ATTEMPTS { let publication_before = runtime .project - .complete_index_publication_at(&runtime.storage_path) + .complete_index_publication_at(&runtime.project_root, &runtime.storage_path) .map_err(map_api_error)?; let mut value = read_stdio_status_resource_uncached(runtime, state)?; completed_refresh = completed_refresh.or_else(|| { @@ -6205,7 +6265,7 @@ fn read_stdio_status_resource_base_cached( }); let publication_after = runtime .project - .complete_index_publication_at(&runtime.storage_path) + .complete_index_publication_at(&runtime.project_root, &runtime.storage_path) .map_err(map_api_error)?; let cache_key = stdio_status_cache_key_with_publication( runtime, @@ -6418,7 +6478,7 @@ fn read_stdio_status_resource_uncached( fn stdio_complete_publication_fingerprint(runtime: &RuntimeContext) -> String { match runtime .project - .complete_index_publication_at(&runtime.storage_path) + .complete_index_publication_at(&runtime.project_root, &runtime.storage_path) { Ok(publication) => stdio_publication_fingerprint(publication.as_ref()), Err(error) => format!("error:{error:?}"), @@ -8126,7 +8186,7 @@ fn read_stdio_template_resource( .map_err(map_api_error), StdioResource::Snippet(node_id) => runtime .browser - .snippet_context(node_id.clone(), 4) + .snippet_context_observational(node_id.clone(), 4) .map(|value| serde_json::json!(value)) .map_err(map_api_error), StdioResource::Trail(node_id) => runtime @@ -8494,9 +8554,20 @@ mod tests { let ordinary = json!({ "params": {"arguments": {"query": "RenamedAnchor", "repo_text": "auto"}} }); - assert!(stdio_tool_observes_complete_core("search", &exact)); + assert!(!stdio_tool_observes_complete_core("search", &exact)); assert!(!stdio_tool_observes_complete_core("search", &ordinary)); assert!(!stdio_tool_observes_complete_core("search", &json!({}))); + // repo_text=off is a source-backed read: it waits for a fresh + // complete core through a CoreOnly activation instead of binding a + // retained one. + assert_eq!( + stdio_tool_read_class("search", &exact), + codestory_runtime::OperationReadClass::SourceBacked + ); + assert_eq!( + stdio_tool_read_class("search", &ordinary), + codestory_runtime::OperationReadClass::Retrieval + ); assert_eq!( stdio_public_operation_name("search", &exact), "exact_search" @@ -8598,6 +8669,31 @@ mod tests { &valid.uri, )); + // Same-binding controls on a populated session: a live capability + // reads in its owning session and is refused in a foreign one. `valid` + // was already evicted by the churn above, so register a live grant. + let live = session + .diagnostics_v3 + .lock() + .unwrap() + .register_at( + diagnostic_binding(Uuid::new_v4().to_string()), + br#"{"live":true}"#.to_vec(), + Instant::now(), + ) + .unwrap(); + let own = diagnostic_resource_read(&mut session, "own-live", &live.uri); + assert_eq!( + own.pointer("/result/contents/0/text"), + Some(&json!("{\"live\":true}")), + "the owning session must read its live capability: {own}" + ); + unavailable(&diagnostic_resource_read( + &mut other_session, + "foreign-live", + &live.uri, + )); + let malformed = diagnostic_resource_read( &mut session, "malformed", @@ -9352,7 +9448,15 @@ mod tests { "graph_assisted", "{tool} query resolution must keep one retrieval pin through response assembly" ); - assert_eq!(stdio_public_operation_name(tool, &id), "graph"); + assert_eq!( + stdio_public_operation_name(tool, &id), + if tool == "snippet" { + "source_snippet" + } else { + "graph" + }, + "{tool} id selection keeps its read-class operation name" + ); } assert_eq!( stdio_public_operation_name( @@ -9526,8 +9630,19 @@ mod tests { }), }; - queued.admit(stdio_cancellation_line(r#""request-1""#), Some(&active)); + // A wrong-target cancellation on a fresh active request must not set + // its flags: prove isolation before the own-target cancel runs. queued.admit(stdio_cancellation_line("\"other\""), Some(&active)); + assert!( + !cancelled.load(Ordering::Acquire), + "a cancellation for another request must not flag this one" + ); + assert!( + !client_cancelled.load(Ordering::Acquire), + "a cancellation for another request must not flag this one" + ); + + queued.admit(stdio_cancellation_line(r#""request-1""#), Some(&active)); assert!( cancelled.load(Ordering::Acquire), @@ -10218,12 +10333,24 @@ mod tests { #[tokio::test] async fn stdio_serve_loop_answers_the_running_request_when_termination_arrives() { static ENTERED: AtomicBool = AtomicBool::new(false); + static RELEASE: AtomicBool = AtomicBool::new(false); + static OBSERVED_TERMINATION: AtomicBool = AtomicBool::new(false); fn handler( _: &mut StdioServerSession, line: &str, - _: &Arc, + cancelled: &Arc, ) -> Option { ENTERED.store(true, Ordering::Release); + // Hold the request until termination actually signals it, so the + // running-request overlap this test claims is real. RELEASE is the + // cleanup escape if the serve loop abandons the worker instead. + while !RELEASE.load(Ordering::Acquire) { + if cancelled.load(Ordering::Acquire) { + OBSERVED_TERMINATION.store(true, Ordering::Release); + break; + } + std::thread::sleep(Duration::from_millis(1)); + } Some(stdio_jsonrpc_success( stdio_message_id(line).unwrap_or_default(), json!({"served": true}), @@ -10253,7 +10380,15 @@ mod tests { .await .expect("termination drains the running request"); + // Free the worker before asserting so a failure cannot strand it. + let observed_termination = OBSERVED_TERMINATION.load(Ordering::Acquire); + RELEASE.store(true, Ordering::Release); + assert_eq!(outcome, StdioServeOutcome::Terminated); + assert!( + observed_termination, + "the request must still be running when termination signals it" + ); let responses = stdio_written_responses(&output); assert_eq!(responses.len(), 1, "{responses:?}"); @@ -10357,6 +10492,11 @@ mod tests { .clone(); let worker_gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); activation.set_worker_start_gate_for_test(Some(Arc::clone(&worker_gate))); + let _cleanup = PreparationTestCleanup { + activation: activation.clone(), + gate: Arc::clone(&worker_gate), + restore_snapshot: None, + }; let (mut client_input, server_input) = tokio::io::duplex(4096); let (server_output, client_output) = tokio::io::duplex(4096); @@ -10488,6 +10628,11 @@ mod tests { .clone(); let worker_gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); activation.set_worker_start_gate_for_test(Some(Arc::clone(&worker_gate))); + let _cleanup = PreparationTestCleanup { + activation: activation.clone(), + gate: Arc::clone(&worker_gate), + restore_snapshot: None, + }; let (mut client_input, server_input) = tokio::io::duplex(4096); let (server_output, client_output) = tokio::io::duplex(4096); @@ -10858,6 +11003,11 @@ mod tests { .clone(); let worker_gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); activation.set_worker_start_gate_for_test(Some(Arc::clone(&worker_gate))); + let _cleanup = PreparationTestCleanup { + activation: activation.clone(), + gate: Arc::clone(&worker_gate), + restore_snapshot: None, + }; let started = Instant::now(); let response = handle_stdio_message( &mut session, @@ -11137,6 +11287,11 @@ mod tests { .clone(); let worker_gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); activation.set_worker_start_gate_for_test(Some(Arc::clone(&worker_gate))); + let _cleanup = PreparationTestCleanup { + activation: activation.clone(), + gate: Arc::clone(&worker_gate), + restore_snapshot: None, + }; let (mut client_input, server_input) = tokio::io::duplex(4096); let (server_output, client_output) = tokio::io::duplex(4096); let serving = tokio::spawn(serve_stdio_requests( @@ -11325,6 +11480,11 @@ mod tests { .clone(); let worker_gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); activation.set_worker_start_gate_for_test(Some(Arc::clone(&worker_gate))); + let _cleanup = PreparationTestCleanup { + activation: activation.clone(), + gate: Arc::clone(&worker_gate), + restore_snapshot: None, + }; let (mut client_input, server_input) = tokio::io::duplex(4096); let (server_output, client_output) = tokio::io::duplex(4096); let serving = tokio::spawn(serve_stdio_requests( @@ -11441,6 +11601,57 @@ mod tests { ); } + struct EnvVarRestore(&'static str, Option); + + impl EnvVarRestore { + fn capture(name: &'static str) -> Self { + Self(name, std::env::var_os(name)) + } + } + + impl Drop for EnvVarRestore { + fn drop(&mut self) { + unsafe { + match self.1.take() { + Some(value) => std::env::set_var(self.0, value), + None => std::env::remove_var(self.0), + } + } + } + } + + #[test] + fn multi_project_stdio_ignores_mutable_active_workspace_state() { + let _env_lock = crate::config::config_env_test_lock(); + let _multi = EnvVarRestore::capture("CODESTORY_PLUGIN_MULTI_PROJECT"); + let _state = EnvVarRestore::capture("CODESTORY_PLUGIN_ACTIVE_STATE_PATH"); + let (_project, _cache, runtime) = stdio_inspect_only_runtime(); + let active = tempfile::tempdir().expect("active workspace"); + let state_file = tempfile::NamedTempFile::new().expect("active state"); + std::fs::write( + state_file.path(), + serde_json::json!({ "cwd": active.path() }).to_string(), + ) + .expect("write active state"); + unsafe { + std::env::set_var("CODESTORY_PLUGIN_ACTIVE_STATE_PATH", state_file.path()); + std::env::set_var("CODESTORY_PLUGIN_MULTI_PROJECT", "1"); + } + + // Multi-project mode selects its project per request and must not even + // read the ambient active-workspace pointer. + assert!(stdio_workspace_mismatch(&runtime).is_none()); + + // Without the flag the same ambient state is a real mismatch — the + // ignored-state branch above is the discriminating bit, not the file. + unsafe { + std::env::remove_var("CODESTORY_PLUGIN_MULTI_PROJECT"); + } + let mismatch = stdio_workspace_mismatch(&runtime).expect("mismatch without multi-project"); + assert_eq!(mismatch.served_root, runtime.project_root); + assert_eq!(mismatch.active_root, active.path().to_path_buf()); + } + #[test] fn stdio_recommended_next_calls_labels_restart_boundary_as_host_action() { let restart = @@ -12765,6 +12976,7 @@ version = "0.11.20" Some(&retrieval), Some("public-2"), Some(1), + None, ); let response = stdio_jsonrpc_tool_call_from_legacy(json!(1), bound, meta, "search"); @@ -12782,7 +12994,7 @@ version = "0.11.20" ); assert_eq!( response.pointer("/result/_meta/codestory_publication/schema_version"), - Some(&json!(3)) + Some(&json!(4)) ); assert_eq!( response.pointer("/result/_meta/codestory_publication/contract_runtime/cli_version"), @@ -12805,6 +13017,7 @@ version = "0.11.20" None, Some("public-1"), Some(1), + None, ); let response = stdio_jsonrpc_tool_call_from_legacy( json!(1), @@ -12819,6 +13032,7 @@ version = "0.11.20" retrieval_publication: None, operation_id: "public-1".to_string(), attempt: 1, + freshness: codestory_runtime::OperationFreshness::Fresh, }, &payload, ) @@ -12829,7 +13043,7 @@ version = "0.11.20" // an older schema. assert_eq!( response.pointer("/result/_meta/codestory_publication/schema_version"), - Some(&json!(3)) + Some(&json!(4)) ); assert_eq!( response @@ -12935,7 +13149,7 @@ version = "0.11.20" // The launcher reads this stamp out of the frame it suppresses, so it // must be the same contract-only stamp every other adapter publishes. let stamp = &agreed["_meta"]["codestory_publication"]; - assert_eq!(stamp["schema_version"], json!(3)); + assert_eq!(stamp["schema_version"], json!(4)); assert_eq!(stamp["minimum_compatible_schema_version"], json!(3)); assert_eq!(stamp["served_from"], json!("contract_only")); assert_eq!( @@ -12944,7 +13158,7 @@ version = "0.11.20" ); assert_eq!( stamp, - &crate::runtime::codestory_publication_meta(None, None, None, None, false), + &crate::runtime::codestory_publication_meta(None, None, None, None, None), "initialize must not invent a second stamp shape" ); } @@ -14375,7 +14589,7 @@ version = "0.11.20" ) .expect("tool response"); let content = &response["result"]["structuredContent"]; - if content.get("code") == Some(&json!("codestory_preparing")) { + if content.get("kind").and_then(serde_json::Value::as_str) == Some("preparing") { assert!( Instant::now() < deadline, "broad call did not become ready: {content}" @@ -14580,7 +14794,7 @@ version = "0.11.20" ) .expect("packet response"); let content = &response["result"]["structuredContent"]; - if content.get("code") == Some(&json!("codestory_preparing")) { + if content.get("kind").and_then(serde_json::Value::as_str) == Some("preparing") { assert!( Instant::now() < deadline, "packet fixture did not become ready: {content}" @@ -14590,6 +14804,11 @@ version = "0.11.20" )); continue; } + if response.pointer("/result/isError") == Some(&json!(true)) { + panic!( + "packet fixture must converge instead of becoming unavailable: {response}" + ); + } return response; } panic!("packet fixture did not converge within the bounded retry count") @@ -15564,4 +15783,694 @@ version = "0.11.20" "status materialized or resized the SHM wal-index" ); } + /// Parks the indexing worker while `is_indexing` is set and the writer + /// lock is held, so a read runs against a refresh that is genuinely in + /// flight instead of one parked before it started. + struct MidIndexingHold { + gate: Arc<(std::sync::Mutex, std::sync::Condvar)>, + } + + impl MidIndexingHold { + fn arm(storage_path: &Path) -> Self { + let gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + let expected = storage_path.to_path_buf(); + let hook_gate = Arc::clone(&gate); + codestory_runtime::set_mid_indexing_test_hook(Some(Arc::new(move |path: &Path| { + if path == expected.as_path() { + let (released, changed) = &*hook_gate; + let mut released = released.lock().expect("mid-index gate"); + while !*released { + released = changed.wait(released).expect("mid-index gate"); + } + } + }))); + Self { gate } + } + } + + impl Drop for MidIndexingHold { + fn drop(&mut self) { + let (released, changed) = &*self.gate; + *released.lock().expect("mid-index gate") = true; + changed.notify_all(); + codestory_runtime::set_mid_indexing_test_hook(None); + } + } + + struct LiveStdioFixture { + _project: tempfile::TempDir, + _cache: tempfile::TempDir, + project_root: PathBuf, + storage_path: PathBuf, + activation: codestory_runtime::ActivationService, + input: tokio::io::DuplexStream, + output: BufReader, + serving: tokio::task::JoinHandle>, + } + + impl LiveStdioFixture { + async fn send_call(&mut self, id: &str, name: &str, arguments: serde_json::Value) { + self.input + .write_all( + format!( + "{}\n", + json!({ + "jsonrpc": "2.0", + "id": id, + "method": "tools/call", + "params": {"name": name, "arguments": arguments} + }) + ) + .as_bytes(), + ) + .await + .expect("send tools/call"); + } + + async fn read_response(&mut self, id: &str, timeout: Duration) -> serde_json::Value { + tokio::time::timeout(timeout, async { + loop { + let mut line = String::new(); + self.output + .read_line(&mut line) + .await + .expect("read response line"); + let frame: serde_json::Value = + serde_json::from_str(&line).expect("response JSON-RPC frame"); + if frame.get("id") == Some(&json!(id)) { + return frame; + } + } + }) + .await + .unwrap_or_else(|_| panic!("response for {id} never arrived")) + } + + /// Drain any frames for `id` that arrive inside `window`; the caller + /// asserts none arrived, proving the call stayed pending. + async fn collect_frames_for( + &mut self, + id: &str, + window: Duration, + ) -> Vec { + let deadline = Instant::now() + window; + let mut frames = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return frames; + } + let mut line = String::new(); + match tokio::time::timeout(remaining, self.output.read_line(&mut line)).await { + Ok(Ok(0)) | Err(_) => return frames, + Ok(Ok(_)) => { + let frame: serde_json::Value = + serde_json::from_str(&line).expect("response JSON-RPC frame"); + if frame.get("id") == Some(&json!(id)) { + frames.push(frame); + } + } + Ok(Err(error)) => panic!("read pending response: {error}"), + } + } + } + } + + async fn live_stdio_fixture(files: &[(&str, &str)]) -> LiveStdioFixture { + let project = tempfile::tempdir().expect("project"); + let cache = tempfile::tempdir().expect("cache"); + for (name, contents) in files { + let path = project.path().join(name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).expect("source directory"); + } + std::fs::write(path, contents).expect("write source file"); + } + let mut session = StdioServerSession::new(None); + session.startup = stdio_multi_project_startup(cache.path()); + session + .select_project(project.path().to_str()) + .expect("select project"); + let (activation, storage_path) = { + let runtime = &session + .active_project + .as_ref() + .expect("selected project") + .runtime; + (runtime.activation.clone(), runtime.storage_path.clone()) + }; + let (client_input, server_input) = tokio::io::duplex(4096); + let (server_output, client_output) = tokio::io::duplex(8192); + let serving = tokio::spawn(serve_stdio_requests( + session, + BufReader::new(server_input), + server_output, + std::future::pending::<()>(), + handle_stdio_message, + STDIO_TERMINATION_DRAIN_BUDGET, + )); + LiveStdioFixture { + project_root: project.path().to_path_buf(), + _project: project, + _cache: cache, + storage_path, + activation, + input: client_input, + output: BufReader::new(client_output), + serving, + } + } + + /// Warm the project to a complete publication and return the grounding + /// payload with a symbol id for `name`. + async fn warm_fixture_and_symbol_id( + fixture: &mut LiveStdioFixture, + symbol_name: &str, + ) -> serde_json::Value { + fixture + .send_call( + "warm-ground", + "ground", + json!({"project": fixture.project_root, "budget": "balanced"}), + ) + .await; + let ground = fixture + .read_response("warm-ground", Duration::from_secs(60)) + .await; + let result = &ground["result"]["structuredContent"]; + assert!(result.is_object(), "warm-up ground must converge: {ground}"); + let node_id = result["files"] + .as_array() + .into_iter() + .flatten() + .flat_map(|file| file["symbols"].as_array().into_iter().flatten()) + .chain(result["root_symbols"].as_array().into_iter().flatten()) + .find(|symbol| { + symbol["label"] + .as_str() + .is_some_and(|label| label.starts_with(&format!("{symbol_name} @ "))) + }) + .and_then(|symbol| symbol["id"].as_str()) + .unwrap_or_else(|| panic!("warm-up ground must expose {symbol_name}: {result}")) + .to_string(); + assert_eq!( + ground.pointer("/result/_meta/codestory_publication/freshness/state"), + Some(&json!("fresh")), + "a warm read with no refresh in flight reports fresh: {ground}" + ); + json!({"ground": result.clone(), "node_id": node_id}) + } + + #[tokio::test] + async fn graph_only_reads_serve_retained_publication_as_historical_during_refresh() { + let mut fixture = + live_stdio_fixture(&[("lib.rs", "pub fn admitted_anchor() -> u32 { 1 }\n")]).await; + warm_fixture_and_symbol_id(&mut fixture, "admitted_anchor").await; + + std::fs::write( + fixture.project_root.join("lib.rs"), + "pub fn admitted_anchor() -> u32 { 2 }\npub fn added_after() {}\n", + ) + .expect("mutate indexed source"); + // The refresh worker parks while it owns the writer lock with + // `is_indexing` set: reads that reach the controller observe a refresh + // that is genuinely in flight, not one parked before indexing. + let _hold = MidIndexingHold::arm(&fixture.storage_path); + + fixture + .send_call( + "refresh-ground", + "ground", + json!({"project": fixture.project_root, "budget": "balanced"}), + ) + .await; + let ground = fixture + .read_response("refresh-ground", Duration::from_secs(30)) + .await; + let result = &ground["result"]["structuredContent"]; + assert!(result.is_object(), "ground must serve, not fail: {ground}"); + let meta = &ground["result"]["_meta"]["codestory_publication"]; + assert_eq!( + meta["freshness"]["state"], + json!("historical"), + "a graph-only read during refresh answers from the retained publication: {ground}" + ); + assert_eq!( + meta["freshness"]["reason"], + json!("refresh_in_progress"), + "{ground}" + ); + assert_eq!( + meta["served_from"], + json!("last_complete_publication"), + "{ground}" + ); + let wire = serde_json::to_string(&ground).expect("serialize ground"); + assert!( + !wire.contains("invalid_argument") && !wire.contains("activation_preparing"), + "graph-only reads must not fail as a bad argument while indexing runs: {wire}" + ); + } + + #[tokio::test] + async fn source_backed_snippet_waits_for_the_fresh_core_during_refresh() { + let mut fixture = + live_stdio_fixture(&[("lib.rs", "pub fn source_anchor() -> u32 { 1 }\n")]).await; + let warm = warm_fixture_and_symbol_id(&mut fixture, "source_anchor").await; + let node_id = warm["node_id"].as_str().expect("symbol id").to_string(); + let retained_generation = warm["ground"]["_meta"] + .get("codestory_publication") + .cloned(); + std::fs::write( + fixture.project_root.join("lib.rs"), + "pub fn source_anchor() -> u32 { 2 }\npub fn appended_marker() {}\n", + ) + .expect("mutate indexed source"); + let _hold = MidIndexingHold::arm(&fixture.storage_path); + + fixture + .send_call( + "held-snippet", + "snippet", + json!({"project": fixture.project_root, "id": node_id}), + ) + .await; + let pending = fixture + .collect_frames_for("held-snippet", Duration::from_secs(6)) + .await; + assert!( + pending.is_empty(), + "a source-backed read must wait for the fresh core, not answer from retained: {pending:?}" + ); + + drop(_hold); + let response = fixture + .read_response("held-snippet", Duration::from_secs(60)) + .await; + let snippet = &response["result"]["structuredContent"]; + assert!( + snippet["snippet"].as_str().is_some(), + "snippet must serve after the refresh completes: {response}" + ); + let meta = &response["result"]["_meta"]["codestory_publication"]; + assert_eq!( + meta["freshness"]["state"], + json!("fresh"), + "the waited answer comes from the fresh publication: {response}" + ); + assert_eq!( + meta["served_from"], + json!("complete_publication"), + "{response}" + ); + let _ = retained_generation; + } + + #[tokio::test] + async fn source_backed_deadline_returns_activation_preparing_with_exact_resume() { + let project = tempfile::tempdir().expect("project"); + let cache = tempfile::tempdir().expect("cache"); + std::fs::write( + project.path().join("lib.rs"), + "pub fn resume_source_anchor() -> u32 { 1 }\n", + ) + .expect("source file"); + let mut session = StdioServerSession::new(None); + session.startup = stdio_multi_project_startup(cache.path()); + session.preparation_wait_budget_for_test = Some(Duration::from_millis(120)); + session + .select_project(project.path().to_str()) + .expect("select project"); + let (activation, _storage_path) = { + let runtime = &session + .active_project + .as_ref() + .expect("selected project") + .runtime; + (runtime.activation.clone(), runtime.storage_path.clone()) + }; + let worker_gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + activation.set_worker_start_gate_for_test(Some(Arc::clone(&worker_gate))); + let _cleanup = PreparationTestCleanup { + activation: activation.clone(), + gate: Arc::clone(&worker_gate), + restore_snapshot: None, + }; + let (mut client_input, server_input) = tokio::io::duplex(4096); + let (server_output, client_output) = tokio::io::duplex(8192); + let serving = tokio::spawn(serve_stdio_requests( + session, + BufReader::new(server_input), + server_output, + std::future::pending::<()>(), + handle_stdio_message, + STDIO_TERMINATION_DRAIN_BUDGET, + )); + client_input + .write_all( + format!( + "{}\n", + json!({ + "jsonrpc": "2.0", + "id": "deadline-snippet", + "method": "tools/call", + "params": { + "name": "snippet", + "arguments": { + "project": project.path(), + "id": "any-node" + } + } + }) + ) + .as_bytes(), + ) + .await + .expect("send snippet request"); + let mut output = BufReader::new(client_output); + let first = tokio::time::timeout(Duration::from_secs(10), async { + loop { + let mut line = String::new(); + output + .read_line(&mut line) + .await + .expect("read deadline response"); + let frame: serde_json::Value = + serde_json::from_str(&line).expect("deadline JSON-RPC"); + if frame.get("id") == Some(&json!("deadline-snippet")) { + return frame; + } + } + }) + .await + .expect("bounded preparation deadline response"); + let deadline = &first["result"]["structuredContent"]; + assert_eq!(deadline["kind"], json!("preparing"), "{first}"); + assert_eq!(deadline["deadline_exceeded"], json!(true), "{first}"); + let snapshot = activation.snapshot().expect("running activation"); + assert_eq!( + deadline["resume_operation_id"].as_str(), + Some(snapshot.operation_id.as_str()), + "resume must name the exact running operation: {first}" + ); + assert_eq!( + deadline["resume_operation_attempt"].as_u64(), + Some(u64::from(snapshot.attempt)), + "{first}" + ); + drop(serving); + } + + #[tokio::test] + async fn exact_search_and_packet_wait_for_the_fresh_publication() { + let mut fixture = + live_stdio_fixture(&[("lib.rs", "pub fn exact_anchor() -> u32 { 1 }\n")]).await; + warm_fixture_and_symbol_id(&mut fixture, "exact_anchor").await; + std::fs::write( + fixture.project_root.join("lib.rs"), + "pub fn exact_anchor() -> u32 { 2 }\npub fn second_marker() {}\n", + ) + .expect("mutate indexed source"); + let _hold = MidIndexingHold::arm(&fixture.storage_path); + + fixture + .send_call( + "held-exact", + "search", + json!({ + "project": fixture.project_root, + "query": "second_marker", + "repo_text": "off" + }), + ) + .await; + let pending = fixture + .collect_frames_for("held-exact", Duration::from_secs(6)) + .await; + assert!( + pending.is_empty(), + "exact search must wait for the fresh core: {pending:?}" + ); + drop(_hold); + let response = fixture + .read_response("held-exact", Duration::from_secs(60)) + .await; + let wire = serde_json::to_string(&response).expect("serialize exact response"); + assert!( + wire.contains("second_marker"), + "exact search must observe the refreshed source: {wire}" + ); + assert_eq!( + response.pointer("/result/_meta/codestory_publication/freshness/state"), + Some(&json!("fresh")), + "{response}" + ); + + std::fs::write( + fixture.project_root.join("lib.rs"), + "pub fn exact_anchor() -> u32 { 3 }\npub fn third_marker() {}\n", + ) + .expect("mutate indexed source again"); + let _hold = MidIndexingHold::arm(&fixture.storage_path); + fixture + .send_call( + "held-packet", + "packet", + json!({ + "project": fixture.project_root, + "question": "what does third_marker do" + }), + ) + .await; + let pending = fixture + .collect_frames_for("held-packet", Duration::from_secs(6)) + .await; + assert!( + pending.is_empty(), + "packet must spend its wait on the refresh instead of answering early: {pending:?}" + ); + drop(_hold); + let response = fixture + .read_response("held-packet", Duration::from_secs(60)) + .await; + let is_error = response.pointer("/result/isError") == Some(&json!(true)) + || response.pointer("/error").is_some(); + assert!( + is_error || response["result"]["structuredContent"].is_object(), + "packet may only answer with a fresh publication or a typed refusal: {response}" + ); + } + + #[tokio::test] + async fn pinned_observers_report_drift_while_a_refresh_waits() { + let project = tempfile::tempdir().expect("project"); + let cache = tempfile::tempdir().expect("cache"); + std::fs::write( + project.path().join("lib.rs"), + "pub fn observed_anchor() -> u32 { 1 }\n", + ) + .expect("source file"); + let mut session = StdioServerSession::new(None); + session.startup = stdio_multi_project_startup(cache.path()); + session + .select_project(project.path().to_str()) + .expect("select project"); + let activation = session + .active_project + .as_ref() + .expect("selected project") + .runtime + .activation + .clone(); + let (mut client_input, server_input) = tokio::io::duplex(4096); + let (server_output, client_output) = tokio::io::duplex(8192); + let serving = tokio::spawn(serve_stdio_requests( + session, + BufReader::new(server_input), + server_output, + std::future::pending::<()>(), + handle_stdio_message, + STDIO_TERMINATION_DRAIN_BUDGET, + )); + let mut output = BufReader::new(client_output); + // Warm the project to a complete publication. + client_input + .write_all( + format!( + "{}\n", + json!({"jsonrpc":"2.0","id":"warm","method":"tools/call", + "params":{"name":"ground","arguments":{"project":project.path(),"budget":"balanced"}}}) + ) + .as_bytes(), + ) + .await + .expect("send warm-up"); + loop { + let mut line = String::new(); + tokio::time::timeout(Duration::from_secs(60), output.read_line(&mut line)) + .await + .expect("warm-up response") + .expect("warm-up line"); + let frame: serde_json::Value = serde_json::from_str(&line).expect("warm JSON"); + if frame.get("id") == Some(&json!("warm")) { + assert!(frame["result"]["structuredContent"].is_object(), "{frame}"); + break; + } + } + + std::fs::write( + project.path().join("lib.rs"), + "pub fn observed_anchor() -> u32 { 2 }\n", + ) + .expect("mutate indexed source"); + // Park the refresh before its worker starts: the operation is in + // flight and the core is retained, but the controller is not yet + // mid-index, which is exactly the window a pinned observer must still + // answer in. + let worker_gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + activation.set_worker_start_gate_for_test(Some(Arc::clone(&worker_gate))); + let _cleanup = PreparationTestCleanup { + activation: activation.clone(), + gate: Arc::clone(&worker_gate), + restore_snapshot: None, + }; + + // Trigger the refresh with a graph call that returns historically, + // then ask the pinned observer while the worker is still parked. + client_input + .write_all( + format!( + "{}\n", + json!({"jsonrpc":"2.0","id":"trigger","method":"tools/call", + "params":{"name":"ground","arguments":{"project":project.path(),"budget":"balanced"}}}) + ) + .as_bytes(), + ) + .await + .expect("send refresh trigger"); + loop { + let mut line = String::new(); + tokio::time::timeout(Duration::from_secs(30), output.read_line(&mut line)) + .await + .expect("trigger response") + .expect("trigger line"); + let frame: serde_json::Value = serde_json::from_str(&line).expect("trigger JSON"); + if frame.get("id") == Some(&json!("trigger")) { + assert_eq!( + frame.pointer("/result/_meta/codestory_publication/freshness/state"), + Some(&json!("historical")), + "the trigger read itself serves historically: {frame}" + ); + break; + } + } + assert!( + activation.snapshot().is_some_and(|snapshot| matches!( + snapshot.state, + codestory_runtime::ActivationState::Preparing + | codestory_runtime::ActivationState::Updating + )), + "the refresh must still be in flight for the pinned-observer assertion" + ); + + client_input + .write_all( + format!( + "{}\n", + json!({"jsonrpc":"2.0","id":"affected-mid-refresh","method":"tools/call", + "params":{"name":"affected","arguments":{"project":project.path(),"paths":["lib.rs"]}}}) + ) + .as_bytes(), + ) + .await + .expect("send affected during refresh"); + let affected = loop { + let mut line = String::new(); + tokio::time::timeout(Duration::from_secs(15), output.read_line(&mut line)) + .await + .expect("affected response while refresh parked") + .expect("affected line"); + let frame: serde_json::Value = serde_json::from_str(&line).expect("affected JSON"); + if frame.get("id") == Some(&json!("affected-mid-refresh")) { + break frame; + } + }; + let result = &affected["result"]["structuredContent"]; + assert!( + result.is_object(), + "affected must still answer from the pinned core: {affected}" + ); + assert!( + result["uncovered_inputs"] + .as_array() + .is_some_and(|inputs| inputs.iter().any(|input| { + input["path"] == "lib.rs" && input["classification"] == "stale_index" + })), + "affected must report the drift against the pinned publication, not refresh it away: {affected}" + ); + drop(serving); + } + + #[tokio::test] + async fn failed_replacement_serves_historical_graph_and_refuses_source_reads() { + let mut fixture = + live_stdio_fixture(&[("lib.rs", "pub fn failing_anchor() -> u32 { 1 }\n")]).await; + let warm = warm_fixture_and_symbol_id(&mut fixture, "failing_anchor").await; + let node_id = warm["node_id"].as_str().expect("symbol id").to_string(); + + // An incomplete member inventory fences the replacement: the refresh + // fails closed and the previous publication stays retained. + std::fs::write( + fixture.project_root.join("codestory_workspace.json"), + r#"{"members":["missing"]}"#, + ) + .expect("fence the replacement"); + + fixture + .send_call( + "failed-ground", + "ground", + json!({"project": fixture.project_root, "budget": "balanced"}), + ) + .await; + let ground = fixture + .read_response("failed-ground", Duration::from_secs(60)) + .await; + let meta = &ground["result"]["_meta"]["codestory_publication"]; + assert_eq!( + meta["freshness"]["state"], + json!("historical"), + "a graph-only read after a failed replacement stays on the retained publication: {ground}" + ); + assert_eq!( + meta["freshness"]["reason"], + json!("replacement_failed"), + "{ground}" + ); + + fixture + .send_call( + "failed-snippet", + "snippet", + json!({"project": fixture.project_root, "id": node_id}), + ) + .await; + let response = fixture + .read_response("failed-snippet", Duration::from_secs(60)) + .await; + let wire = serde_json::to_string(&response).expect("serialize snippet response"); + assert!( + response.pointer("/result/isError") == Some(&json!(true)) + || response + .pointer("/result/structuredContent/error") + .is_some() + || wire.contains("source_discovery_incomplete") + || wire.contains("project_unavailable"), + "a source-backed read must surface the causal refresh failure, not retained bytes: {wire}" + ); + assert!( + !wire.contains("\"snippet\":"), + "no retained snippet body may be served: {wire}" + ); + } } diff --git a/crates/codestory-cli/src/stdio_v3/diagnostics.rs b/crates/codestory-cli/src/stdio_v3/diagnostics.rs index 9dde700bd..130c6bc97 100644 --- a/crates/codestory-cli/src/stdio_v3/diagnostics.rs +++ b/crates/codestory-cli/src/stdio_v3/diagnostics.rs @@ -515,6 +515,42 @@ mod tests { other_session.read_at(&newest.unwrap().uri, now), Err(DiagnosticsReadErrorV3::CapabilityUnavailable) ); + + // The identical binding registered under two distinct session secrets + // must mint distinct capabilities: each own URI reads, each foreign + // URI refuses. Omitting the session secret from the capability token + // would collapse these URIs and fail here. + let shared = binding(Uuid::new_v4().to_string(), 77_000); + let mut session_a = DiagnosticsRegistryV3::new_with_secret([10; 32]); + let mut session_b = DiagnosticsRegistryV3::new_with_secret([11; 32]); + let grant_a = session_a + .register_at(shared.clone(), vec![7], now) + .expect("first session registers the shared binding"); + let grant_b = session_b + .register_at(shared, vec![7], now) + .expect("second session registers the shared binding"); + assert_ne!( + grant_a.uri, grant_b.uri, + "distinct session secrets must bind distinct capability URIs" + ); + assert_eq!( + &*session_a.read_at(&grant_a.uri, now).expect("own URI"), + &[7] + ); + assert_eq!( + &*session_b.read_at(&grant_b.uri, now).expect("own URI"), + &[7] + ); + assert_eq!( + session_b.read_at(&grant_a.uri, now), + Err(DiagnosticsReadErrorV3::CapabilityUnavailable), + "the same binding under a foreign secret must not read" + ); + assert_eq!( + session_a.read_at(&grant_b.uri, now), + Err(DiagnosticsReadErrorV3::CapabilityUnavailable), + "the same binding under a foreign secret must not read" + ); } #[test] diff --git a/crates/codestory-cli/tests/architecture_contracts.rs b/crates/codestory-cli/tests/architecture_contracts.rs index e285fc809..32955c6e0 100644 --- a/crates/codestory-cli/tests/architecture_contracts.rs +++ b/crates/codestory-cli/tests/architecture_contracts.rs @@ -918,10 +918,16 @@ fn public_exact_verifier_compiles_without_qualification_support() { "the inert launcher discovery session lost Rust's {revision} revision" ); } + // LIVE-L2 bumped the publication stamp to schema 4 (the launcher exposes + // it as `publicationSchemaVersion`; minimum-compatible stays 3). assert!( - launcher.contains("publicationSchemaVersion: 3"), + launcher.contains("publicationStampSchemaVersion = 4"), "the inert launcher discovery session must preserve the Rust discovery schema" ); + assert!( + launcher.contains("minimumCompatiblePublicationStampSchemaVersion = 3"), + "the inert launcher must keep schema-3 producers readable" + ); } #[cfg(feature = "proof-qualification-support")] @@ -959,7 +965,8 @@ process.stdout.write(JSON.stringify( session["discoveryContractSha256"], contracts["2025-06-18"], "the launcher must retain Rust's discovery digest without substituting one" ); - assert_eq!(session["publicationSchemaVersion"], 3); + // LIVE-L2 bumped the publication stamp to schema 4. + assert_eq!(session["publicationSchemaVersion"], 4); } #[test] @@ -2020,8 +2027,12 @@ fn production_source_never_spawns_git() { { continue; } + // `test_git.rs` compiles only under `#[cfg(test)]` (the `mod` gate + // lives in the workspace lib.rs) — it is the isolated fixture git + // builder, never product code. if path.starts_with(&benchmark_root) || path == repo_root().join("crates/codestory-runtime/src/test_support.rs") + || path == repo_root().join("crates/codestory-workspace/src/test_git.rs") { continue; } @@ -2063,6 +2074,9 @@ fn crate_source_git_spawns_are_limited_to_named_non_product_boundaries() { "crates/codestory-bench/src/bin/codestory_proof_availability/multilingual_contract.rs" .to_owned(), "crates/codestory-runtime/src/test_support.rs".to_owned(), + // `#[cfg(test)]`-gated isolated fixture git builder (the `mod` gate is + // in codestory-workspace's lib.rs, invisible to this file scanner). + "crates/codestory-workspace/src/test_git.rs".to_owned(), ]); assert_eq!( @@ -3418,14 +3432,14 @@ fn retrieval_annotations_are_classified_by_typed_kind_not_by_prose() { /// /// READY-C (#1654) shipped `validation_receipts` documenting that "replacement, /// truncation, in-place rewriting ... all break the seal", and two tests -/// asserting exactly that. Neither statement holds on Windows: `std::fs` -/// reports no device/inode pair and no inode-change instant there, so a -/// same-length rewrite that restores the modification time produces an -/// identical observation and is answered from the receipt. Nothing contradicted -/// the claim because `codestory-contracts` tests run only on Linux and macOS — -/// the Windows lanes in `source-proof.yml` build `codestory-workspace` and -/// `codestory-llama-sys` test targets only. The limit is therefore stated, in -/// the contract and in the docs, and this is what keeps it stated. +/// asserting exactly that. Windows contradicted both while `std::fs` reported +/// no device/inode pair and no inode-change instant there, and nothing pinned +/// it because `codestory-contracts` tests ran only on Linux and macOS. Windows +/// now observes the same native identity through a bounded handle query +/// (volume serial, file index, NTFS ChangeTime); what remains is the fallback +/// case — a platform or filesystem that reports none of that, including a +/// Windows file whose query fails — and that weaker case must stay named in +/// the contract and the docs, which is what this keeps pinned. #[test] fn the_sealed_receipt_states_its_windows_limit_in_the_contract_and_the_docs() { let receipts = read("crates/codestory-contracts/src/validation_receipts.rs"); diff --git a/crates/codestory-cli/tests/cache_reset_immutable_core.rs b/crates/codestory-cli/tests/cache_reset_immutable_core.rs index db7047241..1cc230b51 100644 --- a/crates/codestory-cli/tests/cache_reset_immutable_core.rs +++ b/crates/codestory-cli/tests/cache_reset_immutable_core.rs @@ -111,9 +111,18 @@ impl Fixture { fixture } - fn set_forward_schema(&self) { + fn set_forward_schema(&self) -> u32 { // Fault a real complete current generation to model a future writer. // This is not an archived schema fixture or a newer-binary claim. + // The forward version is the seeded database's own schema plus one — + // the seeded bytes carry the build's CURRENT_SCHEMA_VERSION, so this + // stays forward no matter which schema the binary ships. + let seeded: u32 = + Connection::open_with_flags(&self.database, OpenFlags::SQLITE_OPEN_READ_ONLY) + .expect("read seeded schema") + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .expect("seeded schema version"); + let forward = seeded + 1; let original = fs::metadata(&self.database) .expect("metadata") .permissions(); @@ -132,7 +141,7 @@ impl Fixture { assert_eq!(&bytes[..16], b"SQLite format 3\0"); // SQLite's user_version is the big-endian word at header offset 60. // Change exactly this fault boundary, without enabling a WAL writer. - bytes[60..64].copy_from_slice(&36_u32.to_be_bytes()); + bytes[60..64].copy_from_slice(&forward.to_be_bytes()); fs::set_permissions(&self.database, writable).expect("fixture writable"); fs::write(&self.database, bytes).expect("forward-schema header fault"); fs::set_permissions(&self.database, original).expect("restore sealed permissions"); @@ -149,7 +158,7 @@ impl Fixture { let version: u32 = reader .query_row("PRAGMA user_version", [], |row| row.get(0)) .expect("forward schema version"); - assert_eq!(version, 36); + assert_eq!(version, forward); for table in ["node", "edge", "index_publication"] { let count: i64 = reader .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { @@ -158,6 +167,7 @@ impl Fixture { .expect("retained complete graph/publication"); assert!(count > 0, "{table} must survive fault injection"); } + forward } fn run(&self, args: &[&str]) -> Output { @@ -236,17 +246,18 @@ fn dry_run_identifies_newer_immutable_core_without_mutation() { #[test] fn confirmed_reset_recovers_newer_immutable_core_and_preserves_annotations() { let fixture = Fixture::new(); - fixture.set_forward_schema(); + let forward = fixture.set_forward_schema(); let refused = fixture.run(&["index", "--refresh", "full", "--format", "json"]); assert!( !refused.status.success(), "newer schema must refuse ordinary full refresh" ); let refused_json: Value = serde_json::from_slice(&refused.stdout).expect("JSON error"); + let expected = format!("Unsupported database schema version: {forward}"); assert!( refused_json["error"]["message"] .as_str() - .is_some_and(|message| message.contains("Unsupported database schema version: 36")), + .is_some_and(|message| message.contains(&expected)), "{refused:?}" ); let annotations = snapshot_annotations(&fixture.cache); diff --git a/crates/codestory-cli/tests/cli_error_contracts.rs b/crates/codestory-cli/tests/cli_error_contracts.rs index ba02f604d..2bf34119a 100644 --- a/crates/codestory-cli/tests/cli_error_contracts.rs +++ b/crates/codestory-cli/tests/cli_error_contracts.rs @@ -1479,9 +1479,10 @@ fn choose_flag_resolves_by_displayed_alternative_number_when_available() { .expect("run symbol help"); assert_success(&help, "symbol --help failed"); let help_text = String::from_utf8_lossy(&help.stdout); - if !help_text.contains("--choose") { - return; - } + assert!( + help_text.contains("--choose"), + "symbol --help must advertise --choose for the numbered-alternative flow: {help_text}" + ); let workspace = tempdir().expect("workspace dir"); let cache_dir = tempdir().expect("cache dir"); diff --git a/crates/codestory-cli/tests/doctor_ready_observational.rs b/crates/codestory-cli/tests/doctor_ready_observational.rs index a9cb6e5e7..e039e4915 100644 --- a/crates/codestory-cli/tests/doctor_ready_observational.rs +++ b/crates/codestory-cli/tests/doctor_ready_observational.rs @@ -245,6 +245,224 @@ fn assert_unavailable_verdict(command: &str, json: &Value, reason: &str) { } } +/// Seed a real publication before constructing an incompatible generation. +fn prepare_current_generation(project: &Path, cache: &Path) -> PathBuf { + fs::create_dir(project).expect("project"); + fs::write( + project.join("lib.rs"), + "pub fn alpha() -> i32 { 1 }\npub fn beta() -> i32 { alpha() }\n", + ) + .expect("source"); + let seed: Value = serde_json::from_str(&run_cli( + project, + cache, + &["index", "--refresh", "full", "--format", "json"], + )) + .expect("seed index json"); + assert!(seed["summary"]["stats"]["node_count"].as_u64().unwrap_or(0) > 0); + active_generation_database(cache) +} + +fn active_generation_database(cache: &Path) -> PathBuf { + let pointer: Value = serde_json::from_slice( + &fs::read(cache.join("core/publication.json")).expect("committed core pointer"), + ) + .expect("core pointer json"); + cache + .join("core/generations") + .join( + pointer["active"]["generation_id"] + .as_str() + .expect("generation id"), + ) + .join("codestory.db") +} + +#[test] +fn doctor_preserves_incompatible_generations_and_reports_recovery() { + for newer in [false, true] { + assert_incompatible_generation_is_observational("doctor", newer); + } +} + +#[test] +fn ready_preserves_incompatible_generations_and_reports_recovery() { + for newer in [false, true] { + assert_incompatible_generation_is_observational("ready", newer); + } +} + +fn assert_incompatible_generation_is_observational(command: &str, newer: bool) { + let fixture = tempdir().expect("fixture"); + let project = fixture.path().join("project"); + let cache = fixture.path().join("cache"); + let current = schema_version(&prepare_current_generation(&project, &cache)); + let schema = if newer { current + 1 } else { current - 1 }; + let database = test_support::set_active_core_schema_version(&cache, schema); + // A plain read-only SQLite open of a WAL database can materialize -shm/-wal + // in a writable directory, so probe the durable version before the baseline + // snapshot rather than letting the probe pollute the comparison. + assert_eq!(schema_version(&database), schema); + let before = snapshot_tree(&cache); + + let output = run_cli(&project, &cache, &[command, "--format", "json"]); + let json: Value = serde_json::from_str(&output).expect("diagnostic json"); + assert_eq!( + schema_version(&database), + schema, + "{command} migrated the incompatible generation" + ); + assert_eq!( + snapshot_tree(&cache), + before, + "{command} changed stale cache files or sidecars" + ); + if !newer { + assert_eq!( + json["core_status"], "upgrade_required", + "{command}: {output}" + ); + assert_unavailable_verdict(command, &json, &format!("schema {schema}")); + } else { + assert_eq!(json["core_status"], "newer_schema", "{command}: {output}"); + let verdicts = if command == "doctor" { + &json["readiness"] + } else { + &json["verdicts"] + }; + for verdict in verdicts.as_array().unwrap() { + assert_eq!(verdict["status"], "repair_index"); + let minimum = verdict["minimum_next"].as_array().unwrap(); + assert!( + minimum[0].as_str().unwrap().contains("cache reset") + && minimum[0].as_str().unwrap().contains("--derived-only") + && minimum[0].as_str().unwrap().contains("--dry-run"), + "{verdict}" + ); + assert!( + minimum[1].as_str().unwrap().contains("--confirm"), + "{verdict}" + ); + assert!( + minimum[2].as_str().unwrap().contains("--refresh full"), + "{verdict}" + ); + let full = verdict["full_repair"].as_array().unwrap(); + assert_eq!( + &full[..minimum.len()], + minimum.as_slice(), + "full recovery must retain the ordered reset and rebuild steps" + ); + assert!(full.last().unwrap().as_str().unwrap().contains("doctor")); + } + if command == "doctor" { + assert!( + json["next_commands"][0] + .as_str() + .unwrap() + .contains("cache reset"), + "{json}" + ); + } + } +} + +#[test] +fn doctor_lists_other_cached_projects_with_stale_core_schema() { + let fixture = tempdir().expect("fixture"); + let project_a = fixture.path().join("project-a"); + let cache_a = fixture.path().join("cache-a"); + fs::create_dir(&project_a).expect("project a"); + fs::write(project_a.join("a.rs"), "pub fn alpha() {}\n").expect("source a"); + run_cli( + &project_a, + &cache_a, + &["index", "--refresh", "full", "--format", "json"], + ); + + // A second project indexed through the derived process cache root lands + // beside other project caches exactly as an operator's ambient cache does. + let project_b = fixture.path().join("project-b"); + fs::create_dir(&project_b).expect("project b"); + fs::write(project_b.join("b.rs"), "pub fn beta() {}\n").expect("source b"); + // The derived project cache lives under the process cache root, which the + // CLI resolves from CODESTORY_STDIO_CACHE_ROOT when it is set. + let process_root = fixture.path().join("stdio-cache"); + let indexed = test_support::cli_command() + .args(["index", "--refresh", "full", "--format", "json"]) + .arg("--project") + .arg(&project_b) + .env("CODESTORY_CACHE_ROOT", fixture.path().join("global-cache")) + .env("CODESTORY_STDIO_CACHE_ROOT", &process_root) + .env("CODESTORY_PLUGIN_DATA", fixture.path().join("plugin-data")) + .env("CODESTORY_TEST_EMBED_ALLOW_CPU", "1") + .output() + .expect("run sibling index"); + assert!( + indexed.status.success(), + "sibling index failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&indexed.stdout), + String::from_utf8_lossy(&indexed.stderr) + ); + // The process cache root also holds non-project directories (retention, + // models); derive the sibling's cache the same way the CLI does. + let cache_b = process_root.join(codestory_workspace::workspace_id_v3_for_root(&project_b)); + assert!( + cache_b.join("core").is_dir(), + "derived sibling cache missing: {}", + cache_b.display() + ); + let current = schema_version(&active_generation_database(&cache_b)); + for schema in [current - 1, current + 1] { + let database_b = test_support::set_active_core_schema_version(&cache_b, schema); + let process_before = snapshot_tree(&process_root); + + let output = run_cli(&project_a, &cache_a, &["doctor", "--format", "json"]); + let json: Value = serde_json::from_str(&output).expect("doctor json"); + let stale = json["stale_cached_cores"] + .as_array() + .expect("stale_cached_cores array"); + let entry = stale + .iter() + .find(|entry| entry["found_schema"].as_u64() == Some(schema as u64)) + .unwrap_or_else(|| panic!("doctor must report the stale sibling cache: {output}")); + let required = entry["required_schema"].as_u64().expect("required schema"); + assert!( + required == current as u64, + "required schema must be the current core schema: {entry}" + ); + { + let root = entry["project_root"] + .as_str() + .expect("attributed sibling project root"); + let action = entry["next_action"].as_str().expect("next action"); + if schema > current { + assert!( + action.contains("cache reset") + && action.contains("--derived-only") + && action.contains("--dry-run"), + "newer sibling must name derived-cache recovery: {action}" + ); + } else { + assert!( + action.contains("index --project") && action.contains("--refresh full"), + "older sibling must name full refresh: {action}" + ); + } + assert!( + action.contains(root), + "next action must target the sibling's project root: {action}" + ); + } + assert_eq!( + snapshot_tree(&process_root), + process_before, + "the cross-project scan must not create, migrate, or recover cache state" + ); + assert_eq!(schema_version(&database_b), schema); + } +} + fn prepare_complete_schema31(project: &Path, cache: &Path) { fs::create_dir(project).expect("project"); fs::write( @@ -360,10 +578,16 @@ fn row_count(database: &Connection, table: &str) -> i64 { } fn schema_version(database: &Path) -> u32 { - Connection::open_with_flags(database, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) - .expect("read database") - .query_row("PRAGMA user_version", [], |row| row.get(0)) - .expect("schema version") + // A plain read-only open of a WAL database can still materialize -shm/-wal + // in a writable directory; immutable keeps the probe strictly observational. + let uri = format!("file:{}?mode=ro&immutable=1", database.display()); + Connection::open_with_flags( + &uri, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_URI, + ) + .expect("read database") + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .expect("schema version") } fn snapshot_tree(root: &Path) -> BTreeMap { @@ -399,7 +623,18 @@ fn snapshot_tree_if_exists(root: &Path) -> Option> { } fn run_cli(project: &Path, cache: &Path, args: &[&str]) -> String { - let output = test_support::cli_command() + let output = run_cli_output(project, cache, args); + assert!( + output.status.success(), + "{args:?} failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("UTF-8 output") +} + +fn run_cli_output(project: &Path, cache: &Path, args: &[&str]) -> std::process::Output { + test_support::cli_command() .args(args) .arg("--project") .arg(project) @@ -419,14 +654,231 @@ fn run_cli(project: &Path, cache: &Path, args: &[&str]) -> String { ) .env("CODESTORY_TEST_EMBED_ALLOW_CPU", "1") .output() - .expect("run CLI"); + .expect("run CLI") +} + +/// Issue #2531-4: the default markdown failure output must carry the same +/// `context.causes` chain and `next_action` guidance the JSON envelope reports. +/// The vehicle is a real `lock_wait_timeout`: a held local-refresh state guard +/// makes `ready --wait-fresh` exhaust its bounded wait. +#[test] +fn markdown_failure_matches_json_on_a_lock_wait_timeout() { + let fixture = tempdir().expect("fixture"); + let project = fixture.path().join("project"); + let cache = fixture.path().join("cache"); + fs::create_dir(&project).expect("project"); + fs::write(project.join("lib.rs"), "pub fn alpha() {}\n").expect("source"); + run_cli( + &project, + &cache, + &["index", "--refresh", "full", "--format", "json"], + ); + fs::write( + project.join("lib.rs"), + "pub fn alpha() {}\npub fn beta() {}\n", + ) + .expect("source edit"); + + let guard_path = + cache.join(codestory_contracts::owned_artifacts::LOCAL_REFRESH_STATE_GUARD_FILE); + let holder = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&guard_path) + .expect("open refresh state guard"); + codestory_contracts::bounded_locks::acquire_with_deadline( + &holder, + codestory_contracts::bounded_locks::FileLockKind::Exclusive, + codestory_contracts::bounded_locks::LockDeadline::immediate(), + None, + ) + .expect("hold refresh state guard"); + + let json_run = run_cli_output( + &project, + &cache, + &["ready", "--wait-fresh", "--format", "json"], + ); assert!( - output.status.success(), - "{args:?} failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) + !json_run.status.success(), + "ready --wait-fresh must fail behind a held guard" ); - String::from_utf8(output.stdout).expect("UTF-8 output") + let envelope: Value = serde_json::from_slice(&json_run.stdout).expect("json failure envelope"); + let message = envelope["error"]["message"] + .as_str() + .expect("failure message"); + assert!( + message.contains("local refresh state guard"), + "the refusal must name the contended guard: {envelope}" + ); + let causes = envelope["context"]["causes"] + .as_array() + .expect("json failure causes"); + assert!( + causes.iter().any(|cause| { + cause + .as_str() + .is_some_and(|cause| cause.contains("lock_wait_timeout")) + }), + "json failure must carry the real lock_wait_timeout cause: {envelope}" + ); + + let markdown_run = run_cli_output(&project, &cache, &["ready", "--wait-fresh"]); + assert!(!markdown_run.status.success()); + let stderr = String::from_utf8_lossy(&markdown_run.stderr); + assert!( + stderr.contains(&format!("Error: {message}")), + "markdown failure must lead with the same message: {stderr}" + ); + // The two runs are separate processes: elapsed-time fields like the wait + // budget legitimately differ by a millisecond. Mask digits before comparing + // so parity is judged on the cause text, not the timing sample. + let mask_digits = |text: &str| -> String { + text.chars() + .map(|ch| if ch.is_ascii_digit() { '#' } else { ch }) + .collect() + }; + let stderr_masked = mask_digits(&stderr); + for cause in causes { + let cause = cause.as_str().expect("cause text"); + assert!( + stderr_masked.contains(&mask_digits(cause)), + "markdown failure dropped a JSON cause ({cause}): {stderr}" + ); + } + if let Some(details) = envelope["error"]["details"].as_object() + && let Some(next) = details.get("minimum_next").and_then(Value::as_array) + { + for action in next { + let action = action.as_str().expect("next action text"); + assert!( + stderr.contains(action), + "markdown failure dropped a JSON next_action ({action}): {stderr}" + ); + } + } +} + +/// Issue #2531-4: `doctor --support-bundle` writes one local file holding the +/// doctor report plus the process's diagnostics records. The records were +/// redacted at capture time, so the bundle must not leak a redacted field in +/// clear — proven here by a sentinel that only ever appears inside a failed +/// command's error chain. +#[test] +fn doctor_support_bundle_writes_the_redacted_diagnostics_records() { + let fixture = tempdir().expect("fixture"); + let project = fixture.path().join("project"); + let cache = fixture.path().join("cache"); + fs::create_dir(&project).expect("project"); + fs::write(project.join("lib.rs"), "pub fn alpha() {}\n").expect("source"); + run_cli( + &project, + &cache, + &["index", "--refresh", "full", "--format", "json"], + ); + + // A real failure records a command_failure with the whole error redacted; + // the sentinel only exists inside that error chain. + let sentinel = format!("missing-project-{}", std::process::id()); + let missing = fixture.path().join(&sentinel); + let failed = run_cli_output(&missing, &cache, &["ready"]); + assert!( + !failed.status.success(), + "ready on a missing project must fail" + ); + let failure_stderr = String::from_utf8_lossy(&failed.stderr); + assert!( + failure_stderr.contains(&sentinel), + "the live error names the missing project, proving the sentinel is a real disclosure risk: {failure_stderr}" + ); + + // Seed a panic-shaped record the way the panic hook writes one; the bundle + // assembles whatever records the sink already holds. The process + // diagnostics sink lives under the ambient cache root (CODESTORY_CACHE_ROOT + // in this fixture), captured once at process start. + let diagnostics_dir = fixture.path().join("global-cache").join("diagnostics"); + fs::create_dir_all(&diagnostics_dir).expect("diagnostics dir"); + use std::io::Write as _; + let mut log = fs::OpenOptions::new() + .append(true) + .create(true) + .open(diagnostics_dir.join("codestory.jsonl")) + .expect("open diagnostics log"); + writeln!( + log, + "{}", + concat!( + "{\"event\":\"panic\",\"level\":\"ERROR\",\"payload\":\"[redacted]\",", + "\"payload_kind\":\"str\",\"payload_bytes\":43,", + "\"location\":\"crates/codestory-indexer/src/lib.rs:3555:14\",", + "\"schema_version\":1,\"timestamp_unix_ms\":1,\"pid\":1,", + "\"correlation_id\":\"test\"}" + ) + ) + .expect("seed panic record"); + + let bundle = fixture.path().join("support-bundle.json"); + let bundle_arg = bundle.to_str().expect("bundle path").to_string(); + run_cli( + &project, + &cache, + &[ + "doctor", + "--format", + "json", + "--support-bundle", + &bundle_arg, + ], + ); + + let raw = fs::read_to_string(&bundle).expect("bundle file"); + let document: Value = serde_json::from_str(&raw).expect("bundle json"); + let canonical_project = project.canonicalize().expect("canonical project root"); + assert_eq!( + document["report"]["project"].as_str(), + Some(canonical_project.to_str().expect("project path")), + "bundle must carry the doctor report: {document}" + ); + let records = document["diagnostics"] + .as_array() + .expect("bundle diagnostics records"); + assert!( + records + .iter() + .any(|record| record["event"] == "command_failure"), + "bundle must carry the recorded command failure: {records:?}" + ); + assert!( + records.iter().any(|record| { + record["event"] == "panic" + && record["location"] == "crates/codestory-indexer/src/lib.rs:3555:14" + && record["payload_bytes"] == 43 + }), + "bundle must carry the panic site file:line:column and payload size: {records:?}" + ); + assert!( + raw.contains("[redacted]"), + "redaction markers survive: {raw}" + ); + assert!( + !raw.contains(&sentinel), + "a redacted error field must never reach the bundle in clear: {raw}" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + assert_eq!( + fs::metadata(&bundle) + .expect("bundle metadata") + .permissions() + .mode() + & 0o777, + 0o600, + "support bundles are private by default" + ); + } } mod test_support; diff --git a/crates/codestory-cli/tests/http_transport_contracts.rs b/crates/codestory-cli/tests/http_transport_contracts.rs index 27de942c2..3ac214b4b 100644 --- a/crates/codestory-cli/tests/http_transport_contracts.rs +++ b/crates/codestory-cli/tests/http_transport_contracts.rs @@ -923,14 +923,14 @@ fn http_smoke_keeps_existing_routes_and_default_semantics_against_indexed_repo() ); } assert!( - search["_meta"]["codestory_publication"]["schema_version"] == 3 + search["_meta"]["codestory_publication"]["schema_version"] == 4 && search["_meta"]["codestory_publication"]["minimum_compatible_schema_version"] == 3 && search["_meta"]["codestory_publication"]["served_from"] == "complete_publication" && search["_meta"]["codestory_publication"]["core_publication"]["mode"] == "full" && search["_meta"]["codestory_publication"]["core_publication"].is_object() && search["_meta"]["codestory_publication"]["retrieval_publication"].is_object() && search["_meta"]["codestory_publication"]["operation"]["operation_id"].is_string(), - "/search success metadata must retain schema 3 and the complete core and retrieval publications plus operation identity: {search}" + "/search success metadata must retain schema 4 and the complete core and retrieval publications plus operation identity: {search}" ); required_nonempty_string(&search, "/publication/core/project_id"); let public_core_generation_id = diff --git a/crates/codestory-cli/tests/runtime_backed_flows.rs b/crates/codestory-cli/tests/runtime_backed_flows.rs index cbf186317..a17ef0039 100644 --- a/crates/codestory-cli/tests/runtime_backed_flows.rs +++ b/crates/codestory-cli/tests/runtime_backed_flows.rs @@ -251,6 +251,31 @@ fn index_auto_dry_run_reports_missing_core_without_writing() { assert_eq!(snapshot_file_bytes(workspace.path()), source_before); } +/// The published core lives under the immutable generation layout; resolve the +/// active generation's database through the publication pointer rather than a +/// guessed path. +fn published_core_storage(cache_dir: &Path) -> PathBuf { + let publication: Value = serde_json::from_slice( + &fs::read(cache_dir.join("core").join("publication.json")) + .expect("read core publication pointer"), + ) + .expect("parse core publication pointer"); + let generation = publication["active"]["generation_id"] + .as_str() + .expect("active generation id"); + let storage = cache_dir + .join("core") + .join("generations") + .join(generation) + .join("codestory.db"); + assert!( + storage.is_file(), + "active generation database must exist: {}", + storage.display() + ); + storage +} + fn publish_schema_29_projection_fixture(workspace: &Path, cache_dir: &Path) -> PathBuf { fs::create_dir_all(cache_dir).expect("create explicit cache"); let index = run_cli_with_cache( @@ -264,7 +289,11 @@ fn publish_schema_29_projection_fixture(workspace: &Path, cache_dir: &Path) -> P String::from_utf8_lossy(&index.stderr), String::from_utf8_lossy(&index.stdout) ); - let storage_path = cache_dir.join("codestory.db"); + let storage_path = published_core_storage(cache_dir); + // The published generation is sealed read-only; this fixture deliberately + // rewrites the pinned core into a schema-29 shape inside the isolated test + // cache, so unseal the file (and any WAL siblings) first. + set_db_writable(&storage_path, true); let connection = rusqlite::Connection::open(&storage_path).expect("open indexed fixture"); let structural_counts = connection .query_row( @@ -317,9 +346,24 @@ fn publish_schema_29_projection_fixture(workspace: &Path, cache_dir: &Path) -> P ) .expect("downgrade projection fixture"); drop(connection); + set_db_writable(&storage_path, false); storage_path } +fn set_db_writable(path: &Path, writable: bool) { + for candidate in [ + path.to_path_buf(), + path.with_extension("db-wal"), + path.with_extension("db-shm"), + ] { + if let Ok(metadata) = fs::metadata(&candidate) { + let mut permissions = metadata.permissions(); + permissions.set_readonly(!writable); + fs::set_permissions(&candidate, permissions).expect("adjust fixture permissions"); + } + } +} + fn remove_workspace_source(workspace: &Path) { for entry in fs::read_dir(workspace).expect("list workspace source") { let path = entry.expect("workspace entry").path(); @@ -472,17 +516,20 @@ fn raised_source_file_cap_preserves_lexical_recall() { } #[test] -#[ignore = "builds indexed runtime fixtures; run explicitly when touching CLI/runtime read-command flows"] fn read_commands_support_explicit_auto_refresh_after_indexing() { let workspace = copy_tictactoe_workspace(); index_workspace(workspace.path()); + // Plain `search` fails closed without a full agent sidecar; the core-only + // surface is `--repo-text off` (see `search_json_fails_closed_without_full_sidecars`). let search = run_cli( workspace.path(), &[ "search", "--query", "TicTacToe", + "--repo-text", + "off", "--refresh", "auto", "--format", @@ -496,9 +543,9 @@ fn read_commands_support_explicit_auto_refresh_after_indexing() { ); let json: Value = serde_json::from_slice(&search.stdout).expect("parse search json"); - assert_publication_metadata(&json, "search", true); + assert_publication_metadata(&json, "search", false); assert!( - json["indexed_symbol_hits"] + json["evidence"] .as_array() .is_some_and(|hits| !hits.is_empty()), "auto-refresh search should still return indexed symbol hits" @@ -506,7 +553,6 @@ fn read_commands_support_explicit_auto_refresh_after_indexing() { } #[test] -#[ignore = "builds indexed runtime fixtures; run explicitly when touching CLI/runtime read-command flows"] fn symbol_query_file_filter_resolves_expected_fixture() { let workspace = copy_tictactoe_workspace(); index_workspace(workspace.path()); @@ -541,7 +587,6 @@ fn symbol_query_file_filter_resolves_expected_fixture() { } #[test] -#[ignore = "builds indexed runtime fixtures; run explicitly when touching CLI/runtime read-command flows"] fn query_command_runs_search_filter_limit_pipeline() { let workspace = copy_tictactoe_workspace(); index_workspace(workspace.path()); @@ -576,7 +621,6 @@ fn query_command_runs_search_filter_limit_pipeline() { } #[test] -#[ignore = "builds indexed runtime fixtures; run explicitly when touching CLI/runtime read-command flows"] fn query_symbol_prefers_same_exact_target_as_symbol_command() { let workspace = copy_tictactoe_workspace(); index_workspace(workspace.path()); @@ -621,7 +665,6 @@ fn query_symbol_prefers_same_exact_target_as_symbol_command() { } #[test] -#[ignore = "builds indexed runtime fixtures; run explicitly when touching CLI/runtime read-command flows"] fn trail_command_default_width_matches_query_dsl_trail() { let workspace = copy_tictactoe_workspace(); index_workspace(workspace.path()); @@ -666,7 +709,6 @@ fn trail_command_default_width_matches_query_dsl_trail() { } #[test] -#[ignore = "builds indexed runtime fixtures; run explicitly when touching CLI publication metadata"] fn graph_cli_json_surfaces_share_publication_metadata() { let workspace = copy_tictactoe_workspace(); index_workspace(workspace.path()); @@ -724,7 +766,6 @@ fn graph_cli_json_surfaces_share_publication_metadata() { } #[test] -#[ignore = "builds full retrieval fixtures; run explicitly when touching broad CLI publication metadata"] fn broad_cli_json_surfaces_share_core_and_retrieval_publications() { let workspace = broad_metadata_workspace(); index_workspace(workspace.path()); @@ -800,7 +841,6 @@ fn broad_cli_json_surfaces_share_core_and_retrieval_publications() { } #[test] -#[ignore = "builds a schema-29 indexed fixture and executes the projection-only CLI writer"] fn republish_projections_cli_uses_stored_core_after_all_source_is_removed() { let workspace = copy_tictactoe_workspace(); let cache = tempdir().expect("explicit cache"); @@ -826,12 +866,19 @@ fn republish_projections_cli_uses_stored_core_after_all_source_is_removed() { .as_u64() .is_some_and(|count| count > 0) ); - let connection = rusqlite::Connection::open(&storage_path).expect("open republished core"); + // The republish publishes a new immutable generation migrated to the + // current schema; the schema-29 fixture stays sealed. + let republished = published_core_storage(cache.path()); + assert_ne!( + republished, storage_path, + "republish must publish a distinct generation" + ); + let connection = rusqlite::Connection::open(&republished).expect("open republished core"); assert_eq!( connection .query_row("PRAGMA user_version", [], |row| row.get::<_, u32>(0)) .expect("read migrated schema"), - 30 + 36 ); assert_eq!( connection @@ -847,14 +894,18 @@ fn republish_projections_cli_uses_stored_core_after_all_source_is_removed() { } #[test] -#[ignore = "builds a schema-29 indexed fixture and verifies CLI writer-lock ordering"] fn republish_projections_cli_acquires_writer_lock_before_schema_migration() { let workspace = copy_tictactoe_workspace(); let cache = tempdir().expect("explicit cache"); let storage_path = publish_schema_29_projection_fixture(workspace.path(), cache.path()); remove_workspace_source(workspace.path()); let bytes_before = fs::read(&storage_path).expect("read schema-29 bytes"); - let lock_path = storage_path.with_extension("index-writer.lock"); + // The writer lock is taken on the logical storage path + // (`/codestory.db`), which resolves to the active generation. + let lock_path = cache + .path() + .join("codestory.db") + .with_extension("index-writer.lock"); let writer_lock = fs::OpenOptions::new() .read(true) .write(true) @@ -903,11 +954,11 @@ fn republish_projections_cli_acquires_writer_lock_before_schema_migration() { } #[test] -#[ignore = "builds a schema-29 indexed fixture and verifies retained policy cap drift fails closed"] fn republish_projections_cli_rejects_legacy_policy_cap_drift_without_mutation() { let workspace = copy_tictactoe_workspace(); let cache = tempdir().expect("explicit cache"); let storage_path = publish_schema_29_projection_fixture(workspace.path(), cache.path()); + set_db_writable(&storage_path, true); let connection = rusqlite::Connection::open(&storage_path).expect("open retained fixture"); connection .execute( @@ -919,6 +970,7 @@ fn republish_projections_cli_rejects_legacy_policy_cap_drift_without_mutation() .execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") .expect("checkpoint retained fixture"); drop(connection); + set_db_writable(&storage_path, false); remove_workspace_source(workspace.path()); let bytes_before = fs::read(&storage_path).expect("read retained bytes"); diff --git a/crates/codestory-cli/tests/search_json_output.rs b/crates/codestory-cli/tests/search_json_output.rs index 267d033d2..13a8cbd81 100644 --- a/crates/codestory-cli/tests/search_json_output.rs +++ b/crates/codestory-cli/tests/search_json_output.rs @@ -513,7 +513,7 @@ fn context_rejects_removed_hybrid_tuning_flags_as_unknown_args() { } #[test] -#[ignore = "live full-retrieval contract; requires the managed embedding runtime"] +#[ignore = "cli-live-sidecar lane; requires the managed embedding runtime (docs/contributors/testing-matrix.md#cli-live-sidecar-contracts)"] fn search_json_emits_sidecar_primary_results_without_repo_text_fallback() { let workspace = tempdir().expect("workspace dir"); write_retrieval_fixture(workspace.path()); @@ -689,7 +689,7 @@ fn search_json_emits_sidecar_primary_results_without_repo_text_fallback() { } #[test] -#[ignore = "live full-sidecar contract; requires finalized sidecar ranking evidence"] +#[ignore = "cli-live-sidecar lane; requires finalized sidecar ranking evidence (docs/contributors/testing-matrix.md#cli-live-sidecar-contracts)"] fn exact_symbol_queries_preserve_fast_path_and_top_rank() { let workspace = tempdir().expect("workspace dir"); write_search_quality_fixture(workspace.path()); @@ -750,7 +750,7 @@ fn exact_symbol_queries_preserve_fast_path_and_top_rank() { } #[test] -#[ignore = "live full-sidecar contract; requires finalized symbol/route fixture evidence"] +#[ignore = "cli-live-sidecar lane; requires finalized symbol/route fixture evidence (docs/contributors/testing-matrix.md#cli-live-sidecar-contracts)"] fn symbol_json_exposes_typed_route_endpoint_metadata() { let workspace = tempdir().expect("workspace dir"); write_search_quality_fixture(workspace.path()); @@ -856,7 +856,7 @@ fn symbol_json_exposes_typed_route_endpoint_metadata() { } #[test] -#[ignore = "live full-sidecar contract; requires finalized sidecar field-filtered search evidence"] +#[ignore = "cli-live-sidecar lane; requires finalized sidecar field-filtered search evidence (docs/contributors/testing-matrix.md#cli-live-sidecar-contracts)"] fn field_qualified_search_filters_kind_path_name_and_language() { let workspace = tempdir().expect("workspace dir"); write_search_quality_fixture(workspace.path()); diff --git a/crates/codestory-cli/tests/stdio_protocol_contracts.rs b/crates/codestory-cli/tests/stdio_protocol_contracts.rs index 28f2dc3c3..95c178795 100644 --- a/crates/codestory-cli/tests/stdio_protocol_contracts.rs +++ b/crates/codestory-cli/tests/stdio_protocol_contracts.rs @@ -198,7 +198,7 @@ fn public_v3_negotiates_revision_native_evidence_discovery() { assert!(digest.bytes().all(|byte| byte.is_ascii_hexdigit())); assert_eq!( initialized.pointer("/_meta/codestory_publication/schema_version"), - Some(&json!(3)) + Some(&json!(4)) ); assert_eq!( initialized.pointer("/_meta/codestory_publication/minimum_compatible_schema_version"), @@ -616,7 +616,25 @@ fn public_v3_cli_keeps_experimental_verification_out_of_default_help() { .args(["--help"]) .output() .expect("run top-level help"); + assert!( + output.status.success(), + "top-level help failed: {}", + String::from_utf8_lossy(&output.stderr) + ); let top_level = String::from_utf8(output.stdout).expect("UTF-8 top-level help"); + // The command must be listed as a subcommand row, not merely mentioned in + // prose: the help preamble already names `search`/`packet` in its lane + // examples, so a bare `contains` cannot tell a hidden command from an + // advertised one. + for command in ["packet", "search"] { + assert!( + top_level.lines().any(|line| { + let trimmed = line.trim_start(); + trimmed == command || trimmed.starts_with(&format!("{command} ")) + }), + "top-level help must list the `{command}` subcommand row: {top_level}" + ); + } assert!(!top_level.contains("prove-call-path"), "{top_level}"); assert!( !top_level.contains("verify-indexed-direct-calls"), @@ -1181,12 +1199,6 @@ fn tool_result_code(response: &Value) -> Option { if response.pointer("/result/structuredContent/kind") == Some(&json!("preparing")) { return Some("codestory_preparing".to_string()); } - if let Some(code) = response - .pointer("/result/structuredContent/code") - .and_then(Value::as_str) - { - return Some(code.to_string()); - } response .pointer("/result/content/0/text") .and_then(Value::as_str) @@ -1317,9 +1329,32 @@ fn assert_search_repaired_before_terminal_activation( "a terminal activation failure must name its cause: {error}" ); assert_eq!(error["retry_tool"], Value::Null); + // The terminal branch must prove the repair left a completed generation, + // not merely a directory: an empty or partial replacement would satisfy + // `is_dir` alone. + let completed = fs::read_dir(search_generations) + .expect("search repair must leave a generations directory") + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .filter_map(|entry| { + let marker_path = entry.path().join(".codestory-complete.json"); + let marker: Value = serde_json::from_slice(&fs::read(&marker_path).ok()?).ok()?; + Some((entry.file_name().to_string_lossy().into_owned(), marker)) + }) + .find(|(name, marker)| { + marker["schema_version"] == json!(1) + && marker["generation_id"].as_str() == Some(name.as_str()) + && marker["symbol_count"] + .as_u64() + .is_some_and(|count| count > 0) + && marker["tantivy_doc_count"] + .as_u64() + .is_some_and(|count| count > 0) + }); assert!( - search_generations.is_dir(), - "search repair must complete before the terminal package limitation is reported" + completed.is_some(), + "search repair must complete a generation with a matching marker \ + before the terminal package limitation is reported" ); let ground_id = format!("{id}-local-ground"); let ground = send_json( @@ -1936,7 +1971,7 @@ fn initialize_negotiates_the_protocol_revision_and_stamps_the_wire_contract() { ); assert_eq!( agreed.pointer("/_meta/codestory_publication/schema_version"), - Some(&json!(3)), + Some(&json!(4)), "the session-start stamp publishes the evidence-only v3 response schema: {agreed}" ); assert_eq!( @@ -2039,7 +2074,7 @@ fn tool_results_carry_the_publication_schema_that_defines_their_vocabulary() { let result = assert_success_envelope(&response, json!("ground-stamp")); assert_eq!( result.pointer("/_meta/codestory_publication/schema_version"), - Some(&json!(3)), + Some(&json!(4)), "a served payload must name the schema its vocabulary belongs to: {response}" ); assert_eq!( @@ -2215,6 +2250,124 @@ fn stdio_status_observes_unbuilt_index_and_ground_activates_it() { assert_allowed_surface(&refreshed, "ground", true, "local_navigation", "ready"); } +/// A 0.17.6-era cache carries schema 35 under this binary's schema 36. The +/// observational status surface must return the typed upgrade refusal with the +/// managed refresh command — never `internal` — and the activation-owned +/// `ground` call must rebuild instead of failing. +#[test] +fn stdio_status_reports_stale_core_schema_and_ground_rebuilds() { + let fixture = indexed_fixture(); + let database = test_support::set_active_core_schema_version(fixture.cache_dir.path(), 35); + let stale_bytes = fs::read(&database).expect("read downgraded generation"); + let mut server = spawn_stdio_server(&fixture); + initialize_stdio_server(&mut server, "init-stale-schema"); + + let status = send_json( + &mut server, + stdio_status_request("status-stale", fixture.workspace.path()), + ); + let status_text = status + .pointer("/result/content/0/text") + .and_then(Value::as_str) + .and_then(|text| serde_json::from_str::(text).ok()) + .unwrap_or_else(|| status["result"].clone()); + let code = status_text["code"].as_str().unwrap_or_default().to_string(); + assert_eq!( + code, "core_schema_upgrade_required", + "status on a stale core must be typed, not internal: {status}" + ); + let command = status_text["next_commands"] + .as_array() + .or_else(|| { + status_text + .pointer("/details/next_commands") + .and_then(Value::as_array) + }) + .and_then(|commands| { + commands + .iter() + .find(|command| { + command + .as_str() + .is_some_and(|text| text.contains("index --project")) + }) + .cloned() + }) + .unwrap_or_else(|| panic!("status must name the managed refresh: {status}")); + assert!( + command + .as_str() + .is_some_and(|text| text.contains("--refresh full")), + "refresh command must be a full refresh: {command}" + ); + assert_eq!( + fs::read(&database).expect("reread stale generation"), + stale_bytes, + "an observational status read must not mutate the stale generation" + ); + + let resource = send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "status-resource-stale", + "method": "resources/read", + "params": {"uri": "codestory://status", "project": fixture.workspace.path()} + }), + ); + let resource_error = assert_error_envelope(&resource, json!("status-resource-stale")); + assert_eq!( + resource_error["data"]["code"], + json!("core_schema_upgrade_required"), + "resources/read status must surface the typed schema refusal: {resource}" + ); + assert!( + resource_error["data"]["details"]["next_commands"] + .as_array() + .is_some_and(|commands| commands.iter().any(|command| { + command.as_str().is_some_and(|text| { + text.contains("index --project") && text.contains("--refresh full") + }) + })), + "resources/read must carry the managed refresh command: {resource_error}" + ); + + // The activation-owned call treats the stale core as cold: it rebuilds into + // a new generation and converges (or reports exact retry state), never an + // internal failure. + let ground = send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "ground-stale", + "method": "tools/call", + "params": {"name": "ground", "arguments": {"budget": "strict"}} + }), + ); + match tool_result_code(&ground).as_deref() { + Some("codestory_preparing") => { + assert_tool_preparing(&ground, json!("ground-stale")); + } + Some(code) => panic!("ground on a stale core must rebuild, not fail: {code}"), + None => { + let grounding = assert_tool_success(&ground, json!("ground-stale")); + assert!( + grounding["stats"]["file_count"] + .as_u64() + .is_some_and(|count| count > 0), + "rebuilt ground call should return a repository map: {ground}" + ); + } + } + if database.exists() { + assert_eq!( + fs::read(&database).expect("reread old generation"), + stale_bytes, + "rebuild must leave the previous generation image intact" + ); + } +} + #[test] fn notification_messages_do_not_produce_responses() { let fixture = indexed_fixture(); @@ -2560,7 +2713,43 @@ fn multi_project_stdio_routes_interleaved_requests_by_explicit_project() { ); } - let first_symbol = assert_tool_success( + // Query-resolution is retrieval-class; ground is graph-only and lists + // first_only with its stable id for the id-based symbol call below. + let first_ground = assert_tool_success( + &send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "multi-first-ground", + "method": "tools/call", + "params": { + "name": "ground", + "arguments": {"project": first.path(), "budget": "strict"} + } + }), + ), + json!("multi-first-ground"), + ) + .clone(); + let first_node_id = first_ground["root_symbols"] + .as_array() + .into_iter() + .flatten() + .chain( + first_ground["files"] + .as_array() + .into_iter() + .flatten() + .flat_map(|file| file["symbols"].as_array().into_iter().flatten()), + ) + .find(|symbol| { + symbol["label"] + .as_str() + .is_some_and(|label| label.starts_with("first_only")) + }) + .and_then(|symbol| symbol["id"].as_str()) + .unwrap_or_else(|| panic!("first project should expose first_only: {first_ground:#}")); + assert_tool_success( &send_json( &mut server, json!({ @@ -2569,18 +2758,12 @@ fn multi_project_stdio_routes_interleaved_requests_by_explicit_project() { "method": "tools/call", "params": { "name": "symbol", - "arguments": {"project": first.path(), "query": "first_only"} + "arguments": {"project": first.path(), "id": first_node_id} } }), ), json!("multi-first-symbol"), - ) - .clone(); - let first_node_id = first_symbol - .pointer("/node/id") - .or_else(|| first_symbol.pointer("/resolution/resolved/node_id")) - .and_then(Value::as_str) - .unwrap_or_else(|| panic!("first project should resolve first_only: {first_symbol}")); + ); let wrong_project_uri = format!( "codestory://symbol/{}?project={}", strict_resource_component(first_node_id), @@ -2649,6 +2832,9 @@ fn multi_project_packet_repairs_keep_operation_identity_project_scoped() { }) .collect::>(); let mut server = spawn_multi_project_stdio_server(cache_root.path()); + // A peer holding the writer lock is a wait, not a failure: each project's + // activation parks at the peer-writer stage and the short latency budget + // returns a resumable preparing envelope naming that operation. let packet_request = |id: &str, project: &Path| { json!({ "jsonrpc": "2.0", @@ -2658,7 +2844,8 @@ fn multi_project_packet_repairs_keep_operation_identity_project_scoped() { "name": "packet", "arguments": { "project": project, - "question": "How does AppController open a project?" + "question": "How does AppController open a project?", + "latency_budget_ms": 1000 } } }) @@ -2668,17 +2855,16 @@ fn multi_project_packet_repairs_keep_operation_identity_project_scoped() { for (index, project) in projects.iter().enumerate() { let id = format!("multi-packet-{index}"); let response = send_json(&mut server, packet_request(&id, project.path())); - let unavailable = assert_tool_error(&response, json!(id)); + let preparing = assert_tool_preparing(&response, json!(id)); assert_eq!( - unavailable["code"], - json!("codestory_unavailable"), - "the original call must report the writer-lock failure: {unavailable}" + preparing["operation"]["stage"], + json!("waiting_for_peer_writer"), + "the call must be parked on the peer's writer lock: {preparing}" ); - assert_eq!(unavailable["cause_code"], json!("cache_busy")); - assert_eq!(unavailable["state"], json!("unavailable")); - assert_eq!(unavailable["operation"]["state"], json!("retryable")); + assert!(preparing["resume_operation_id"].is_string()); + assert!(preparing["resume_operation_attempt"].is_u64()); operation_ids.push( - unavailable["operation"]["operation_id"] + preparing["operation"]["operation_id"] .as_str() .expect("project activation operation id") .to_string(), @@ -4075,11 +4261,130 @@ fn snippet_tool_exact_id_navigates_structural_evidence_but_query_stays_typed() { }), ); let error = assert_tool_error(&query_response, json!("snippet-structural-query")); + // A query snippet is a retrieval-class read: where managed retrieval is + // unavailable the refusal is the typed availability envelope; where it is + // available the resolver still reports a typed no-match. Either way the + // wire carries a typed refusal, never an untyped error or stale bytes. assert!( - error["message"] + error["code"].as_str() == Some("codestory_unavailable") + || error["message"] + .as_str() + .is_some_and(|message| message.contains("No symbol matched query")), + "stdio query snippet should retain typed refusal: {error:#}" + ); +} + +/// A symbol-identified snippet read must never serve bytes that fail the +/// indexed content hash. Mutating the bound file after indexing makes the +/// observational `codestory://snippet/` resource answer with the typed +/// `source_stale` code, while the `snippet` tool waits for the managed +/// refresh and then reports the stale id as `not_found`. +#[test] +fn snippet_id_reads_return_source_stale_when_indexed_bytes_change() { + let fixture = indexed_fixture(); + let mut server = spawn_stdio_server(&fixture); + + let ground_response = send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "ground-stale-snippet", + "method": "tools/call", + "params": {"name": "ground", "arguments": {"budget": "balanced"}} + }), + ); + let grounding = assert_tool_success(&ground_response, json!("ground-stale-snippet")); + let node_id = grounding["root_symbols"] + .as_array() + .into_iter() + .flatten() + .chain( + grounding["files"] + .as_array() + .into_iter() + .flatten() + .flat_map(|file| file["symbols"].as_array().into_iter().flatten()), + ) + .find(|symbol| { + symbol["label"] + .as_str() + .is_some_and(|label| label.starts_with("tiny-stdio-contract-fixture @ ")) + }) + .and_then(|symbol| symbol["id"].as_str()) + .unwrap_or_else(|| panic!("grounding should expose the Cargo package: {grounding:#}")) + .to_string(); + + let manifest_path = fixture.workspace.path().join("Cargo.toml"); + let manifest_original = fs::read_to_string(&manifest_path).expect("read manifest"); + let manifest_mutated = + manifest_original.replace("tiny-stdio-contract-fixture", "tiny-stdio-contract-staledx"); + assert_eq!(manifest_original.len(), manifest_mutated.len()); + let modified = fs::metadata(&manifest_path) + .expect("manifest metadata") + .modified() + .expect("manifest mtime"); + fs::write(&manifest_path, manifest_mutated).expect("mutate indexed manifest"); + fs::OpenOptions::new() + .write(true) + .open(&manifest_path) + .expect("open manifest") + .set_times(std::fs::FileTimes::new().set_modified(modified)) + .expect("restore manifest mtime"); + + // The snippet resource is an observational read: it pins the publication + // that activation already produced instead of refreshing first, so the + // drifted bytes reach the hash check and surface the typed stale code. + let resource_response = send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "snippet-stale-resource", + "method": "resources/read", + "params": { + "uri": format!("codestory://snippet/{node_id}"), + "project": fixture.workspace.path() + } + }), + ); + let resource_error = assert_error_envelope(&resource_response, json!("snippet-stale-resource")); + assert!( + resource_error["message"] + .as_str() + .is_some_and(|message| message.contains("source_stale")), + "the snippet resource must surface the typed stale-source code: {resource_error:#}" + ); + assert!( + !resource_error["message"] .as_str() - .is_some_and(|message| message.contains("No symbol matched query")), - "stdio query snippet should retain typed graph filtering: {error:#}" + .is_some_and(|message| message.contains("staledx")), + "the stale-source error must not carry bytes the index never saw: {resource_error:#}" + ); + + let tool_response = send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "snippet-stale-tool", + "method": "tools/call", + "params": {"name": "snippet", "arguments": {"id": node_id}} + }), + ); + // The snippet tool is a source-backed read: managed activation detects + // the drift, waits for the refresh, and republishes. The pinned id from + // the previous generation then no longer resolves, so the exact refusal + // is `not_found` -- and the wire still never carries the mutated bytes + // under the old identity. + let tool_error = assert_tool_error(&tool_response, json!("snippet-stale-tool")); + assert_eq!( + tool_error["code"].as_str(), + Some("not_found"), + "snippet tool waits for the refresh, so the stale id resolves to nothing: {tool_error:#}" + ); + assert!( + !serde_json::to_string(&tool_error) + .expect("serialize tool error") + .contains("staledx"), + "the refusal must not carry bytes the index never saw: {tool_error:#}" ); } @@ -4918,12 +5223,15 @@ fn transcript_reads_project_resource() { assert_eq!(content["mimeType"], "application/json"); let text = content["text"].as_str().expect("project resource text"); let project: Value = serde_json::from_str(text).expect("project resource json text"); + let root = project + .get("project_root") + .or_else(|| project.get("root")) + .and_then(Value::as_str) + .expect("project resource should include a project root field"); assert!( - project - .get("project_root") - .or_else(|| project.get("root")) - .is_some(), - "project resource should include a project root field: {project}" + codestory_workspace::same_workspace_path(Path::new(root), fixture.workspace.path()), + "project resource must describe this fixture's canonical root, not an \ + unrelated project: {project}" ); } @@ -5994,13 +6302,48 @@ fn independent_clients_serve_one_complete_generation_while_refresh_is_owned() { .expect("ground serving generation"); assert!(served_generation >= generation); + // A query-resolved symbol call is retrieval-class; the root-symbol + // resource is graph-only and hands back the stable id instead. + let pre_symbols_response = send_json( + &mut ground_client, + json!({ + "jsonrpc": "2.0", + "id": "concurrent-root-symbols-lookup", + "method": "resources/read", + "params": { + "uri": "codestory://symbols/root", + "project": fixture.workspace.path() + } + }), + ); + let pre_symbols = json_resource_content( + assert_success_envelope( + &pre_symbols_response, + json!("concurrent-root-symbols-lookup"), + ), + "codestory://symbols/root", + ); + let app_controller_id = pre_symbols + .as_array() + .into_iter() + .flatten() + .find(|symbol| { + symbol["display_name"] == json!("AppController") + || symbol["label"] == json!("AppController") + || symbol["label"] + .as_str() + .is_some_and(|label| label.starts_with("AppController ")) + }) + .and_then(|symbol| symbol["id"].as_str()) + .unwrap_or_else(|| panic!("root symbols should expose AppController: {pre_symbols}")) + .to_string(); let symbol_response = send_json( &mut ground_client, json!({ "jsonrpc": "2.0", "id": "concurrent-symbol", "method": "tools/call", - "params": {"name": "symbol", "arguments": {"query": "AppController"}} + "params": {"name": "symbol", "arguments": {"id": app_controller_id}} }), ); let symbol = assert_tool_success(&symbol_response, json!("concurrent-symbol")); @@ -6058,12 +6401,19 @@ fn two_stdio_processes_observe_only_complete_generations_during_real_refresh() { "params": {"uri": "codestory://status", "project": fixture.workspace.path()} }), ); - let old_generation = json_resource_content( + let warmup_content = json_resource_content( assert_success_envelope(&warmup_status, json!("warmup-generation")), "codestory://status", - )["index_publication"]["generation"] + ); + let old_generation = warmup_content["index_publication"]["generation"] .as_u64() .expect("old complete generation"); + let writer_lock_path = PathBuf::from( + warmup_content["storage_path"] + .as_str() + .expect("status storage_path"), + ) + .with_extension("index-writer.lock"); let mut writer_client = spawn_stdio_server(&fixture); initialize_stdio_server(&mut writer_client, "writer-initialize"); thread::sleep(Duration::from_millis(25)); @@ -6092,40 +6442,65 @@ fn two_stdio_processes_observe_only_complete_generations_during_real_refresh() { (writer_client, response) }); - let lock_path = fixture.cache_dir.path().join("local-refresh.lock"); - let lock_deadline = Instant::now() + Duration::from_secs(10); - while !lock_path.exists() { - if writer.is_finished() { - break; + // Owner-scoped barrier: probe the real index-writer lock the refresh run + // holds end to end. A failed try-lock proves the writer is inside the + // publication boundary; a reader request answered in that window proves + // reads were admitted while the writer still held it. Releasing a + // successful probe immediately is safe because the production acquire + // retries within its spawn-ghost budget. + let writer_lock_file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&writer_lock_path) + .expect("open index-writer lock"); + let mut admitted_during_writer_hold = false; + let lock_deadline = Instant::now() + Duration::from_secs(120); + while !admitted_during_writer_hold { + if writer_lock_file + .try_lock_exclusive() + .expect("probe index-writer lock") + { + writer_lock_file.unlock().expect("release probed lock"); + assert!( + !writer.is_finished(), + "writer finished without ever holding the index-writer lock" + ); + } else { + let concurrent_ground = send_json( + &mut reader_client, + json!({ + "jsonrpc": "2.0", + "id": "reader-ground-during-lock", + "method": "resources/read", + "params": { + "uri": "codestory://grounding", + "project": fixture.workspace.path() + } + }), + ); + let concurrent_ground = json_resource_content( + assert_success_envelope(&concurrent_ground, json!("reader-ground-during-lock")), + "codestory://grounding", + ); + assert!( + concurrent_ground["stats"]["file_count"] + .as_u64() + .is_some_and(|count| count == 5 || count == 101), + "concurrent resource read observed neither complete file set: {concurrent_ground}" + ); + admitted_during_writer_hold = true; } assert!( Instant::now() < lock_deadline, - "writer did not acquire the local refresh lock" + "writer never held the index-writer lock while a reader ran" ); thread::sleep(Duration::from_millis(10)); } - - let concurrent_ground = send_json( - &mut reader_client, - json!({ - "jsonrpc": "2.0", - "id": "reader-ground-during-lock", - "method": "resources/read", - "params": { - "uri": "codestory://grounding", - "project": fixture.workspace.path() - } - }), - ); - let concurrent_ground = json_resource_content( - assert_success_envelope(&concurrent_ground, json!("reader-ground-during-lock")), - "codestory://grounding", - ); assert!( - concurrent_ground["stats"]["file_count"] - .as_u64() - .is_some_and(|count| count == 5 || count == 101), - "concurrent resource read observed neither complete file set: {concurrent_ground}" + admitted_during_writer_hold, + "no reader request was admitted while the writer held the publication boundary" ); // Workspace-wide default-concurrency runs can heavily contend with the @@ -6207,43 +6582,883 @@ fn two_stdio_processes_observe_only_complete_generations_during_real_refresh() { assert_tool_success(&writer_status, json!("writer-start-refresh")); } -#[test] -fn tools_call_local_graph_refreshes_long_lived_index_after_source_mutation() { - let fixture = indexed_fixture(); - let mut server = spawn_stdio_server(&fixture); - let tools = assert_success_envelope( - &send_json( - &mut server, - json!({ - "jsonrpc": "2.0", - "id": "tool-refresh-catalog", - "method": "tools/list" - }), - ), - json!("tool-refresh-catalog"), +// Shared-cache two-process fixtures below pin the cross-process peer-writer +// contract: while one process holds the index-writer lock mid-refresh, a +// second process's activation waits on the publication boundary instead of +// failing `cache_busy`, then adopts whatever the peer published. + +/// `.index-writer.hold` marker companion to the lock path: the file +/// holds a refresh parked mid-flight while it exists (see +/// `wait_out_index_writer_hold_marker`). +fn stdio_writer_paths(storage_path: &str) -> (PathBuf, PathBuf) { + let storage = PathBuf::from(storage_path); + ( + storage.with_extension("index-writer.lock"), + storage.with_extension("index-writer-hold"), ) - .clone(); - let snippet_output_schema = tool_output_schema(&tools, "snippet") - .pointer("/oneOf/0/allOf/0") - .expect("snippet successful output schema") - .clone(); +} - let ground_before = send_json( - &mut server, - json!({ - "jsonrpc": "2.0", - "id": "tool-refresh-ground-before", - "method": "tools/call", - "params": { - "name": "ground", - "arguments": {"budget": "strict"} - } - }), - ); - let ground_before = assert_tool_success(&ground_before, json!("tool-refresh-ground-before")); - let node_count_before = ground_before - .pointer("/stats/node_count") - .and_then(Value::as_u64) +/// Spawn `index --refresh incremental` as a child process. With the +/// writer-hold marker armed it parks mid-refresh still holding the +/// index-writer lock, so tests control exactly when the peer publishes. +fn spawn_peer_index_refresh(fixture: &StdioFixture) -> Child { + let mut command = test_support::cli_command(); + command + .arg("index") + .arg("--refresh") + .arg("incremental") + .arg("--format") + .arg("json") + .arg("--project") + .arg(fixture.workspace.path()) + .arg("--cache-dir") + .arg(fixture.cache_dir.path()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + allow_explicit_cpu_embeddings(&mut command); + command.spawn().expect("spawn peer index refresh") +} + +/// Wait until some other process holds the index-writer lock: a failed probe +/// proves ownership sits elsewhere. +fn wait_for_peer_held_writer(writer_lock_path: &Path, context: &str) { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(writer_lock_path) + .expect("open index-writer lock"); + let deadline = Instant::now() + Duration::from_secs(60); + loop { + if !file.try_lock_exclusive().expect("probe index-writer lock") { + return; + } + file.unlock().expect("release probed lock"); + assert!(Instant::now() < deadline, "{context}"); + thread::sleep(Duration::from_millis(10)); + } +} + +/// Write `request` without consuming a response: interleaved +/// `notifications/progress` frames are read separately by the caller. +fn write_stdio_request(server: &mut StdioServer, request: &Value) { + writeln!(server.stdin, "{request}").expect("write request line"); + server.stdin.flush().expect("flush request line"); +} + +/// Send `request` on a background thread and stream every frame — progress +/// notifications included — through the channel. The frame matching the +/// request `id` is the response and ends the stream. +fn stream_stdio_request( + mut server: StdioServer, + request: Value, +) -> ( + std::sync::mpsc::Receiver, + thread::JoinHandle, +) { + let (tx, rx) = std::sync::mpsc::channel(); + let id = request.get("id").cloned(); + let handle = thread::spawn(move || { + write_stdio_request(&mut server, &request); + loop { + let frame = read_json(&mut server); + let is_response = frame.get("id") == id.as_ref(); + let _ = tx.send(frame); + if is_response { + break; + } + } + server + }); + (rx, handle) +} + +/// Drain `rx` until the response frame for `id` arrives, returning it along +/// with every notification frame observed before it. +fn wait_for_streamed_response( + rx: &std::sync::mpsc::Receiver, + id: &str, + deadline: Duration, +) -> (Value, Vec) { + let started = Instant::now(); + let mut notifications = Vec::new(); + loop { + let remaining = deadline.saturating_sub(started.elapsed()); + assert!( + !remaining.is_zero(), + "response for {id} did not arrive within {deadline:?}" + ); + match rx.recv_timeout(remaining) { + Ok(frame) if frame.get("id") == Some(&json!(id)) => { + return (frame, notifications); + } + Ok(frame) => notifications.push(frame), + Err(_) => panic!("response for {id} did not arrive within {deadline:?}"), + } + } +} + +/// A request armed with a progress token must surface the peer-wait stage +/// message while the activation is parked on the peer's writer lock. +fn wait_for_peer_wait_progress(rx: &std::sync::mpsc::Receiver, id: &str) -> Vec { + let deadline = Instant::now() + Duration::from_secs(60); + let mut seen = Vec::new(); + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + let frame = rx + .recv_timeout(remaining.max(Duration::from_millis(1))) + .unwrap_or_else(|_| panic!("no frames observed for {id} while waiting for the peer")); + if frame.get("id") == Some(&json!(id)) { + panic!("request {id} answered before reaching the peer-writer wait: {frame}"); + } + let is_wait_stage = frame.get("method") == Some(&json!("notifications/progress")) + && frame.pointer("/params/message").and_then(Value::as_str) + == Some("CodeStory is waiting for another session to finish indexing"); + seen.push(frame); + if is_wait_stage { + return seen; + } + assert!( + Instant::now() < deadline, + "request {id} never reported the peer-writer wait stage: {seen:?}" + ); + } +} + +/// Collect the fixture's storage path, current generation, and one stable +/// symbol id from a warm reader process before any drift. +fn warmup_reader_state(fixture: &StdioFixture) -> (StdioServer, String, u64, String) { + let mut reader = spawn_stdio_server(fixture); + let warmup_status = send_json( + &mut reader, + json!({ + "jsonrpc": "2.0", + "id": "peer-warmup-status", + "method": "resources/read", + "params": {"uri": "codestory://status", "project": fixture.workspace.path()} + }), + ); + let status = json_resource_content( + assert_success_envelope(&warmup_status, json!("peer-warmup-status")), + "codestory://status", + ); + let storage_path = status["storage_path"] + .as_str() + .expect("status storage_path") + .to_string(); + let old_generation = status["index_publication"]["generation"] + .as_u64() + .expect("warm complete generation"); + let symbols_response = send_json( + &mut reader, + json!({ + "jsonrpc": "2.0", + "id": "peer-warmup-symbols", + "method": "resources/read", + "params": { + "uri": "codestory://symbols/root", + "project": fixture.workspace.path() + } + }), + ); + let root_symbols = json_resource_content( + assert_success_envelope(&symbols_response, json!("peer-warmup-symbols")), + "codestory://symbols/root", + ); + let symbol_id = root_symbols + .as_array() + .into_iter() + .flatten() + .find(|symbol| { + symbol["display_name"] == json!("AppController") + || symbol["label"] == json!("AppController") + || symbol["label"] + .as_str() + .is_some_and(|label| label.starts_with("AppController ")) + }) + .and_then(|symbol| symbol["id"].as_str()) + .unwrap_or_else(|| panic!("root symbols should expose AppController: {root_symbols}")) + .to_string(); + (reader, storage_path, old_generation, symbol_id) +} + +/// The current complete generation as a fresh observational status read sees +/// it, so the test counts publications from shared storage. +fn observed_generation(reader: &mut StdioServer, fixture: &StdioFixture, id: &str) -> u64 { + let response = send_json( + reader, + json!({ + "jsonrpc": "2.0", + "id": id, + "method": "resources/read", + "params": {"uri": "codestory://status", "project": fixture.workspace.path()} + }), + ); + let status = json_resource_content( + assert_success_envelope(&response, json!(id)), + "codestory://status", + ); + status["index_publication"]["generation"] + .as_u64() + .expect("observed complete generation") +} + +#[test] +fn two_stdio_processes_peer_writer_wait_adopts_the_peer_publication() { + let fixture = indexed_fixture(); + let (mut reader, storage_path, old_generation, symbol_id) = warmup_reader_state(&fixture); + let (writer_lock_path, hold_marker_path) = stdio_writer_paths(&storage_path); + + fs::write( + fixture.workspace.path().join("src/peer_adopted.rs"), + "pub fn peer_adopted() -> usize { 7 } +", + ) + .expect("drift the workspace"); + fs::write(&hold_marker_path, "hold").expect("arm writer-hold marker"); + let peer = spawn_peer_index_refresh(&fixture); + wait_for_peer_held_writer( + &writer_lock_path, + "the peer refresh never held the index-writer lock", + ); + + let snippet_server = spawn_stdio_server(&fixture); + let packet_server = spawn_stdio_server(&fixture); + let (snippet_rx, snippet_thread) = stream_stdio_request( + snippet_server, + json!({ + "jsonrpc": "2.0", + "id": "peer-wait-snippet", + "method": "tools/call", + "params": { + "name": "snippet", + "arguments": {"id": symbol_id}, + "_meta": {"progressToken": "peer-snippet-progress"} + } + }), + ); + let (packet_rx, packet_thread) = stream_stdio_request( + packet_server, + json!({ + "jsonrpc": "2.0", + "id": "peer-wait-packet", + "method": "tools/call", + "params": { + "name": "packet", + "arguments": { + "question": "How does AppController open a project?", + "latency_budget_ms": 60000 + }, + "_meta": {"progressToken": "peer-packet-progress"} + } + }), + ); + + // Both calls must be parked inside the peer-writer wait — progress + // reporting proves the stage — with no response on the wire while the + // peer still holds the writer. + wait_for_peer_wait_progress(&snippet_rx, "peer-wait-snippet"); + wait_for_peer_wait_progress(&packet_rx, "peer-wait-packet"); + thread::sleep(Duration::from_millis(500)); + for (rx, id) in [ + (&snippet_rx, "peer-wait-snippet"), + (&packet_rx, "peer-wait-packet"), + ] { + while let Ok(frame) = rx.try_recv() { + assert_ne!( + frame.get("id"), + Some(&json!(id)), + "request {id} answered while the peer still held the writer: {frame}" + ); + } + } + + fs::remove_file(&hold_marker_path).expect("release the parked peer refresh"); + let peer_output = peer.wait_with_output().expect("wait for peer refresh exit"); + assert!( + peer_output.status.success(), + "peer refresh failed +stdout: +{} +stderr: +{}", + String::from_utf8_lossy(&peer_output.stdout), + String::from_utf8_lossy(&peer_output.stderr) + ); + + let (snippet_response, _) = + wait_for_streamed_response(&snippet_rx, "peer-wait-snippet", Duration::from_secs(120)); + let (packet_response, _) = + wait_for_streamed_response(&packet_rx, "peer-wait-packet", Duration::from_secs(120)); + let _snippet_server = snippet_thread.join().expect("join snippet request"); + let _packet_server = packet_thread.join().expect("join packet request"); + + // Exactly one new complete generation exists — the peer's — and both + // waiters answer from it rather than publishing again. + let published = observed_generation(&mut reader, &fixture, "peer-adopted-generation"); + assert_eq!( + published, + old_generation + 1, + "exactly one publication must occur: the peer's" + ); + assert_tool_success(&snippet_response, json!("peer-wait-snippet")); + let snippet_result = assert_success_envelope(&snippet_response, json!("peer-wait-snippet")); + assert_eq!( + snippet_result["_meta"]["codestory_publication"]["publication"]["generation"], + json!(published), + "the waiting snippet must answer from the peer's publication" + ); + assert_eq!( + snippet_result["_meta"]["codestory_publication"]["freshness"]["state"], + json!("fresh") + ); + let packet_result = assert_success_envelope(&packet_response, json!("peer-wait-packet")); + if packet_result.get("isError").and_then(Value::as_bool) == Some(true) { + // This build carries no embedded embedding model, so a full + // activation can wait on the peer, adopt its publication, and still + // terminate at dense preparation. The wire answer must be that typed + // limitation — never a lock-contention refusal — and the observed + // operation proves the peer's publication was already in place. + let error = assert_tool_error(&packet_response, json!("peer-wait-packet")); + assert_eq!( + error["cause_code"], + json!("native_model_not_embedded"), + "the waited packet must not fail on writer contention: {error}" + ); + assert_eq!( + error["operation"]["retained_core_publication"]["generation"], + json!(published), + "the packet activation must have adopted the peer's publication" + ); + } else { + assert_tool_success(&packet_response, json!("peer-wait-packet")); + assert_eq!( + packet_result["_meta"]["codestory_publication"]["publication"]["generation"], + json!(published) + ); + assert_eq!( + packet_result["_meta"]["codestory_publication"]["freshness"]["state"], + json!("fresh") + ); + } +} + +#[test] +fn two_stdio_processes_peer_writer_wait_returns_resumable_preparing() { + let fixture = indexed_fixture(); + let (mut reader, storage_path, old_generation, _symbol_id) = warmup_reader_state(&fixture); + let (writer_lock_path, hold_marker_path) = stdio_writer_paths(&storage_path); + + fs::write( + fixture.workspace.path().join("src/peer_resume.rs"), + "pub fn peer_resume() -> usize { 8 } +", + ) + .expect("drift the workspace"); + fs::write(&hold_marker_path, "hold").expect("arm writer-hold marker"); + let peer = spawn_peer_index_refresh(&fixture); + wait_for_peer_held_writer( + &writer_lock_path, + "the peer refresh never held the index-writer lock", + ); + + // A bounded packet latency budget expires while the peer still holds the + // writer: the caller gets a resumable preparing envelope plus progress. + // The budget must span the first activation slice plus one progress tick + // so the join-wait loop reports the parked stage before returning. + // send_json reads only one line, so progress frames are drained until the + // matching response id arrives. + let mut packet_server = spawn_stdio_server(&fixture); + write_stdio_request( + &mut packet_server, + &json!({ + "jsonrpc": "2.0", + "id": "peer-deadline-packet", + "method": "tools/call", + "params": { + "name": "packet", + "arguments": { + "question": "How does AppController open a project?", + "latency_budget_ms": 10000 + }, + "_meta": {"progressToken": "peer-deadline-progress"} + } + }), + ); + let deadline = Instant::now() + Duration::from_secs(60); + let mut notifications = Vec::new(); + let preparing_response = loop { + let frame = read_json(&mut packet_server); + assert!( + Instant::now() < deadline, + "the packet call never returned its preparing envelope" + ); + if frame.get("id") == Some(&json!("peer-deadline-packet")) { + break frame; + } + notifications.push(frame); + }; + let preparing = assert_success_envelope(&preparing_response, json!("peer-deadline-packet")); + let structured = preparing["structuredContent"].clone(); + assert_eq!( + structured["kind"], + json!("preparing"), + "the expired call must return the resumable preparing envelope: {preparing_response}" + ); + assert_eq!(structured["state"], json!("preparing")); + assert_eq!(structured["deadline_exceeded"], json!(true)); + assert!( + structured["resume_operation_id"].is_string() + && structured["resume_operation_attempt"].is_u64(), + "the preparing envelope must carry callable resume identity: {structured}" + ); + assert!( + notifications.iter().any(|frame| { + frame.get("method") == Some(&json!("notifications/progress")) + && frame.pointer("/params/progressToken") == Some(&json!("peer-deadline-progress")) + }), + "a progress-token request must emit progress while waiting on the peer: {notifications:?}" + ); + // The timed-out request left its activation running in the background; + // it is still parked on the peer's writer lock. + assert_eq!( + structured["operation"]["stage"], + json!("waiting_for_peer_writer"), + "the preparing envelope must describe the peer-writer wait: {structured}" + ); + + fs::remove_file(&hold_marker_path).expect("release the parked peer refresh"); + let peer_output = peer.wait_with_output().expect("wait for peer refresh exit"); + assert!( + peer_output.status.success(), + "peer refresh failed +stderr: +{}", + String::from_utf8_lossy(&peer_output.stderr) + ); + let published = observed_generation(&mut reader, &fixture, "peer-resume-generation"); + assert_eq!(published, old_generation + 1); + + // Resume on the same connection with the envelope's exact minimum_next + // arguments: the still-running shared activation adopts the peer's + // publication, finishes, and the resumed call answers from it. The + // replayed latency budget is short, so another preparing envelope is a + // legal intermediate answer. + let resume_arguments = structured["minimum_next"]["arguments"].clone(); + let deadline = Instant::now() + Duration::from_secs(120); + let mut resume_attempt = 0usize; + loop { + resume_attempt += 1; + let resume_response = send_json( + &mut packet_server, + json!({ + "jsonrpc": "2.0", + "id": format!("peer-deadline-resume-{resume_attempt}"), + "method": "tools/call", + "params": {"name": "packet", "arguments": resume_arguments} + }), + ); + let resume_result = assert_success_envelope( + &resume_response, + json!(format!("peer-deadline-resume-{resume_attempt}")), + ); + if resume_result["structuredContent"]["kind"] == json!("preparing") { + assert!( + Instant::now() < deadline, + "the resumed call never converged after the peer published" + ); + continue; + } + if resume_result.get("isError").and_then(Value::as_bool) == Some(true) { + // A build without the embedded embedding model waits, adopts the + // peer's core, then ends at dense preparation. The wire answer is + // that typed limitation — never a contention error or an unknown + // operation. + let error = assert_tool_error( + &resume_response, + json!(format!("peer-deadline-resume-{resume_attempt}")), + ); + let cause = error["cause_code"] + .as_str() + .or_else(|| error["details"]["cause_code"].as_str()); + assert_eq!( + cause, + Some("native_model_not_embedded"), + "the resumed call must not fail on writer contention: {error}" + ); + break; + } + assert_tool_success(&resume_response, json!("peer-deadline-resume")); + assert_eq!( + resume_result["_meta"]["codestory_publication"]["publication"]["generation"], + json!(published), + "the resumed call must answer from the peer's publication: {resume_response}" + ); + assert_eq!( + resume_result["_meta"]["codestory_publication"]["freshness"]["state"], + json!("fresh") + ); + break; + } +} + +#[test] +fn two_stdio_processes_cancelling_a_peer_waiter_leaves_everything_running() { + let fixture = indexed_fixture(); + let (mut reader, storage_path, old_generation, symbol_id) = warmup_reader_state(&fixture); + let (writer_lock_path, hold_marker_path) = stdio_writer_paths(&storage_path); + + fs::write( + fixture.workspace.path().join("src/peer_cancel.rs"), + "pub fn peer_cancel() -> usize { 9 } +", + ) + .expect("drift the workspace"); + fs::write(&hold_marker_path, "hold").expect("arm writer-hold marker"); + let peer = spawn_peer_index_refresh(&fixture); + wait_for_peer_held_writer( + &writer_lock_path, + "the peer refresh never held the index-writer lock", + ); + + // Park the waiter's snippet on the peer's writer lock; the progress + // frames prove which stage the request reached. + let mut waiter = spawn_stdio_server(&fixture); + write_stdio_request( + &mut waiter, + &json!({ + "jsonrpc": "2.0", + "id": "peer-cancel-snippet", + "method": "tools/call", + "params": { + "name": "snippet", + "arguments": {"id": symbol_id}, + "_meta": {"progressToken": "peer-cancel-progress"} + } + }), + ); + let progress_deadline = Instant::now() + Duration::from_secs(60); + loop { + let frame = read_json(&mut waiter); + assert_ne!( + frame.get("id"), + Some(&json!("peer-cancel-snippet")), + "the waiting request must not answer before cancellation: {frame}" + ); + if frame.pointer("/params/message").and_then(Value::as_str) + == Some("CodeStory is waiting for another session to finish indexing") + { + break; + } + assert!( + Instant::now() < progress_deadline, + "the waiter never reached the peer-writer stage" + ); + } + + // Cancelling the request frees the worker promptly without touching the + // shared activation it joined. + let cancel_sent = Instant::now(); + write_stdio_request( + &mut waiter, + &json!({ + "jsonrpc": "2.0", + "method": "notifications/cancelled", + "params": {"requestId": "peer-cancel-snippet"} + }), + ); + write_stdio_request( + &mut waiter, + &json!({ + "jsonrpc": "2.0", + "id": "peer-cancel-status", + "method": "resources/read", + "params": {"uri": "codestory://status", "project": fixture.workspace.path()} + }), + ); + let status_deadline = Instant::now() + Duration::from_secs(30); + let mut saw_cancelled_response = false; + let status_response = loop { + let frame = read_json(&mut waiter); + assert!( + Instant::now() < status_deadline, + "the cancelled request never released the connection" + ); + if frame.get("id") == Some(&json!("peer-cancel-snippet")) { + saw_cancelled_response = true; + } + if frame.get("id") == Some(&json!("peer-cancel-status")) { + break frame; + } + }; + assert!( + !saw_cancelled_response, + "a client-cancelled request id is owed no response" + ); + assert!( + cancel_sent.elapsed() < Duration::from_secs(30), + "the cancelled request must release the worker promptly" + ); + let status = json_resource_content( + assert_success_envelope(&status_response, json!("peer-cancel-status")), + "codestory://status", + ); + let assert_still_waiting = |status: &Value| { + let operation = &status["current_operation"]; + assert_eq!( + operation["stage"], + json!("waiting_for_peer_writer"), + "cancelling the request must not cancel the shared activation: {status}" + ); + assert!( + matches!( + operation["state"].as_str(), + Some("preparing" | "updating" | "working_locally") + ), + "the shared activation must still be running for other callers: {status}" + ); + }; + assert_still_waiting(&status); + + // The cancelled waiter must not have killed the shared activation: after + // a settle window it is still parked on the peer's writer lock. + std::thread::sleep(Duration::from_secs(2)); + let settled_status_response = send_json( + &mut waiter, + json!({ + "jsonrpc": "2.0", + "id": "peer-cancel-status-settled", + "method": "resources/read", + "params": {"uri": "codestory://status", "project": fixture.workspace.path()} + }), + ); + let settled_status = json_resource_content( + assert_success_envelope( + &settled_status_response, + json!("peer-cancel-status-settled"), + ), + "codestory://status", + ); + assert_still_waiting(&settled_status); + + fs::remove_file(&hold_marker_path).expect("release the parked peer refresh"); + let peer_output = peer.wait_with_output().expect("wait for peer refresh exit"); + assert!( + peer_output.status.success(), + "the peer refresh must publish normally despite the cancelled waiter" + ); + let published = observed_generation(&mut reader, &fixture, "peer-cancel-generation"); + assert_eq!(published, old_generation + 1); + + let retry_response = send_json( + &mut waiter, + json!({ + "jsonrpc": "2.0", + "id": "peer-cancel-retry", + "method": "tools/call", + "params": {"name": "snippet", "arguments": {"id": symbol_id}} + }), + ); + let retried = assert_tool_success(&retry_response, json!("peer-cancel-retry")); + let retry_result = assert_success_envelope(&retry_response, json!("peer-cancel-retry")); + assert_eq!( + retry_result["_meta"]["codestory_publication"]["publication"]["generation"], + json!(published), + "a later call must answer from the peer publication: {retried:?}" + ); +} + +#[test] +fn two_stdio_processes_peer_writer_death_lets_the_waiter_publish() { + let fixture = indexed_fixture(); + let (mut reader, storage_path, old_generation, symbol_id) = warmup_reader_state(&fixture); + let (writer_lock_path, hold_marker_path) = stdio_writer_paths(&storage_path); + + fs::write( + fixture.workspace.path().join("src/peer_dead.rs"), + "pub fn peer_dead() -> usize { 10 } +", + ) + .expect("drift the workspace"); + fs::write(&hold_marker_path, "hold").expect("arm writer-hold marker"); + let mut peer = spawn_peer_index_refresh(&fixture); + wait_for_peer_held_writer( + &writer_lock_path, + "the peer refresh never held the index-writer lock", + ); + + let snippet_server = spawn_stdio_server(&fixture); + let (snippet_rx, snippet_thread) = stream_stdio_request( + snippet_server, + json!({ + "jsonrpc": "2.0", + "id": "peer-death-snippet", + "method": "tools/call", + "params": { + "name": "snippet", + "arguments": {"id": symbol_id}, + "_meta": {"progressToken": "peer-death-progress"} + } + }), + ); + wait_for_peer_wait_progress(&snippet_rx, "peer-death-snippet"); + + // Kill the owner without publishing: the OS releases the writer lock and + // the parked waiter takes over — where the still-armed marker parks it + // again, mid-refresh, holding the lock this time. + peer.kill().expect("kill the peer writer"); + let _ = peer.wait().expect("reap the peer writer"); + wait_for_peer_held_writer( + &writer_lock_path, + "the waiter never took over the index-writer lock after the peer died", + ); + + // The previous publication stays readable while the waiter owns the + // writer: a graph-only read is answered from it and labelled historical. + let mut graph_only = spawn_stdio_server(&fixture); + let ground_response = send_json( + &mut graph_only, + json!({ + "jsonrpc": "2.0", + "id": "peer-death-ground", + "method": "tools/call", + "params": {"name": "ground", "arguments": {"budget": "strict"}} + }), + ); + let ground = assert_tool_success(&ground_response, json!("peer-death-ground")); + assert!(ground["stats"]["file_count"].as_u64().is_some()); + let ground_result = assert_success_envelope(&ground_response, json!("peer-death-ground")); + assert_eq!( + ground_result["_meta"]["codestory_publication"]["publication"]["generation"], + json!(old_generation), + "the retained publication must stay readable while the waiter writes" + ); + assert_eq!( + ground_result["_meta"]["codestory_publication"]["freshness"]["state"], + json!("historical") + ); + + fs::remove_file(&hold_marker_path).expect("release the parked waiter"); + let (snippet_response, _) = + wait_for_streamed_response(&snippet_rx, "peer-death-snippet", Duration::from_secs(120)); + let _snippet_server = snippet_thread.join().expect("join snippet request"); + assert_tool_success(&snippet_response, json!("peer-death-snippet")); + let snippet_result = assert_success_envelope(&snippet_response, json!("peer-death-snippet")); + assert_eq!( + snippet_result["_meta"]["codestory_publication"]["freshness"]["state"], + json!("fresh") + ); + + // The dead peer published nothing; the waiter's own refresh is the only + // new complete generation. + let published = observed_generation(&mut reader, &fixture, "peer-death-generation"); + assert_eq!( + published, + old_generation + 1, + "the waiter performs exactly one refresh after the peer dies" + ); + assert_eq!( + snippet_result["_meta"]["codestory_publication"]["publication"]["generation"], + json!(published) + ); +} + +#[test] +fn two_stdio_processes_graph_only_read_during_peer_refresh_is_historical() { + let fixture = indexed_fixture(); + let (mut reader, storage_path, old_generation, _symbol_id) = warmup_reader_state(&fixture); + let (writer_lock_path, hold_marker_path) = stdio_writer_paths(&storage_path); + + fs::write( + fixture.workspace.path().join("src/peer_historical.rs"), + "pub fn peer_historical() -> usize { 11 } +", + ) + .expect("drift the workspace"); + fs::write(&hold_marker_path, "hold").expect("arm writer-hold marker"); + let peer = spawn_peer_index_refresh(&fixture); + wait_for_peer_held_writer( + &writer_lock_path, + "the peer refresh never held the index-writer lock", + ); + + let mut graph_only = spawn_stdio_server(&fixture); + let ground_response = send_json( + &mut graph_only, + json!({ + "jsonrpc": "2.0", + "id": "peer-historical-ground", + "method": "tools/call", + "params": {"name": "ground", "arguments": {"budget": "strict"}} + }), + ); + let ground = assert_tool_success(&ground_response, json!("peer-historical-ground")); + assert!(ground["stats"]["file_count"].as_u64().is_some()); + let ground_result = assert_success_envelope(&ground_response, json!("peer-historical-ground")); + let publication_meta = &ground_result["_meta"]["codestory_publication"]; + assert_eq!( + publication_meta["freshness"]["state"], + json!("historical"), + "the graph-only read must be labelled historical while a peer writes" + ); + assert_eq!( + publication_meta["freshness"]["reason"], + json!("peer_writer"), + "the peer-writer wait stage must identify the retained answer: {publication_meta}" + ); + assert_eq!( + publication_meta["publication"]["generation"], + json!(old_generation), + "the historical answer must come from the retained publication" + ); + + fs::remove_file(&hold_marker_path).expect("release the parked peer refresh"); + let peer_output = peer.wait_with_output().expect("wait for peer refresh exit"); + assert!(peer_output.status.success()); + assert_eq!( + observed_generation(&mut reader, &fixture, "peer-historical-generation"), + old_generation + 1 + ); +} + +#[test] +fn tools_call_local_graph_refreshes_long_lived_index_after_source_mutation() { + let fixture = indexed_fixture(); + let mut server = spawn_stdio_server(&fixture); + let tools = assert_success_envelope( + &send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "tool-refresh-catalog", + "method": "tools/list" + }), + ), + json!("tool-refresh-catalog"), + ) + .clone(); + let snippet_output_schema = tool_output_schema(&tools, "snippet") + .pointer("/oneOf/0/allOf/0") + .expect("snippet successful output schema") + .clone(); + + let ground_before = send_json( + &mut server, + json!({ + "jsonrpc": "2.0", + "id": "tool-refresh-ground-before", + "method": "tools/call", + "params": { + "name": "ground", + "arguments": {"budget": "strict"} + } + }), + ); + let ground_before = assert_tool_success(&ground_before, json!("tool-refresh-ground-before")); + let node_count_before = ground_before + .pointer("/stats/node_count") + .and_then(Value::as_u64) .expect("ground before mutation node count"); let files_before = send_json( @@ -6347,28 +7562,29 @@ fn tools_call_local_graph_refreshes_long_lived_index_after_source_mutation() { "ground should serve refreshed graph stats after mutation; before={node_count_before}, after={node_count_after}, snapshot={ground_after}" ); - let symbol_response = send_json( - &mut server, - json!({ - "jsonrpc": "2.0", - "id": "tool-refresh-symbol", - "method": "tools/call", - "params": { - "name": "symbol", - "arguments": {"query": "stdio_tool_added_after_mutation"} - } - }), - ); - let symbol = assert_tool_success(&symbol_response, json!("tool-refresh-symbol")); - let node_id = symbol - .pointer("/node/id") - .and_then(Value::as_str) - .or_else(|| { - symbol - .pointer("/resolution/resolved/node_id") - .and_then(Value::as_str) + // Query-resolution is retrieval-class and unavailable without broad + // retrieval; the refreshed ground response already lists the new symbol + // with its stable id, which is the graph-only way to select it. + let node_id = ground_after["root_symbols"] + .as_array() + .into_iter() + .flatten() + .chain( + ground_after["files"] + .as_array() + .into_iter() + .flatten() + .flat_map(|file| file["symbols"].as_array().into_iter().flatten()), + ) + .find(|symbol| { + symbol["label"] + .as_str() + .is_some_and(|label| label.starts_with("stdio_tool_added_after_mutation")) + }) + .and_then(|symbol| symbol["id"].as_str()) + .unwrap_or_else(|| { + panic!("refreshed ground should expose the post-mutation function: {ground_after:#}") }) - .unwrap_or_else(|| panic!("symbol should resolve the post-mutation function: {symbol}")) .to_string(); for (tool, id) in [ @@ -6468,10 +7684,10 @@ fn tools_call_local_graph_refreshes_long_lived_index_after_source_mutation() { ); assert!( matches!( - search_error.pointer("/code").and_then(Value::as_str), + tool_result_code(&search_response).as_deref(), Some("codestory_preparing" | "codestory_unavailable") ), - "broad search should use the normal readiness response after local graph refresh: {search_response}" + "broad search should use the normal readiness response after local graph refresh: {search_error}" ); } @@ -7363,29 +8579,40 @@ fn mcp_graph_caller_scope_hides_stored_test_call_until_explicitly_included() { let mut server = spawn_stdio_server(&fixture); initialize_stdio_server(&mut server, "init-scope"); - let test_entry = call_graph_tool( + // Query-resolution is retrieval-class; ground is graph-only and lists the + // fixture functions with their stable ids. + let grounding = call_graph_tool( &mut server, - "symbol-test-entry", - "symbol", - json!({ - "project": fixture.workspace.path(), - "query": "test_entry" - }), - ); - let test_id = test_entry["node"]["id"] - .as_str() - .expect("test_entry id") - .to_string(); - let leaf = call_graph_tool( - &mut server, - "symbol-leaf", - "symbol", + "scope-ground", + "ground", json!({ "project": fixture.workspace.path(), - "query": "leaf" + "budget": "strict" }), ); - let leaf_id = leaf["node"]["id"].as_str().expect("leaf id").to_string(); + let scope_symbol_id = |name: &str| { + grounding["root_symbols"] + .as_array() + .into_iter() + .flatten() + .chain( + grounding["files"] + .as_array() + .into_iter() + .flatten() + .flat_map(|file| file["symbols"].as_array().into_iter().flatten()), + ) + .find(|symbol| { + symbol["label"] + .as_str() + .is_some_and(|label| label.starts_with(name)) + }) + .and_then(|symbol| symbol["id"].as_str()) + .unwrap_or_else(|| panic!("ground should expose {name}: {grounding:#}")) + .to_string() + }; + let test_id = scope_symbol_id("test_entry"); + let leaf_id = scope_symbol_id("leaf"); for (name, extra) in [ ( diff --git a/crates/codestory-cli/tests/test_support/mod.rs b/crates/codestory-cli/tests/test_support/mod.rs index 2bc6e5ec2..89e799940 100644 --- a/crates/codestory-cli/tests/test_support/mod.rs +++ b/crates/codestory-cli/tests/test_support/mod.rs @@ -82,3 +82,51 @@ fn thread_name() -> String { fn install_id() -> String { format!("integration-{}-{}", std::process::id(), thread_name()) } + +/// Downgrade the published core's durable `user_version` in place, emulating a +/// core written by an older release. Returns the generation database path so +/// callers can prove the image is untouched afterward. The same fixture shape +/// backs the I2 #3c probe-stage case, so keep it a schema downgrade of a real +/// published generation rather than a hand-built database. +pub fn set_active_core_schema_version(cache_dir: &std::path::Path, version: u32) -> PathBuf { + let pointer: serde_json::Value = serde_json::from_slice( + &std::fs::read(cache_dir.join("core/publication.json")).expect("committed core pointer"), + ) + .expect("core pointer json"); + let generation = pointer["active"]["generation_id"] + .as_str() + .expect("active generation id") + .to_string(); + let database = cache_dir + .join("core") + .join("generations") + .join(generation) + .join("codestory.db"); + let metadata = std::fs::metadata(&database).expect("generation metadata"); + let mut permissions = metadata.permissions(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + permissions.set_mode(permissions.mode() | 0o200); + } + #[cfg(not(unix))] + { + permissions.set_readonly(false); + } + std::fs::set_permissions(&database, permissions).expect("unseal generation image"); + { + let connection = rusqlite::Connection::open(&database).expect("open generation image"); + connection + .pragma_update(None, "user_version", version) + .expect("downgrade durable schema version"); + connection + .execute_batch("PRAGMA wal_checkpoint(TRUNCATE);") + .expect("checkpoint schema downgrade"); + } + for suffix in ["-wal", "-shm", "-journal"] { + let mut sidecar = database.as_os_str().to_owned(); + sidecar.push(suffix); + let _ = std::fs::remove_file(PathBuf::from(sidecar)); + } + database +} diff --git a/crates/codestory-contracts/Cargo.toml b/crates/codestory-contracts/Cargo.toml index 7655ac9c6..b1d2192b9 100644 --- a/crates/codestory-contracts/Cargo.toml +++ b/crates/codestory-contracts/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-contracts" -version = "0.17.6" +version = "0.17.7" edition = "2024" [dependencies] @@ -16,5 +16,11 @@ uuid = { workspace = true } serde_with = { workspace = true } specta = { workspace = true } +[target.'cfg(windows)'.dependencies] +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_Storage_FileSystem", +] } + [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/codestory-contracts/src/api.rs b/crates/codestory-contracts/src/api.rs index d2eb07284..ce1b1483b 100644 --- a/crates/codestory-contracts/src/api.rs +++ b/crates/codestory-contracts/src/api.rs @@ -78,11 +78,13 @@ pub use dto::{ pub use errors::{ ApiError, ApiErrorDetails, COMMAND_FAILURE_SCHEMA_VERSION, CommandFailureEnvelope, DiskSpacePressureDto, EmbeddingCapacityPressureDto, EmbeddingRetryStateDto, + PeerWriterDiagnosticsDto, PeerWriterHolderDto, PeerWriterHolderRecordDto, }; pub use events::{ AppEventPayload, ArtifactCacheAccessTimings, ArtifactCachePolicyDto, CorePromotionTimings, - DatabaseSnapshotCopyTimings, FullRefreshWallTimings, IncrementalCoreWallTimings, - IncrementalPlanProbeOutcomeDto, IncrementalPlanProbeTimings, IncrementalScheduledPathActionDto, + CoreRetentionOutcomeDto, DatabaseSnapshotCopyTimings, FullRefreshWallTimings, + IncrementalCoreWallTimings, IncrementalPlanProbeOutcomeDto, IncrementalPlanProbeTimings, + IncrementalProbeUnavailableStageDto, IncrementalScheduledPathActionDto, IncrementalScheduledPathDto, IncrementalScheduledPathReasonDto, IndexingPhaseTimings, ProjectionPersistenceFamilyTimings, ProjectionPersistenceTimings, PromotedValidationDto, }; diff --git a/crates/codestory-contracts/src/api/dto.rs b/crates/codestory-contracts/src/api/dto.rs index 0b3d770ff..b3f4a5442 100644 --- a/crates/codestory-contracts/src/api/dto.rs +++ b/crates/codestory-contracts/src/api/dto.rs @@ -3331,6 +3331,59 @@ mod packet_tests { assert_eq!(decoded, evidence); assert!(decoded.validation_errors().is_empty()); + + // A derive round-trip cannot catch a renamed or respelled field, so the + // persisted wire shape is pinned key by key. + let value = serde_json::to_value(&evidence).expect("evidence value"); + assert_eq!( + value, + serde_json::json!({ + "schema_version": EMBEDDING_VECTOR_PRODUCER_EVIDENCE_VERSION, + "producer": { + "name": "codestory-llama-sys", + "version": "1.2.3", + }, + "model": { + "model_id": "model-v1", + "model_sha256": "a".repeat(64), + "model_size_bytes": 1024, + "tokenizer_sha256": "b".repeat(64), + "config_sha256": "c".repeat(64), + }, + "semantics": { + "dimension": 384, + "query_prefix": "query: ", + "document_prefix": "passage: ", + "pooling": "mean", + "normalization": "l2", + "element_type": "f32", + "vector_schema_version": 2, + }, + "engine": { + "engine": "llama.cpp", + "engine_build_id": "build-v1", + "backend": "metal", + "device_id": "gpu-0", + "device_class": "apple-gpu", + "accelerator_kind": "metal", + }, + "execution": { + "eligibility": "eligible", + "observed_state": "smoke_passed", + "observation_source": "runtime_probe", + "smoke_elapsed_ms": 8, + "observed_at_epoch_ms": 123, + }, + "publication": { + "core_generation_id": "core-1", + "core_run_id": "run-1", + "retrieval_generation": "retrieval-1", + "retrieval_input_hash": "d".repeat(64), + "semantic_generation": "semantic-1", + }, + }), + "the persisted producer-evidence wire shape is a durable contract" + ); } #[test] @@ -3500,6 +3553,56 @@ mod packet_tests { encoded ); } + + // Pin the literal `kind` tags and field names the derive round-trip + // cannot catch. `exact_path`, `free_query`, and `qualified_symbol` are + // already pinned on the wire by the CLI stdio tests + // (codestory-cli/src/stdio_arguments.rs and stdio_catalog.rs); the + // remaining three variants are spelled out here. + assert_eq!( + serde_json::to_value(PacketProbeDto::SymbolId { id: "42".into() }) + .expect("serialize symbol_id probe"), + serde_json::json!({"kind": "symbol_id", "id": "42"}) + ); + assert_eq!( + serde_json::to_value(PacketProbeDto::FileSymbol { + path: "src/lib.rs".into(), + symbol: "AppController".into(), + }) + .expect("serialize file_symbol probe"), + serde_json::json!({ + "kind": "file_symbol", + "path": "src/lib.rs", + "symbol": "AppController", + }) + ); + assert_eq!( + serde_json::to_value(PacketProbeDto::Continuation { + contract_version: PACKET_PROBE_CONTRACT_VERSION, + project_id: "project-v3".into(), + core_generation_id: "core-generation".into(), + retrieval_generation: Some("retrieval-generation".into()), + selector: PacketContinuationSelectorV1 { + stable_identity: "node:42".into(), + path: None, + symbol_id: Some("42".into()), + reason: crate::compilation::PacketStructuralGapReasonV1::DisconnectedSeed, + }, + }) + .expect("serialize continuation probe"), + serde_json::json!({ + "kind": "continuation", + "contract_version": 1, + "project_id": "project-v3", + "core_generation_id": "core-generation", + "retrieval_generation": "retrieval-generation", + "selector": { + "stable_identity": "node:42", + "symbol_id": "42", + "reason": "disconnected_seed", + }, + }) + ); } #[test] diff --git a/crates/codestory-contracts/src/api/errors.rs b/crates/codestory-contracts/src/api/errors.rs index d87041aec..3e1464df7 100644 --- a/crates/codestory-contracts/src/api/errors.rs +++ b/crates/codestory-contracts/src/api/errors.rs @@ -35,6 +35,46 @@ pub struct ApiErrorDetails { pub disk_space: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub coverage_gaps: Vec, + /// Present on `peer_writer_active`: the recorded owner of the contended + /// writer lock, or `unknown` when no owner record could be read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub peer_writer: Option, +} + +/// Diagnostics attached to a writer-scope `lock_wait_timeout`. The holder is +/// the owner record the locking peer published beside the lock file, or the +/// string `unknown` when no record could be read. `next_action` always tells +/// the operator to wait for or stop that process and never delete the lock. +#[derive(Debug, Clone, Serialize, Deserialize, Type, PartialEq, Eq)] +pub struct PeerWriterDiagnosticsDto { + pub holder: PeerWriterHolderDto, + pub next_action: String, +} + +/// The recorded owner of a contended writer lock. `Unknown` serializes as the +/// string `unknown` so a missing owner record is distinguishable from a peer +/// whose diagnostics were never published. +#[derive(Debug, Clone, Serialize, Deserialize, Type, PartialEq, Eq)] +#[serde(untagged)] +pub enum PeerWriterHolderDto { + Recorded(PeerWriterHolderRecordDto), + Unknown(String), +} + +/// The observed fields of a live or stale lock owner record. +#[derive(Debug, Clone, Serialize, Deserialize, Type, PartialEq, Eq)] +pub struct PeerWriterHolderRecordDto { + pub pid: u32, + pub operation: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub project_id: Option, + /// `acquired_at_epoch_ms` from the owner record: when the holder took the + /// lock. + pub since_epoch_ms: i64, + /// `true` only when the recorded PID still exists and its process start + /// identity matches the record, so a stale record from a dead process + /// reads `alive: false`. + pub alive: bool, } #[derive(Debug, Clone, Serialize, Deserialize, Type, PartialEq, Eq)] @@ -111,6 +151,7 @@ impl ApiErrorDetails { embedding_retry: None, disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, } } @@ -128,6 +169,7 @@ impl ApiErrorDetails { embedding_retry: None, disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, } } @@ -158,6 +200,7 @@ impl ApiErrorDetails { embedding_retry: None, disk_space: None, coverage_gaps, + peer_writer: None, } } } @@ -223,6 +266,7 @@ impl ApiError { retry_condition: "after_space_available".into(), }), coverage_gaps: Vec::new(), + peer_writer: None, }, ) } @@ -276,6 +320,7 @@ impl ApiError { }), disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, }, ) } @@ -300,6 +345,7 @@ impl ApiError { embedding_retry: Some(retry), disk_space: None, coverage_gaps: Vec::new(), + peer_writer: None, }, ) } @@ -353,6 +399,22 @@ mod tests { assert_eq!(decoded, envelope); assert_eq!(decoded.schema_version, COMMAND_FAILURE_SCHEMA_VERSION); + + // The shared CLI error envelope is a wire contract; pin the literal + // spellings a derive round-trip cannot catch. + let value: serde_json::Value = + serde_json::from_str(&json).expect("parse envelope as value"); + assert_eq!( + value, + serde_json::json!({ + "schema_version": COMMAND_FAILURE_SCHEMA_VERSION, + "error": { + "code": "invalid_argument", + "message": "bad input", + }, + "context": {"argument": "--format"}, + }) + ); } #[test] diff --git a/crates/codestory-contracts/src/api/events.rs b/crates/codestory-contracts/src/api/events.rs index a51fe3117..51d94be65 100644 --- a/crates/codestory-contracts/src/api/events.rs +++ b/crates/codestory-contracts/src/api/events.rs @@ -181,23 +181,75 @@ pub enum IncrementalPlanProbeOutcomeDto { ProbeUnavailable, } +/// The stage where an incremental plan probe failed before it could compute +/// the refresh plan. +/// +/// `ProbeUnavailable` reports always carry the stage so a `null` plan count is +/// attributed to the step that never produced it instead of reading as a +/// measured zero. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Type)] +#[serde(rename_all = "snake_case")] +pub enum IncrementalProbeUnavailableStageDto { + /// The published core could not be opened for freshness observation. + OpenCore, + /// No complete index publication could be read from the core. + Publication, + /// The workspace manifest could not be built. + WorkspaceManifest, + /// Stored refresh inputs could not be read. + RefreshInputs, + /// The refresh plan could not be built against the source policy. + Policy, + /// Stored file-coverage diagnostics could not be read. + Coverage, + /// Stored source-policy exclusions could not be read. + Exclusions, + /// The dense-anchor publication manifest could not be read. + DenseAnchor, + /// The symbol-document contract check could not be evaluated. + DocContract, + /// The search index path for the publication could not be resolved. + SearchIndexLocation, + /// The publication generation id was not a usable identity. + GenerationId, + /// Admitted sources could not all be sealed. + SourceSeals, +} + /// Work an incremental refresh avoided by proving its plan was empty. /// /// `live_database_file_bytes` is the on-disk size of the published core /// database file (not its SQLite logical image, and not including WAL). The /// skipped-copy counters are zero on every run that proceeded. +/// `files_to_index`/`files_to_remove` are `null` when the probe never reached +/// the stage that computes them; a measured empty plan reports `0`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Type)] pub struct IncrementalPlanProbeTimings { pub outcome: IncrementalPlanProbeOutcomeDto, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub probe_unavailable_stage: Option, pub probe_ms: u32, - pub files_to_index: u32, - pub files_to_remove: u32, + pub files_to_index: Option, + pub files_to_remove: Option, pub live_database_file_bytes: u64, pub skipped_database_copies: u32, pub skipped_database_copy_bytes: u64, pub skipped_search_state_rebuild: bool, } +/// What the post-publication core-retention pass of one run observed. +/// +/// `pruning_suppressed` marks a pass that was fenced off or stopped before it +/// could prune; `reason` names the stable cause (for example +/// `fenced_by_active_retrieval_publication`) when suppression was external. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Type)] +pub struct CoreRetentionOutcomeDto { + pub reclaimed_images: u32, + pub pruning_suppressed: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, Type)] #[serde(rename_all = "snake_case")] pub enum ArtifactCachePolicyDto { @@ -406,6 +458,8 @@ pub struct IndexingPhaseTimings { #[serde(default, skip_serializing_if = "Option::is_none")] pub incremental_plan_probe: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub core_retention: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub incremental_coverage_validation_ms: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub incremental_proof_projection_ms: Option, @@ -632,6 +686,7 @@ mod tests { staged_snapshot_copy: None, core_promotion: None, incremental_plan_probe: None, + core_retention: None, incremental_coverage_validation_ms: None, incremental_proof_projection_ms: None, incremental_semantic_scope_ms: None, @@ -1049,9 +1104,10 @@ mod tests { fn test_indexing_phase_timings_round_trips_incremental_plan_probe() { let probe = IncrementalPlanProbeTimings { outcome: IncrementalPlanProbeOutcomeDto::ShortCircuited, + probe_unavailable_stage: None, probe_ms: 7, - files_to_index: 0, - files_to_remove: 0, + files_to_index: Some(0), + files_to_remove: Some(0), live_database_file_bytes: 4_096, skipped_database_copies: 3, skipped_database_copy_bytes: 12_288, diff --git a/crates/codestory-contracts/src/call_path_public.rs b/crates/codestory-contracts/src/call_path_public.rs index ead6993ff..4948bbac0 100644 --- a/crates/codestory-contracts/src/call_path_public.rs +++ b/crates/codestory-contracts/src/call_path_public.rs @@ -266,7 +266,7 @@ fn refutation_schema() -> Value { }) } -fn gap_schema() -> Value { +fn gap_variants() -> Vec { let selector_gap = |kind| { closed_object_schema(vec![ ("kind", enum_schema(&[kind])), @@ -285,23 +285,26 @@ fn gap_schema() -> Value { ), ]) }; - json!({ - "type":"object", - "oneOf":[ - closed_object_schema(vec![("kind", enum_schema(&["unclassified_source_text"]))]), - closed_object_schema(vec![ - ("kind", enum_schema(&["unresolved_material_clause"])), - ("clause_id", string_schema()), - ("reason", enum_schema(&[ + vec![ + closed_object_schema(vec![("kind", enum_schema(&["unclassified_source_text"]))]), + closed_object_schema(vec![ + ("kind", enum_schema(&["unresolved_material_clause"])), + ("clause_id", string_schema()), + ( + "reason", + enum_schema(&[ "missing_selector_resolution", "ambiguous_selector_resolution", "unsupported_interpretation", - ])), - ]), - closed_object_schema(vec![ - ("kind", enum_schema(&["material_token_misclassified"])), - ("clause_id", string_schema()), - ("guard_families", json!({ + ]), + ), + ]), + closed_object_schema(vec![ + ("kind", enum_schema(&["material_token_misclassified"])), + ("clause_id", string_schema()), + ( + "guard_families", + json!({ "type":"array", "items":enum_schema(&[ "quoted_or_backticked_identifier", @@ -316,26 +319,42 @@ fn gap_schema() -> Value { "minItems":1, "maxItems":8, "uniqueItems":true, - })), - ]), - selector_gap("selector_missing"), - selector_gap("selector_ambiguous"), - selector_gap("non_callable_selector"), - step_gap("direct_call_missing"), - step_gap("recursive_call_not_representable"), - step_gap("source_window_too_large"), - step_gap("invalid_utf8"), - step_gap("source_line_out_of_range"), - step_gap("edge_containment_unproven"), - step_gap("missing_direct_call_receipt"), - step_gap("receipt_or_edge_already_used"), - step_gap("projection_exclusion_conflicts_with_required_receipt"), - closed_object_schema(vec![("kind", enum_schema(&["kernel_search_budget_exceeded"]))]) - ] + }), + ), + ]), + selector_gap("selector_missing"), + selector_gap("selector_ambiguous"), + selector_gap("non_callable_selector"), + step_gap("direct_call_missing"), + step_gap("recursive_call_not_representable"), + step_gap("source_window_too_large"), + step_gap("invalid_utf8"), + step_gap("source_line_out_of_range"), + step_gap("edge_containment_unproven"), + step_gap("missing_direct_call_receipt"), + step_gap("receipt_or_edge_already_used"), + step_gap("projection_exclusion_conflicts_with_required_receipt"), + closed_object_schema(vec![( + "kind", + enum_schema(&["kernel_search_budget_exceeded"]), + )]), + ] +} + +fn gap_schema() -> Value { + json!({ + "type":"object", + "oneOf":gap_variants() }) } +fn output_budget_exceeded_gap_schema() -> Value { + closed_object_schema(vec![("kind", enum_schema(&["output_budget_exceeded"]))]) +} + fn budget_disposition_schema() -> Value { + let mut gap_kinds = gap_variants(); + gap_kinds.push(output_budget_exceeded_gap_schema()); closed_object_schema(vec![ ("kind", enum_schema(&["unknown"])), ("contract_digest", sha256_schema()), @@ -343,9 +362,10 @@ fn budget_disposition_schema() -> Value { "gaps", json!({ "type":"array", - "items":closed_object_schema(vec![("kind", enum_schema(&["output_budget_exceeded"]))]), + "items":{"oneOf":gap_kinds}, "minItems":1, - "maxItems":1 + "maxItems":257, + "uniqueItems":true }), ), ]) diff --git a/crates/codestory-contracts/src/core_publication.rs b/crates/codestory-contracts/src/core_publication.rs index 7906da2e8..1b454dde0 100644 --- a/crates/codestory-contracts/src/core_publication.rs +++ b/crates/codestory-contracts/src/core_publication.rs @@ -65,5 +65,31 @@ mod tests { assert_eq!(decoded, pointer); assert_ne!(decoded.active, decoded.rollback.expect("rollback")); + + // This pointer is a persisted on-disk contract, so its member spellings + // are pinned literally. `active` is additionally covered by the store + // tamper test indexing value["active"] (codestory-store + // core_generation.rs); `rollback` and `receipt_digest` are pinned here. + let value: serde_json::Value = + serde_json::from_slice(&encoded).expect("parse pointer as value"); + assert_eq!( + value, + serde_json::json!({ + "schema_version": CORE_PUBLICATION_POINTER_SCHEMA_VERSION, + "active": { + "generation_id": "generation-2", + "run_id": "run-2", + "logical_bytes": 8_192, + "published_at_epoch_ms": 2, + }, + "rollback": { + "generation_id": "generation-1", + "run_id": "run-1", + "logical_bytes": 4_096, + "published_at_epoch_ms": 1, + }, + "receipt_digest": "a".repeat(64), + }) + ); } } diff --git a/crates/codestory-contracts/src/validation_receipts.rs b/crates/codestory-contracts/src/validation_receipts.rs index 3ce62d641..ac66a75db 100644 --- a/crates/codestory-contracts/src/validation_receipts.rs +++ b/crates/codestory-contracts/src/validation_receipts.rs @@ -9,24 +9,28 @@ //! SQLite sidecar all break the seal, so a generation damaged after its first //! validation re-validates instead of hiding behind the earlier verdict. //! -//! **A seal is only as strong as the metadata the platform reports, and one -//! shipped platform reports less.** The paragraph above holds in full on Unix, -//! where `std::fs` exposes a device/inode pair and an inode-change instant. -//! Windows exposes neither through `std::fs`, so a seal taken there compares -//! presence, length, the creation and modification instants, and the read-only -//! bit — nothing that records *that the bytes were rewritten*. A writer that -//! rewrites an artifact in place without changing its length and then restores -//! the modification time produces an observation identical to the sealed one, -//! and the receipt answers for bytes it never read. The same is true of a -//! replacement whose length, creation, and modification instants all match. -//! [`SealFidelity`] names which of the two a given observation is, and -//! [`ArtifactSeal::fidelity`] reports it. This is a stated limit of the +//! **A seal is only as strong as the metadata the platform reports.** The +//! paragraph above holds in full on Unix, where `std::fs` exposes a +//! device/inode pair and an inode-change instant, and on Windows, where the +//! observation opens the file briefly for a volume-serial/file-index pair and +//! the NTFS ChangeTime — the instant a same-length in-place rewrite cannot +//! hide behind a restored modification time. Where the platform reports none +//! of that — a Windows file whose handle query fails or whose filesystem does +//! not report a change instant, or any platform without either mechanism — a +//! seal compares presence, length, the creation and modification instants, and +//! the read-only bit: nothing that records *that the bytes were rewritten*. A +//! writer that rewrites an artifact in place without changing its length and +//! then restores the modification time produces an observation identical to +//! the sealed one, and the receipt answers for bytes it never read. The same +//! is true of a replacement whose length, creation, and modification instants +//! all match. [`SealFidelity`] names which of the two a given observation is, +//! and [`ArtifactSeal::fidelity`] reports it. This is a stated limit of the //! receipt, not an accident of it: on a -//! [`SealFidelity::TimestampsOnly`] platform a receipt proves the artifact was -//! not casually touched; it does not prove the bytes are the ones the +//! [`SealFidelity::TimestampsOnly`] observation a receipt proves the artifact +//! was not casually touched; it does not prove the bytes are the ones the //! validation read. Nothing that must detect deliberate corruption may rest on //! a receipt alone there. Concretely, a consumer whose verdict is receipted -//! carries the limit forward: on a timestamps-only platform an artifact +//! carries the limit forward: under a timestamps-only observation an artifact //! rewritten in place at the same length, with its modification time restored, //! keeps answering with the verdict this process already sealed for it. What //! bounds that is the receipt's process-local lifetime, not the seal — the next @@ -112,19 +116,22 @@ pub enum SealFidelity { /// A rewrite in place breaks the seal even when the writer keeps the length /// and restores the modification time afterwards, and a replacement breaks /// it even when the replacement's bytes and timestamps match. Unix reports - /// this. + /// this through `std::fs`; Windows reports it through a bounded handle + /// query for the volume serial number, the file index, and the NTFS + /// ChangeTime. InodeChangeTracked, /// The observation carries only presence, length, the creation and /// modification instants, and the read-only bit. /// - /// Windows is this platform: `std::fs` reports no device/inode pair and no - /// inode-change instant there, so nothing in the seal records that an - /// artifact's bytes were rewritten. A same-length rewrite in place that - /// restores the modification time is indistinguishable from the sealed - /// observation, and so is a replacement that matches every field. The - /// residual guarantee is real but narrower: a change in presence, length, - /// creation instant, modification instant, or the read-only bit still - /// breaks the seal. + /// This is what remains when the platform reports no native file identity: + /// any platform without a device/inode pair and an inode-change instant, + /// including a Windows file whose handle query fails or whose filesystem + /// cannot report one, so nothing in the seal records that an artifact's + /// bytes were rewritten. A same-length rewrite in place that restores the + /// modification time is indistinguishable from the sealed observation, and + /// so is a replacement that matches every field. The residual guarantee is + /// real but narrower: a change in presence, length, creation instant, + /// modification instant, or the read-only bit still breaks the seal. TimestampsOnly, } @@ -144,12 +151,15 @@ enum SealPresence { len: u64, modified_nanos: i128, created_nanos: i128, - /// Unix device id, `0` where the platform does not report one. + /// Native device identity: the Unix device id or the Windows volume + /// serial number; `0` where the platform does not report one. device: u64, - /// Unix inode number, `0` where the platform does not report one. + /// Native file identity: the Unix inode number or the Windows file + /// index; `0` where the platform does not report one. inode: u64, - /// Unix inode-change instant. This is what catches an in-place rewrite - /// that restores the modification time afterwards. + /// The inode-change instant — Unix ctime, or the NTFS ChangeTime on + /// Windows. This is what catches an in-place rewrite that restores the + /// modification time afterwards. inode_change_nanos: i128, readonly: bool, }, @@ -163,22 +173,25 @@ impl ArtifactSeal { /// device node — is refused rather than sealed. pub fn observe(path: &Path) -> Result { match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.is_file() => Ok(Self { - path: path.to_path_buf(), - presence: SealPresence::Present { - len: metadata.len(), - modified_nanos: metadata - .modified() - .map_or(TIMESTAMP_UNAVAILABLE, system_time_nanos), - created_nanos: metadata - .created() - .map_or(TIMESTAMP_UNAVAILABLE, system_time_nanos), - device: native_device(&metadata), - inode: native_inode(&metadata), - inode_change_nanos: native_inode_change_nanos(&metadata), - readonly: metadata.permissions().readonly(), - }, - }), + Ok(metadata) if metadata.is_file() => { + let (device, inode, inode_change_nanos) = native_identity(path, &metadata); + Ok(Self { + path: path.to_path_buf(), + presence: SealPresence::Present { + len: metadata.len(), + modified_nanos: metadata + .modified() + .map_or(TIMESTAMP_UNAVAILABLE, system_time_nanos), + created_nanos: metadata + .created() + .map_or(TIMESTAMP_UNAVAILABLE, system_time_nanos), + device, + inode, + inode_change_nanos, + readonly: metadata.permissions().readonly(), + }, + }) + } Ok(_) => Err(ArtifactSealError::NotRegularFile { path: path.to_path_buf(), }), @@ -234,9 +247,10 @@ impl ArtifactSeal { /// Whether this pre-link observation still describes the source after an /// owned hard-link operation. /// - /// Creating a hard link changes the inode-change instant on Unix even - /// though it cannot change the file bytes. Every other observable field, - /// including the native file identity where available, must remain fixed. + /// Creating a hard link changes the inode-change instant — Unix ctime and + /// NTFS ChangeTime both move on link-count changes — even though it cannot + /// change the file bytes. Every other observable field, including the + /// native file identity where available, must remain fixed. fn same_source_after_hard_link(&self, after: &Self) -> bool { if self.path != after.path { return false; @@ -336,12 +350,109 @@ fn content_digests(seals: &[ArtifactSeal]) -> Option>> { .collect() } +/// One bounded observation of native file identity: device/inode pair and the +/// inode-change instant. The handle is opened once so all three values come +/// from the same query and cannot tear against each other. +#[cfg(unix)] +fn native_identity(_path: &Path, metadata: &std::fs::Metadata) -> (u64, u64, i128) { + ( + native_device(metadata), + native_inode(metadata), + native_inode_change_nanos(metadata), + ) +} + +/// `std::fs` reports no native file identity on Windows, so the observation +/// opens the file briefly and asks the handle for the volume serial number, +/// the file index, and the NTFS ChangeTime. `FILE_FLAG_OPEN_REPARSE_POINT` +/// keeps the query on the same node `symlink_metadata` accepted, and the +/// share mode admits every concurrent reader and writer. A path that cannot +/// be opened or queried this way keeps the fallback observation, so the seal +/// degrades to [`SealFidelity::TimestampsOnly`] rather than erroring. +#[cfg(windows)] +fn native_identity(path: &Path, _metadata: &std::fs::Metadata) -> (u64, u64, i128) { + windows_file_identity(path).unwrap_or((0, 0, TIMESTAMP_UNAVAILABLE)) +} + +#[cfg(not(any(unix, windows)))] +fn native_identity(_path: &Path, metadata: &std::fs::Metadata) -> (u64, u64, i128) { + ( + native_device(metadata), + native_inode(metadata), + native_inode_change_nanos(metadata), + ) +} + +/// `device`, file index, and `ChangeTime` for `path`, or `None` when the open +/// or either handle query fails. +#[cfg(windows)] +fn windows_file_identity(path: &Path) -> Option<(u64, u64, i128)> { + use std::mem::MaybeUninit; + use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, FILE_BASIC_INFO, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, + FILE_SHARE_WRITE, FileBasicInfo, GetFileInformationByHandle, GetFileInformationByHandleEx, + }; + + let file = std::fs::OpenOptions::new() + .access_mode(FILE_READ_ATTRIBUTES) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path) + .ok()?; + let handle = file.as_raw_handle().cast(); + + let mut information = MaybeUninit::::uninit(); + // SAFETY: `file` owns a valid handle for the duration of the call and the + // output points to correctly sized, writable storage. + if unsafe { GetFileInformationByHandle(handle, information.as_mut_ptr()) } == 0 { + return None; + } + // SAFETY: a successful `GetFileInformationByHandle` initializes all fields. + let information = unsafe { information.assume_init() }; + + let mut basic = MaybeUninit::::uninit(); + // SAFETY: `file` owns a valid handle for the duration of the call and the + // output points to correctly sized, writable storage. + if unsafe { + GetFileInformationByHandleEx( + handle, + FileBasicInfo, + basic.as_mut_ptr().cast(), + std::mem::size_of::() as u32, + ) + } == 0 + { + return None; + } + // SAFETY: a successful `GetFileInformationByHandleEx` initializes all + // fields. + let basic = unsafe { basic.assume_init() }; + drop(file); + + let device = u64::from(information.dwVolumeSerialNumber); + let inode = + (u64::from(information.nFileIndexHigh) << 32) | u64::from(information.nFileIndexLow); + // `ChangeTime` counts 100-ns intervals since 1601-01-01; shift it onto the + // Unix-epoch scale `system_time_nanos` uses. A filesystem that reports no + // change instant degrades this field alone to the sentinel. + const WINDOWS_UNIX_EPOCH_TICKS: i128 = 116_444_736_000_000_000; + let inode_change_nanos = if basic.ChangeTime > 0 { + (i128::from(basic.ChangeTime) - WINDOWS_UNIX_EPOCH_TICKS) * 100 + } else { + TIMESTAMP_UNAVAILABLE + }; + Some((device, inode, inode_change_nanos)) +} + #[cfg(unix)] fn native_device(metadata: &std::fs::Metadata) -> u64 { std::os::unix::fs::MetadataExt::dev(metadata) } -#[cfg(not(unix))] +#[cfg(not(any(unix, windows)))] fn native_device(_metadata: &std::fs::Metadata) -> u64 { 0 } @@ -351,7 +462,7 @@ fn native_inode(metadata: &std::fs::Metadata) -> u64 { std::os::unix::fs::MetadataExt::ino(metadata) } -#[cfg(not(unix))] +#[cfg(not(any(unix, windows)))] fn native_inode(_metadata: &std::fs::Metadata) -> u64 { 0 } @@ -362,7 +473,7 @@ fn native_inode_change_nanos(metadata: &std::fs::Metadata) -> i128 { i128::from(metadata.ctime()) * 1_000_000_000 + i128::from(metadata.ctime_nsec()) } -#[cfg(not(unix))] +#[cfg(not(any(unix, windows)))] fn native_inode_change_nanos(_metadata: &std::fs::Metadata) -> i128 { TIMESTAMP_UNAVAILABLE } @@ -438,11 +549,11 @@ where /// /// A failed validation removes any receipt for `key` and is never cached. /// - /// "Still holds" means what [`SealFidelity`] says it means on this - /// platform. Where the observation is - /// [`SealFidelity::TimestampsOnly`] — Windows — a same-length in-place - /// rewrite that restores the modification time still satisfies the seal and - /// is answered from the receipt. + /// "Still holds" means what [`SealFidelity`] says it means for this + /// observation. Where the observation is + /// [`SealFidelity::TimestampsOnly`], a same-length in-place rewrite that + /// restores the modification time still satisfies the seal and is answered + /// from the receipt. pub fn validate_sealed( &self, key: K, @@ -780,17 +891,18 @@ mod tests { .expect("seal the artifact") .fidelity(); - #[cfg(unix)] + #[cfg(any(unix, windows))] assert_eq!( fidelity, Some(SealFidelity::InodeChangeTracked), - "Unix reports a device/inode pair and an inode-change instant" + "Unix reports a device/inode pair and an inode-change instant; \ + Windows reports the volume serial number, file index, and NTFS ChangeTime" ); - #[cfg(not(unix))] + #[cfg(not(any(unix, windows)))] assert_eq!( fidelity, Some(SealFidelity::TimestampsOnly), - "Windows `std::fs` reports no device/inode pair and no inode-change instant; \ + "this platform reports no device/inode pair and no inode-change instant; \ claiming otherwise would let a corrupted generation answer from its receipt" ); @@ -1092,12 +1204,14 @@ mod tests { /// The seal's strength against a deliberate in-place rewrite, stated /// exactly as far as the platform can carry it. /// - /// This test used to assert detection unconditionally. It runs only where - /// `codestory-contracts` tests run — Linux and macOS — so the assertion was - /// never contradicted, while the same code on Windows answers the rewritten - /// artifact from the earlier receipt. Both outcomes are pinned here against - /// the fidelity the observation itself reports, so neither platform's - /// behaviour can drift and neither can be claimed for the other. + /// This test used to assert detection unconditionally, which Windows + /// contradicted while its seals were timestamps-only. Windows now reports + /// the NTFS ChangeTime through a bounded handle query, so both shipped + /// platforms take the tracked branch; the timestamps-only branch still + /// pins the residual guarantee for an observation whose identity query + /// fails. Both outcomes are pinned against the fidelity the observation + /// itself reports, so neither platform's behaviour can drift and neither + /// can be claimed for the other. #[test] fn an_in_place_rewrite_that_restores_the_modification_time_is_detected_only_at_full_fidelity() { let dir = tempfile::TempDir::new().expect("temp dir"); @@ -1177,11 +1291,10 @@ mod tests { /// it. /// /// A timestamps-only observation carries no file identity, so whether a - /// byte-identical replacement is noticed there depends on whether the new - /// file happens to inherit the old creation instant — which on NTFS it - /// sometimes does. The contract therefore claims nothing about that case - /// on such a platform, and this test claims nothing either; it pins the - /// residual guarantee instead. + /// byte-identical replacement is noticed depends on whether the new file + /// happens to inherit the old creation instant. The contract therefore + /// claims nothing about that case under such an observation, and this + /// test claims nothing either; it pins the residual guarantee instead. #[test] fn replacing_the_artifact_with_identical_bytes_is_detected_only_at_full_fidelity() { let dir = tempfile::TempDir::new().expect("temp dir"); @@ -1235,6 +1348,50 @@ mod tests { } } + /// Windows carries the same native identity as Unix: the volume serial + /// number, the file index, and the NTFS ChangeTime, read through a bounded + /// handle query. A same-length rewrite in place and a timestamp-preserving + /// replacement both break the seal. + #[cfg(windows)] + #[test] + fn a_windows_seal_tracks_the_inode_change_instant_and_replacement_identity() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let artifact = dir.path().join("shard.sqlite3"); + let pinned_modified = 1_700_000_000_000_000_000; + write(&artifact, "generation-a"); + set_modified(&artifact, pinned_modified); + let sealed = ArtifactSeal::observe(&artifact).expect("seal the artifact"); + assert_eq!( + sealed.fidelity(), + Some(SealFidelity::InodeChangeTracked), + "an NTFS file reports a volume/index pair and ChangeTime" + ); + + // Same path, same file index, same length, and the modification time + // put back exactly where it was: only ChangeTime records that the + // bytes were rewritten. + write(&artifact, "generation-X"); + set_modified(&artifact, pinned_modified); + let rewritten = ArtifactSeal::observe(&artifact).expect("re-observe"); + assert_ne!( + rewritten, sealed, + "an in-place rewrite at the same length must break the seal" + ); + + // A copy that preserves the visible timestamps and is then moved over + // the artifact is a different file index: the seal must still break. + let replacement = dir.path().join("shard.sqlite3.new"); + std::fs::copy(&artifact, &replacement).expect("copy bytes"); + set_modified(&replacement, pinned_modified); + std::fs::remove_file(&artifact).expect("remove artifact"); + std::fs::rename(&replacement, &artifact).expect("rename over artifact"); + assert_ne!( + ArtifactSeal::observe(&artifact).expect("observe replacement"), + rewritten, + "a timestamp-preserving copy/remove/rename replacement must break the seal" + ); + } + #[test] fn a_sidecar_appearing_beside_the_artifact_breaks_the_seal() { let dir = tempfile::TempDir::new().expect("temp dir"); diff --git a/crates/codestory-contracts/src/wire.rs b/crates/codestory-contracts/src/wire.rs index 44772a1e5..6215a35bc 100644 --- a/crates/codestory-contracts/src/wire.rs +++ b/crates/codestory-contracts/src/wire.rs @@ -30,7 +30,10 @@ use serde::{Deserialize, Serialize}; /// echoing whatever the client asked for. /// * **v3** — packet, context, and search publish closed evidence projections; /// packet truth dispositions and its evidence opt-out are no longer public. -pub const PUBLICATION_STAMP_SCHEMA_VERSION: u32 = 3; +/// * **v4** — additive: `freshness` reports whether the response came from the +/// fresh current source or a retained publication during a refresh, and +/// `served_from` now derives from that runtime decision. +pub const PUBLICATION_STAMP_SCHEMA_VERSION: u32 = 4; /// Oldest reader schema version that can still interpret a payload stamped with /// [`PUBLICATION_STAMP_SCHEMA_VERSION`] without misreading it. @@ -292,8 +295,8 @@ mod tests { #[test] fn published_stamp_bounds_are_the_documented_values() { assert_eq!( - PUBLICATION_STAMP_SCHEMA_VERSION, 3, - "the evidence-only v3 contract publishes stamp schema 3" + PUBLICATION_STAMP_SCHEMA_VERSION, 4, + "the additive freshness field publishes stamp schema 4" ); assert_eq!(MINIMUM_COMPATIBLE_PUBLICATION_STAMP_SCHEMA_VERSION, 3); assert_eq!(LEGACY_PUBLICATION_STAMP_SCHEMA_VERSION, 0); diff --git a/crates/codestory-indexer/Cargo.toml b/crates/codestory-indexer/Cargo.toml index 2e922dd6f..145b5191c 100644 --- a/crates/codestory-indexer/Cargo.toml +++ b/crates/codestory-indexer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codestory-indexer" -version = "0.17.6" +version = "0.17.7" edition = "2024" [dev-dependencies] diff --git a/crates/codestory-indexer/src/cache.rs b/crates/codestory-indexer/src/cache.rs index 815e5ea9a..e008b977b 100644 --- a/crates/codestory-indexer/src/cache.rs +++ b/crates/codestory-indexer/src/cache.rs @@ -1226,7 +1226,11 @@ mod tests { root, cache_path, source, &config, None, false, true, ) .expect("cache key"); - let route_language = FRAMEWORK_ROUTE_LANGUAGE_NAMES.contains(&config.language_name); + // Independently enumerated oracle: extensions whose languages carry + // route-declaration rules. Reading FRAMEWORK_ROUTE_LANGUAGE_NAMES + // here would be circular — dropping a language from the shared list + // would change the production mix and this oracle together. + let route_language = !matches!(extension, "c" | "cpp" | "sh"); assert_eq!(current != previous, route_language, "{extension}"); } diff --git a/crates/codestory-indexer/src/framework_routes.rs b/crates/codestory-indexer/src/framework_routes.rs index bdbc197a2..041ccda93 100644 --- a/crates/codestory-indexer/src/framework_routes.rs +++ b/crates/codestory-indexer/src/framework_routes.rs @@ -2069,7 +2069,37 @@ app.head("/string-only", documented); ); assert_eq!((parser_tp, parser_fp, parser_fn), (4, 0, 0)); - assert!(lexical_fp > 0 || lexical_fn > 0); + // Pin the lexical scanner's exact defect sets, not just "some defect": + // a repaired line scanner must be observed changing these rows, and + // skipping the defect branch can no longer satisfy `> 0` vacuously. + let lexical_extra = lexical + .difference(&expected) + .cloned() + .collect::>(); + let lexical_missing = expected + .difference(&lexical) + .cloned() + .collect::>(); + assert_eq!( + lexical_extra, + [ + ("DELETE".to_string(), "/nested-path".to_string()), + ("GET".to_string(), "/prefix".to_string()), + ("GET".to_string(), "/unowned".to_string()), + ("HEAD".to_string(), "/string-only".to_string()), + ] + .into_iter() + .collect::>() + ); + assert_eq!( + lexical_missing, + [ + ("POST".to_string(), "/multiline".to_string()), + ("PUT".to_string(), "/static-template".to_string()), + ] + .into_iter() + .collect::>() + ); assert!(parser_routes.iter().all(|route| { route.extraction_provenance == "tree_sitter_query" && route.claim_tier == "parser_backed" @@ -2522,7 +2552,40 @@ const example = `api.head("/string-only", documented);`; ); assert_eq!((parser_tp, parser_fp, parser_fn), (5, 0, 0)); - assert!(lexical_fp > 0 || lexical_fn > 0); + // Pin the lexical scanner's exact defect sets (see the express row). + let lexical_extra = lexical + .difference(&expected) + .cloned() + .collect::>(); + let lexical_missing = expected + .difference(&lexical) + .cloned() + .collect::>(); + assert_eq!( + lexical_extra, + [ + ("/DYNAMIC-METHOD".to_string(), "/dynamic-method".to_string()), + ("GET".to_string(), "/array-method".to_string()), + ("GET".to_string(), "/duplicate-method".to_string()), + ("GET".to_string(), "/missing-handler".to_string()), + ("GET".to_string(), "/spread".to_string()), + ("GET".to_string(), "/unrelated".to_string()), + ] + .into_iter() + .collect::>() + ); + assert_eq!( + lexical_missing, + [ + ("DELETE".to_string(), "/object".to_string()), + ("GET".to_string(), "/simple".to_string()), + ("PATCH".to_string(), "/wrapped".to_string()), + ("POST".to_string(), "/multiline".to_string()), + ("PUT".to_string(), "/static-template".to_string()), + ] + .into_iter() + .collect::>() + ); assert!(parser_routes.iter().all(|route| { route.extraction_provenance == "tree_sitter_query" && route.claim_tier == "parser_backed" diff --git a/crates/codestory-indexer/src/languages/go.rs b/crates/codestory-indexer/src/languages/go.rs index 466f67fd4..389afd6be 100644 --- a/crates/codestory-indexer/src/languages/go.rs +++ b/crates/codestory-indexer/src/languages/go.rs @@ -2000,7 +2000,11 @@ mod complexity_tests { .expect("Go grammar must load"); let tree = parser.parse(&source, None).expect("Go source must parse"); reset_go_navigation_resolution_work(); - let _ = receiver_call_specs(&tree, &source); + let specs = receiver_call_specs(&tree, &source); + assert!( + specs.len() >= call_count, + "each receiver call must produce a spec, or the work bound is vacuous" + ); go_navigation_resolution_work() } diff --git a/crates/codestory-indexer/src/lib.rs b/crates/codestory-indexer/src/lib.rs index 453c694ab..380b28d30 100644 --- a/crates/codestory-indexer/src/lib.rs +++ b/crates/codestory-indexer/src/lib.rs @@ -3551,8 +3551,24 @@ impl WorkspaceIndexer { codestory_workspace::workspace_relative_path(root, &full_path) .unwrap_or_else(|| path.to_path_buf()); let language = structural::structural_language_name(&full_path); - let producer = structural::structural_producer(&full_path) - .expect("admitted structural paths have one producer"); + let Some(producer) = structural::structural_producer(&full_path) else { + // An admitted path with no dispatch producer is a coverage gap, + // never a reason to abort the run. + return Err(incomplete_file_storage( + &full_path, + None, + language, + codestory_contracts::graph::ErrorInfo { + message: format!("No structural producer accepted {:?}", path), + file_id: None, + line: None, + column: None, + is_fatal: false, + index_step: codestory_contracts::graph::IndexStep::Collection, + coverage_reason: Some(FileCoverageReason::CollectorFailure), + }, + )); + }; let structural_size = std::fs::metadata(&full_path) .map_err(|error| { incomplete_file_storage( @@ -17411,6 +17427,9 @@ mod proof_resolution_cache_tests { assert_eq!(file.file_id, new_file); assert_eq!(file.top_level_declarations[0].declaration, new_method); assert_eq!(file.inherent_methods[0].declaration, new_method); + // `owner` is populated on the input but was previously never asserted — + // a rebase that left the stale old owner id would pass everything above. + assert_eq!(file.inherent_methods[0].owner, Some(new_owner)); assert_eq!(file.classes[0].declaration, new_owner); assert_eq!(file.classes[0].methods[0].declaration, new_method); assert_eq!(file.direct_exports[0].declaration, new_owner); diff --git a/crates/codestory-indexer/src/proof_resolution.rs b/crates/codestory-indexer/src/proof_resolution.rs index ce3a624db..f5b5d42c9 100644 --- a/crates/codestory-indexer/src/proof_resolution.rs +++ b/crates/codestory-indexer/src/proof_resolution.rs @@ -20702,7 +20702,12 @@ mod ruby_php_complexity_tests { .expect("Ruby grammar"); let tree = parser.parse(&source, None).expect("Ruby declarations"); reset_ruby_php_resolution_work(); - let _ = RubyResolutionIndex::build(&tree, &source, NodeId(1), &nodes); + let index = RubyResolutionIndex::build(&tree, &source, NodeId(1), &nodes); + assert_eq!( + index.declarations.len(), + declarations, + "every declared function must reach the index, or the work bound is vacuous" + ); ruby_php_resolution_work() } @@ -20776,12 +20781,46 @@ mod ruby_php_complexity_tests { .collect::>(); reset_ruby_php_resolution_work(); let index = JavaKotlinProjectionIndex::prepare(&records, &[]); + let ruby_candidates: usize = index.ruby_functions.values().map(Vec::len).sum(); + assert_eq!( + ruby_candidates, + files * declarations, + "every ruby declaration must reach the projection domain" + ); + let php_candidates: usize = index + .php_domains + .values() + .flat_map(|domain| domain.declarations.values().map(Vec::len)) + .sum(); + assert_eq!( + php_candidates, + files * declarations, + "every php declaration must reach the projection domain" + ); for record in &records { for declaration in &record.file.top_level_declarations { if record.file.language == "ruby" { - let _ = index.ruby_function(&declaration.name, declaration.declaration); + let resolution = + index.ruby_function(&declaration.name, declaration.declaration); + assert_eq!( + resolution.is_some(), + !duplicate, + "a unique ruby declaration must resolve; a duplicated one must not" + ); } else { - let _ = index.resolve_php(&record.file.php_namespace, None, &declaration.name); + let resolution = + index.resolve_php(&record.file.php_namespace, None, &declaration.name); + if duplicate { + assert!( + matches!(resolution, JavaKotlinImportResolution::Ambiguous), + "duplicated php declarations must resolve ambiguously" + ); + } else { + assert!( + matches!(resolution, JavaKotlinImportResolution::Exact { .. }), + "each unique php declaration must resolve exactly" + ); + } } } } @@ -20794,6 +20833,10 @@ mod ruby_php_complexity_tests { ("ruby", ruby_receiver_work(128), ruby_receiver_work(256)), ("php", php_receiver_work(128), php_receiver_work(256)), ] { + assert!( + small > 0, + "{language} work was not counted; a disabled counter makes the bound vacuous" + ); assert!( large <= small.saturating_mul(2).saturating_add(32), "{language} work grew superlinearly: 1x={small}, 2x={large}" @@ -20844,7 +20887,11 @@ mod rust_complexity_tests { .expect("Rust grammar must load"); let tree = parser.parse(source, None).expect("source must parse"); reset_rust_resolution_work(); - let _ = RustResolutionIndex::build(&tree, source, NodeId(1), &[]); + let index = RustResolutionIndex::build(&tree, source, NodeId(1), &[]); + assert!( + !index.calls.is_empty(), + "the measured source must produce call specs, or the work bound is vacuous" + ); rust_resolution_work() } @@ -21012,16 +21059,37 @@ mod rust_complexity_tests { let index = RustProjectionIndex::prepare(&records).expect("projection index"); for item in 0..count { let module = vec![format!("module_{item}")]; - let _ = index.module(&records[0], &module); - let _ = index.declarations(&records[0], &module, &format!("function_{item}")); - let _ = index.types(&records[0], &module, &format!("Owner{item}")); - let _ = index.methods( - &records[0], - &module, - &format!("Owner{item}"), - &format!("method_{item}"), + assert!( + index.module(&records[0], &module).is_some(), + "module_{item} must be indexed, or the projection bound is vacuous" + ); + assert!( + !index + .declarations(&records[0], &module, &format!("function_{item}")) + .is_empty(), + "function_{item} must be indexed" + ); + assert!( + !index + .types(&records[0], &module, &format!("Owner{item}")) + .is_empty(), + "Owner{item} must be indexed" + ); + assert!( + !index + .methods( + &records[0], + &module, + &format!("Owner{item}"), + &format!("method_{item}"), + ) + .is_empty(), + "Owner{item}::method_{item} must be indexed" + ); + assert!( + index.node_file(NodeId(50_000 + item as i64)).is_some(), + "import node 50_000+{item} must be indexed" ); - let _ = index.node_file(NodeId(50_000 + item as i64)); } rust_resolution_work() } @@ -21118,7 +21186,7 @@ mod c_cpp_complexity_tests { use super::*; use tree_sitter::Parser; - fn measured_work(source: &str) -> usize { + fn measured_work(source: &str, expected_calls: usize) -> usize { let mut parser = Parser::new(); parser .set_language(&tree_sitter_cpp::LANGUAGE.into()) @@ -21159,8 +21227,17 @@ mod c_cpp_complexity_tests { file_id, &nodes, ); + assert!( + index.calls.len() >= expected_calls, + "each receiver call must produce a spec, or the work bound is vacuous" + ); for call in &index.calls { - let _ = index.resolve_syntax_claim(call.callee, call.form, &call.raw_target); + let (caller, binding) = + index.resolve_syntax_claim(call.callee, call.form, &call.raw_target); + assert!( + caller.is_some() && !matches!(binding, CachedResolutionBinding::Unsupported), + "each receiver call must bind its caller, or the lookup did no work" + ); } c_cpp_resolution_work() } @@ -21183,8 +21260,8 @@ mod c_cpp_complexity_tests { #[test] fn c_cpp_parser_index_work_is_linear_for_doubled_receivers_and_nested_owners() { - let small = measured_work(&source(64)); - let large = measured_work(&source(128)); + let small = measured_work(&source(64), 64); + let large = measured_work(&source(128), 128); assert!( small >= 64 * 8, "C++ parser work was not fully counted: {small}" @@ -21332,7 +21409,7 @@ mod java_kotlin_complexity_tests { Ok(()) } - fn measured_work(language: &str, source: &str) -> usize { + fn measured_work(language: &str, source: &str, expected_calls: usize) -> usize { let mut parser = Parser::new(); let (grammar, path) = match language { "java" => (tree_sitter_java::LANGUAGE.into(), Path::new("Exact.java")), @@ -21380,8 +21457,23 @@ mod java_kotlin_complexity_tests { reset_java_kotlin_resolution_work(); let index = JavaKotlinResolutionIndex::build(&tree, source, path, language, file_id, &nodes); + assert!( + index.calls.len() >= expected_calls, + "{language}: each receiver call must produce a spec, or the work bound is vacuous" + ); for call in &index.calls { - let _ = index.resolve_syntax_claim(source, call.callee, call.form, &call.raw_target); + let (caller, binding) = + index.resolve_syntax_claim(source, call.callee, call.form, &call.raw_target); + // Dart callables are nominal here: their caller node is not in the + // fixture's callable set, so a legitimately collected dart call may + // return `(None, Unsupported)`. The spec census above is its + // non-vacuity proof; the other languages must bind every call. + if language != "dart" { + assert!( + caller.is_some() && !matches!(binding, CachedResolutionBinding::Unsupported), + "{language}: each receiver call must bind its caller, or the lookup did no work" + ); + } } java_kotlin_resolution_work() } @@ -21575,6 +21667,12 @@ mod java_kotlin_complexity_tests { .last() .is_some_and(|snapshot| snapshot.proof_store_transaction_completed) ); + assert!( + progress + .last() + .is_some_and(|snapshot| snapshot.facts_persisted > 0), + "the measured pipeline must persist call facts, or the work bound is vacuous" + ); store .validate_proof_resolution_publication(&publication) .expect("replay proof resolution"); @@ -21625,8 +21723,8 @@ mod java_kotlin_complexity_tests { #[test] fn java_kotlin_parser_index_work_is_linear_for_doubled_calls_and_nested_owners() { for language in ["java", "kotlin"] { - let small = measured_work(language, &source(language, 64)); - let large = measured_work(language, &source(language, 128)); + let small = measured_work(language, &source(language, 64), 64); + let large = measured_work(language, &source(language, 128), 128); assert!( small >= 64 * 8, "{language} parser work was not fully counted: {small}" @@ -21641,8 +21739,8 @@ mod java_kotlin_complexity_tests { #[test] fn csharp_swift_dart_parser_index_work_is_linear_for_doubled_receivers_and_callers() { for language in ["csharp", "swift", "dart"] { - let small = measured_work(language, &csd_source(language, 64)); - let large = measured_work(language, &csd_source(language, 128)); + let small = measured_work(language, &csd_source(language, 64), 64); + let large = measured_work(language, &csd_source(language, 128), 128); assert!( small >= 64 * 8, "{language} parser work was not fully counted: {small}" @@ -21768,7 +21866,11 @@ mod go_complexity_tests { .expect("Go grammar must load"); let tree = parser.parse(source, None).expect("source must parse"); reset_go_resolution_work(); - let _ = GoResolutionIndex::build(&tree, source, NodeId(1), &[]); + let index = GoResolutionIndex::build(&tree, source, NodeId(1), &[]); + assert!( + !index.calls.is_empty(), + "the measured Go source must produce call specs, or the work bound is vacuous" + ); go_resolution_work() } @@ -21821,7 +21923,11 @@ mod go_complexity_tests { reset_go_resolution_work(); let index = GoProjectionIndex::prepare(&records).expect("Go package projection"); for _ in 0..lookup_count { - let _ = index.resolve_function(&records[0], "proof", "Target"); + let resolution = index.resolve_function(&records[0], "proof", "Target"); + assert!( + matches!(resolution, GoFunctionResolution::Ambiguous), + "every package file declares Target, so the lookup must report ambiguity — Missing or Incomplete would mean the domain was empty" + ); } go_resolution_work() } @@ -21974,7 +22080,11 @@ mod python_complexity_tests { .expect("Python grammar must load"); let tree = parser.parse(&source, None).expect("source must parse"); reset_python_resolution_work(); - let _ = PythonResolutionIndex::build(&tree, &source, NodeId(1), &[]); + let index = PythonResolutionIndex::build(&tree, &source, NodeId(1), &[]); + assert!( + !index.calls.is_empty(), + "the measured Python source must produce call specs, or the work bound is vacuous" + ); python_resolution_work() } @@ -21999,7 +22109,11 @@ mod python_complexity_tests { let tree = parser.parse(&source, None).expect("source must parse"); assert!(!tree.root_node().has_error(), "nested pattern must parse"); reset_python_resolution_work(); - let _ = PythonResolutionIndex::build(&tree, &source, NodeId(1), &[]); + let index = PythonResolutionIndex::build(&tree, &source, NodeId(1), &[]); + assert!( + !index.calls.is_empty(), + "the nested-match source must produce call specs, or the work bound is vacuous" + ); python_resolution_work() } @@ -22039,10 +22153,16 @@ mod python_complexity_tests { resolution.target, RelativeImportResolution::Unique(_) )); - let _ = index.declarations(FileId(2), "target"); + assert!( + !index.declarations(FileId(2), "target").is_empty(), + "module_1.py's target must reach the projection, or the lookup bound is vacuous" + ); let owners = index.classes(FileId(2), "Worker"); assert_eq!(owners.len(), 1); - let _ = index.methods(FileId(2), owners[0], "run"); + assert!( + !index.methods(FileId(2), owners[0], "run").is_empty(), + "Worker::run must reach the projection" + ); } python_resolution_work() } diff --git a/crates/codestory-indexer/src/resolution/mod.rs b/crates/codestory-indexer/src/resolution/mod.rs index 2a2bd1371..162a30925 100644 --- a/crates/codestory-indexer/src/resolution/mod.rs +++ b/crates/codestory-indexer/src/resolution/mod.rs @@ -101,15 +101,6 @@ struct SemanticRequestStats { skipped_requests: usize, } -#[cfg(test)] -#[allow(dead_code)] -#[derive(Default)] -struct ResolutionLookupCache { - same_file_lookup: HashMap<(String, i64, String), Option>, - same_module_lookup: HashMap<(String, String, String), Option>, - global_unique_lookup: HashMap<(String, String), Option>, -} - #[derive(Debug, Clone)] struct CandidateNode { id: i64, @@ -3950,305 +3941,6 @@ fn module_prefix(qualified: &str) -> Option<(String, &'static str)> { None } -#[cfg(test)] -#[allow(dead_code)] -fn find_same_file( - conn: &rusqlite::Connection, - kind_clause: &str, - file_id: Option, - exact: &str, - suffix_dot: &str, - suffix_colon: &str, - lookup_cache: &mut ResolutionLookupCache, -) -> Result> { - let Some(file_id) = file_id else { - return Ok(None); - }; - let cache_key = (kind_clause.to_string(), file_id, exact.to_string()); - if let Some(cached) = lookup_cache.same_file_lookup.get(&cache_key) { - return Ok(*cached); - } - - let exact_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND file_node_id = ?1 - AND serialized_name = ?2 - ORDER BY start_line LIMIT 1", - kind_clause - ); - let resolved = if let Some(id) = conn - .query_row(&exact_query, params![file_id, exact], |row| row.get(0)) - .optional()? - { - Some(id) - } else { - let suffix_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND file_node_id = ?1 - AND (serialized_name LIKE ?2 OR serialized_name LIKE ?3) - ORDER BY start_line LIMIT 1", - kind_clause - ); - conn.query_row( - &suffix_query, - params![file_id, suffix_dot, suffix_colon], - |row| row.get(0), - ) - .optional()? - }; - lookup_cache.same_file_lookup.insert(cache_key, resolved); - Ok(resolved) -} - -#[cfg(test)] -#[allow(dead_code)] -#[allow(clippy::too_many_arguments)] -fn find_same_module( - conn: &rusqlite::Connection, - kind_clause: &str, - module_prefix: &str, - delimiter: &str, - exact: &str, - suffix_dot: &str, - suffix_colon: &str, - lookup_cache: &mut ResolutionLookupCache, -) -> Result> { - let pattern = format!("{}{}%", module_prefix, delimiter); - let cache_key = (kind_clause.to_string(), pattern.clone(), exact.to_string()); - if let Some(cached) = lookup_cache.same_module_lookup.get(&cache_key) { - return Ok(*cached); - } - - let exact_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND qualified_name LIKE ?1 - AND serialized_name = ?2 - ORDER BY start_line LIMIT 1", - kind_clause - ); - let resolved = if let Some(id) = conn - .query_row(&exact_query, params![pattern, exact], |row| row.get(0)) - .optional()? - { - Some(id) - } else { - let suffix_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND qualified_name LIKE ?1 - AND (serialized_name LIKE ?2 OR serialized_name LIKE ?3) - ORDER BY start_line LIMIT 1", - kind_clause - ); - conn.query_row( - &suffix_query, - params![pattern, suffix_dot, suffix_colon], - |row| row.get(0), - ) - .optional()? - }; - lookup_cache.same_module_lookup.insert(cache_key, resolved); - Ok(resolved) -} - -#[cfg(test)] -#[allow(dead_code)] -fn find_global_unique( - conn: &rusqlite::Connection, - kind_clause: &str, - exact: &str, - suffix_dot: &str, - suffix_colon: &str, - lookup_cache: &mut ResolutionLookupCache, -) -> Result> { - let cache_key = (kind_clause.to_string(), exact.to_string()); - if let Some(cached) = lookup_cache.global_unique_lookup.get(&cache_key) { - return Ok(*cached); - } - - let exact_count_query = format!( - "SELECT COUNT(*) FROM node - WHERE kind IN ({}) - AND serialized_name = ?1", - kind_clause - ); - let exact_count: i64 = conn.query_row(&exact_count_query, params![exact], |row| row.get(0))?; - let resolved = if exact_count == 1 { - let exact_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND serialized_name = ?1 - LIMIT 1", - kind_clause - ); - conn.query_row(&exact_query, params![exact], |row| row.get(0)) - .optional()? - } else if exact_count > 1 { - None - } else { - let suffix_count_query = format!( - "SELECT COUNT(*) FROM node - WHERE kind IN ({}) - AND (serialized_name LIKE ?1 OR serialized_name LIKE ?2)", - kind_clause - ); - let suffix_count: i64 = conn.query_row( - &suffix_count_query, - params![suffix_dot, suffix_colon], - |row| row.get(0), - )?; - if suffix_count != 1 { - None - } else { - let suffix_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND (serialized_name LIKE ?1 OR serialized_name LIKE ?2) - LIMIT 1", - kind_clause - ); - conn.query_row(&suffix_query, params![suffix_dot, suffix_colon], |row| { - row.get(0) - }) - .optional()? - } - }; - lookup_cache - .global_unique_lookup - .insert(cache_key, resolved); - Ok(resolved) -} - -#[cfg(test)] -#[allow(dead_code)] -fn find_fuzzy( - conn: &rusqlite::Connection, - kind_clause: &str, - exact: &str, - suffix_dot: &str, - suffix_colon: &str, -) -> Result> { - let exact_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND serialized_name = ?1 - ORDER BY start_line LIMIT 1", - kind_clause - ); - if let Some(id) = conn - .query_row(&exact_query, params![exact], |row| row.get(0)) - .optional()? - { - return Ok(Some(id)); - } - - let suffix_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND (serialized_name LIKE ?1 OR serialized_name LIKE ?2) - ORDER BY start_line LIMIT 1", - kind_clause - ); - if let Some(id) = conn - .query_row(&suffix_query, params![suffix_dot, suffix_colon], |row| { - row.get(0) - }) - .optional()? - { - return Ok(Some(id)); - } - - let fuzzy = format!("%{}%", exact); - let query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND serialized_name LIKE ?1 - ORDER BY start_line LIMIT 1", - kind_clause - ); - conn.query_row(&query, params![fuzzy], |row| row.get(0)) - .optional() - .map_err(Into::into) -} - -#[cfg(test)] -#[allow(dead_code)] -fn collect_candidate_pool( - conn: &rusqlite::Connection, - kind_clause: &str, - names: &[String], - out: &mut Vec, - limit: usize, -) -> Result<()> { - if out.len() >= limit { - return Ok(()); - } - for name in names { - let (exact, suffix_dot, suffix_colon) = name_patterns(name); - let top = find_top_matches(conn, kind_clause, &exact, &suffix_dot, &suffix_colon, 3)?; - for id in top { - record_candidate(out, id); - if out.len() >= limit { - return Ok(()); - } - } - } - Ok(()) -} - -#[cfg(test)] -#[allow(dead_code)] -fn find_top_matches( - conn: &rusqlite::Connection, - kind_clause: &str, - exact: &str, - suffix_dot: &str, - suffix_colon: &str, - limit: usize, -) -> Result> { - let exact_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND serialized_name = ?1 - ORDER BY start_line - LIMIT {}", - kind_clause, limit - ); - let mut out = Vec::with_capacity(limit); - { - let mut stmt = conn.prepare(&exact_query)?; - let rows = stmt.query_map(params![exact], |row| row.get(0))?; - for row in rows { - out.push(row?); - } - } - if out.len() >= limit { - return Ok(out); - } - - let remaining = limit - out.len(); - let suffix_query = format!( - "SELECT id FROM node - WHERE kind IN ({}) - AND (serialized_name LIKE ?1 OR serialized_name LIKE ?2) - ORDER BY start_line - LIMIT {}", - kind_clause, remaining - ); - let mut stmt = conn.prepare(&suffix_query)?; - let rows = stmt.query_map(params![suffix_dot, suffix_colon], |row| row.get(0))?; - for row in rows { - let candidate = row?; - if !out.contains(&candidate) { - out.push(candidate); - } - } - Ok(out) -} - fn candidate_json(candidates: &[i64]) -> Result> { if candidates.is_empty() { return Ok(None); @@ -4256,13 +3948,6 @@ fn candidate_json(candidates: &[i64]) -> Result> { Ok(Some(serde_json::to_string(candidates)?)) } -#[cfg(test)] -fn record_candidate(candidates: &mut Vec, candidate: i64) { - if !candidates.contains(&candidate) { - candidates.push(candidate); - } -} - fn consider_selected(selected: &mut Option<(i64, f32)>, candidate_id: i64, confidence: f32) { let replace = match *selected { Some((_, existing_confidence)) => confidence > existing_confidence, @@ -5041,53 +4726,6 @@ mod tests { Ok(()) } - #[test] - fn test_common_call_certain_semantic_candidate_still_resolves() -> Result<()> { - let conn = Connection::open_in_memory()?; - create_node_table(&conn)?; - let index = CandidateIndex::load(&conn, &[NodeKind::FUNCTION as i32])?; - let flags = ResolutionFlags { - legacy_mode: false, - enable_semantic: true, - store_candidates: false, - parallel_compute: false, - }; - let pass = ResolutionPass { - flags, - policy: ResolutionPolicy::for_flags(flags), - semantic_resolvers: SemanticResolverRegistry::new(true), - go_context: None, - }; - let row = ( - 2_i64, - Some(100_i64), - Some("pkg::core::caller".to_string()), - "caller".to_string(), - "clone".to_string(), - 0, - Some("/repo/lib.rs".to_string()), - Some("1:2:3:4".to_string()), - None, - ); - let semantic_candidates = vec![SemanticResolutionCandidate { - target_node_id: 77_i64, - confidence: ResolutionCertainty::CERTAIN_MIN, - }]; - - let computed = candidate_selection::compute_call_resolution( - &pass, - &index, - &row, - &semantic_candidates, - )?; - assert_eq!( - computed.strategy, - Some(ResolutionStrategy::CallSemanticFallback) - ); - assert_eq!(computed.update.resolved_target_node_id, Some(77_i64)); - Ok(()) - } - #[test] fn test_semantic_language_bucket_matrix() { let expected = [ @@ -5380,55 +5018,6 @@ mod tests { Ok(()) } - #[test] - fn test_exact_lookup_cache_reuses_same_file_key() -> Result<()> { - let conn = Connection::open_in_memory()?; - conn.execute_batch( - "CREATE TABLE node ( - id INTEGER PRIMARY KEY, - kind INTEGER NOT NULL, - serialized_name TEXT NOT NULL, - canonical_id TEXT, - file_node_id INTEGER, - start_line INTEGER NOT NULL DEFAULT 0 - );", - )?; - conn.execute( - "INSERT INTO node (id, kind, serialized_name, file_node_id, start_line) - VALUES (?1, ?2, ?3, ?4, ?5)", - params![77_i64, NodeKind::FUNCTION as i32, "foo", 555_i64, 1_i64], - )?; - - let kind_clause = kind_clause(&[NodeKind::FUNCTION as i32]); - let (exact, suffix_dot, suffix_colon) = name_patterns("foo"); - let mut lookup_cache = ResolutionLookupCache::default(); - - let first = find_same_file( - &conn, - &kind_clause, - Some(555_i64), - &exact, - &suffix_dot, - &suffix_colon, - &mut lookup_cache, - )?; - assert_eq!(first, Some(77_i64)); - assert_eq!(lookup_cache.same_file_lookup.len(), 1); - - let second = find_same_file( - &conn, - &kind_clause, - Some(555_i64), - &exact, - &suffix_dot, - &suffix_colon, - &mut lookup_cache, - )?; - assert_eq!(second, Some(77_i64)); - assert_eq!(lookup_cache.same_file_lookup.len(), 1); - Ok(()) - } - #[test] fn test_candidate_index_same_file_exact_beats_suffix() -> Result<()> { let conn = Connection::open_in_memory()?; @@ -5826,6 +5415,14 @@ mod tests { let ids: Vec = (1..=600_i64).collect(); + let persisted_value = |conn: &Connection| -> Result { + Ok( + conn.query_row("SELECT COALESCE(SUM(value), 0) FROM perf", [], |row| { + row.get(0) + })?, + ) + }; + let no_tx_start = Instant::now(); for id in &ids { conn.execute( @@ -5834,6 +5431,11 @@ mod tests { )?; } let no_tx_elapsed = no_tx_start.elapsed(); + assert_eq!( + persisted_value(&conn)?, + ids.len() as i64, + "autocommit updates must be persisted before the timing is compared" + ); conn.execute("UPDATE perf SET value = 0", [])?; @@ -5846,6 +5448,24 @@ mod tests { Ok(()) })?; let tx_elapsed = tx_start.elapsed(); + assert_eq!( + persisted_value(&conn)?, + ids.len() as i64, + "transaction updates must be committed before the timing is compared" + ); + + // A transaction that aborts must leave prior values untouched; without + // this the smoke test times a block whose writes may never commit. + let aborted = run_in_immediate_transaction(&conn, |tx_conn| -> Result<()> { + tx_conn.execute("UPDATE perf SET value = value + 100", [])?; + Err(anyhow::anyhow!("forced abort")) + }); + assert!(aborted.is_err(), "the injected abort must surface"); + assert_eq!( + persisted_value(&conn)?, + ids.len() as i64, + "an aborted transaction must roll back every staged write" + ); assert!( tx_elapsed < no_tx_elapsed, diff --git a/crates/codestory-indexer/src/structural/blanking.rs b/crates/codestory-indexer/src/structural/blanking.rs index 2f37afcc1..cb8dddb76 100644 --- a/crates/codestory-indexer/src/structural/blanking.rs +++ b/crates/codestory-indexer/src/structural/blanking.rs @@ -143,6 +143,22 @@ mod tests { assert!(blanked.contains("let x = 1;")); assert!(blanked.as_bytes()[0] == b' '); assert!(blanked.as_bytes()[4] == b' '); + // The line boundaries themselves must survive: blanking newline or CR + // bytes to spaces would keep length and still pass the above. + let source_line_bytes = source + .bytes() + .enumerate() + .filter_map(|(index, byte)| (byte == b'\n' || byte == b'\r').then_some(index)) + .collect::>(); + let blanked_line_bytes = blanked + .bytes() + .enumerate() + .filter_map(|(index, byte)| (byte == b'\n' || byte == b'\r').then_some(index)) + .collect::>(); + assert_eq!( + blanked_line_bytes, source_line_bytes, + "blanking must preserve every line-boundary byte position" + ); } #[test] diff --git a/crates/codestory-indexer/src/structural/css.rs b/crates/codestory-indexer/src/structural/css.rs index 4ec0bedf2..f02d4332c 100644 --- a/crates/codestory-indexer/src/structural/css.rs +++ b/crates/codestory-indexer/src/structural/css.rs @@ -927,6 +927,15 @@ mod tests { assert!(kinds.contains(&NodeKind::CONSTANT)); assert!(kinds.contains(&NodeKind::VARIABLE)); assert!(storage.edges.iter().any(|e| e.kind == EdgeKind::MEMBER)); + // `.btn` alone supplies a CONSTANT; pin each named selector family so + // dropping id collection cannot hide behind the class node. + for canonical in ["css:class:btn", "css:id:app", "css:var:--primary"] { + assert!( + find_node(&storage, canonical).is_some(), + "expected `{canonical}` to be minted: {:?}", + storage.nodes + ); + } } #[test] diff --git a/crates/codestory-indexer/src/structural/html.rs b/crates/codestory-indexer/src/structural/html.rs index 1323dcba2..60fd15822 100644 --- a/crates/codestory-indexer/src/structural/html.rs +++ b/crates/codestory-indexer/src/structural/html.rs @@ -422,11 +422,31 @@ mod tests { .any(|n| n.canonical_id.as_deref() == Some("html:id:app")) ); assert!(storage.edges.iter().any(|e| e.kind == EdgeKind::USAGE)); + // `css:class:layout` is minted twice through different paths: the class + // attribute's USAGE linkage and the embedded `