diff --git a/docs/state-space.md b/docs/state-space.md new file mode 100644 index 0000000..217d0dd --- /dev/null +++ b/docs/state-space.md @@ -0,0 +1,29 @@ +# State-space safety + +CodeDecay evaluates **bounded cache/feature-flag state matrices** from fixture +experiments. It does not flush production caches or contact remote flag +providers without explicit configuration. + +## What it can establish + +- State dimensions: flags, config, cache state/version, tenant, cohort, revision +- Bounded pairwise or explicit combinations with coverage accounting +- Cold/warm/stale cache comparisons and flag-interaction oracles +- Distinction between confirmed regression, passed oracle, provider-blocked, + bounds-blocked, and untested/pruned combinations +- Repair tasks that attach a durable regression test id after a confirmed defect + +## What it cannot establish + +- Exhaustive coverage of the full state space +- Production cache/flag behavior +- A `fullyVerified: true` result (always false in this slice) + +## CLI / MCP + +```bash +codedecay state-space --experiment experiment.json --surface src/cache/profile.ts +codedecay state-space --experiment experiment.json --target-kind fixture-local --format json +``` + +MCP tool: `state_space_safety`. diff --git a/packages/cli/src/commands/registry.ts b/packages/cli/src/commands/registry.ts index cf3bf6b..5b871e2 100644 --- a/packages/cli/src/commands/registry.ts +++ b/packages/cli/src/commands/registry.ts @@ -18,6 +18,7 @@ import { import { runMcpCommand as runMcpCommandWithDependencies } from "./mcp"; import { runMigrationCommand as runMigrationCommandWithDependencies } from "./migration"; import { runConcurrencyCommand as runConcurrencyCommandWithDependencies } from "./concurrency"; +import { runStateSpaceCommand as runStateSpaceCommandWithDependencies } from "./state-space"; import { runProductCommand as runProductCommandWithDependencies } from "./product"; import { runRedteamCommand as runRedteamCommandWithDependencies } from "./redteam"; import { runRevalidateCommand as runRevalidateCommandWithDependencies } from "./revalidate"; @@ -104,6 +105,10 @@ export function createCommandHandlers(options: CommandRegistryOptions): Record runStateSpaceCommandWithDependencies(context, { + resolveRepoRoot: getRepoRootForCli, + writeOutput: writeCliOutput + }), memory: (context) => runMemoryCommandWithDependencies(context, { resolveRepoRoot: getRepoRootForCli }), "memory-import": (context) => runMemoryImportCommandWithDependencies(context, { resolveRepoRoot: getRepoRootForCli }), "memory-learn": (context) => runMemoryLearnCommandWithDependencies(context, { resolveRepoRoot: getRepoRootForCli }), diff --git a/packages/cli/src/commands/state-space.ts b/packages/cli/src/commands/state-space.ts new file mode 100644 index 0000000..33d2dbe --- /dev/null +++ b/packages/cli/src/commands/state-space.ts @@ -0,0 +1,24 @@ +import { resolve } from "node:path"; +import { analyzeStateSpaceSafety, renderStateSpaceSafetyMarkdown } from "@submuxhq/codedecay-knowledge"; +import { parseStateSpaceArgs } from "../parsers/args"; +import type { CliCommandContext, CliRuntime, StateSpaceOptions } from "../types"; + +export interface RunStateSpaceCommandDependencies { + resolveRepoRoot(cwd: string, options: StateSpaceOptions): string; + writeOutput(input: { cwd: string; output?: string | undefined; rendered: string; runtime: CliRuntime }): void; +} + +export function runStateSpaceCommand(context: CliCommandContext, dependencies: RunStateSpaceCommandDependencies): void { + const options = parseStateSpaceArgs(context.args); + const cwd = resolve(context.runtimeCwd, options.cwd ?? "."); + const rootDir = dependencies.resolveRepoRoot(cwd, options); + const report = analyzeStateSpaceSafety({ + rootDir, + experimentFile: options.experimentFile, + surfaceFiles: options.surfaceFiles, + targetKind: options.targetKind, + cleanupPlan: options.cleanupPlan + }); + const rendered = options.format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderStateSpaceSafetyMarkdown(report); + dependencies.writeOutput({ cwd: rootDir, output: options.output, rendered, runtime: context.runtime }); +} diff --git a/packages/cli/src/docs/command-docs/analysis.ts b/packages/cli/src/docs/command-docs/analysis.ts index 695d3ce..1594c81 100644 --- a/packages/cli/src/docs/command-docs/analysis.ts +++ b/packages/cli/src/docs/command-docs/analysis.ts @@ -53,6 +53,31 @@ export const ANALYSIS_COMMAND_DOCS: Record = { "Stress-only results stay inconclusive. See docs/concurrency.md." ] }, + "state-space": { + name: "state-space", + summary: "Plan and evaluate bounded cache/feature-flag state matrices.", + usage: ["codedecay state-space [options]"], + description: [ + "Load a seeded state-space experiment fixture, detect cache/flag candidates, generate bounded pairwise or explicit combinations, and evaluate stale-cache / flag-interaction oracles without contacting remote providers by default." + ], + options: [ + { flag: "--experiment ", description: "Repo-local state-space experiment JSON fixture" }, + { flag: "--surface ", description: "Source file to scan for state dimensions; repeatable" }, + { flag: "--target-kind ", description: "fixture-local | disposable-local | remote-unapproved | production-like | unspecified" }, + { flag: "--cleanup-plan ", description: "Disposable target cleanup plan" }, + { flag: "--cwd ", description: "Working directory" }, + { flag: "--format ", description: "Output format" }, + { flag: "--output ", description: "Write report to a file" } + ], + examples: [ + "codedecay state-space --experiment .codedecay/state-space/stale-cache.json --surface src/cache/profile.ts", + "codedecay state-space --experiment experiment.json --target-kind fixture-local --format json" + ], + notes: [ + "Coverage is bounded and never implies exhaustive proof. See docs/state-space.md.", + "Remote flag providers stay blocked unless explicitly configured." + ] + }, runtime: { name: "runtime", summary: "Ingest local runtime exports as redacted engineering evidence.", diff --git a/packages/cli/src/docs/command-docs/order.ts b/packages/cli/src/docs/command-docs/order.ts index e6d25eb..5dbaca6 100644 --- a/packages/cli/src/docs/command-docs/order.ts +++ b/packages/cli/src/docs/command-docs/order.ts @@ -1,3 +1,3 @@ -export const COMMAND_ORDER = ["ai", "session", "context", "analyze", "runtime", "migration", "concurrency", "topology", "benchmark", "snapshot", "redteam", "revalidate", "llm-review", "agent", "loop", "doctor", "config", "memory", "memory-import", "memory-learn", "execute", "differential", "product", "dashboard", "mcp"] as const; +export const COMMAND_ORDER = ["ai", "session", "context", "analyze", "runtime", "migration", "concurrency", "state-space", "topology", "benchmark", "snapshot", "redteam", "revalidate", "llm-review", "agent", "loop", "doctor", "config", "memory", "memory-import", "memory-learn", "execute", "differential", "product", "dashboard", "mcp"] as const; export const UTILITY_COMMAND_ORDER = ["help", "man", "update", "uninstall", "version"] as const; export const ROOT_FLAG_ALIASES = ["--help", "-h", "--version", "-V"] as const; diff --git a/packages/cli/src/parsers/args.ts b/packages/cli/src/parsers/args.ts index a31f32b..df54f81 100644 --- a/packages/cli/src/parsers/args.ts +++ b/packages/cli/src/parsers/args.ts @@ -14,6 +14,7 @@ export { parseMcpArgs } from "./mcp"; export { parseMemoryArgs, parseMemoryImportArgs, parseMemoryLearnArgs, parseMemoryLearningArgs, parseMemorySetupArgs } from "./memory"; export { parseMigrationArgs } from "./migration"; export { parseConcurrencyArgs } from "./concurrency"; +export { parseStateSpaceArgs } from "./state-space"; export { parseRevalidateArgs } from "./revalidate"; export { parseRuntimeArgs } from "./runtime"; export { parseProductArgs } from "./product"; diff --git a/packages/cli/src/parsers/state-space.ts b/packages/cli/src/parsers/state-space.ts new file mode 100644 index 0000000..2e2be33 --- /dev/null +++ b/packages/cli/src/parsers/state-space.ts @@ -0,0 +1,50 @@ +import type { StateSpaceOptions } from "../types"; +import { requireValue } from "./primitives"; +import { HelpRequested, throwUnknownOption } from "./shared"; + +export function parseStateSpaceArgs(args: string[]): StateSpaceOptions { + const options: StateSpaceOptions = { surfaceFiles: [], format: "markdown" }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (!arg) continue; + if (arg === "--help" || arg === "-h") throw new HelpRequested(); + const [flag, inline] = splitArg(arg); + const value = () => inline ?? requireValue(args, index, flag); + if (flag === "--experiment") options.experimentFile = value(); + else if (flag === "--surface") options.surfaceFiles.push(value()); + else if (flag === "--cwd") options.cwd = value(); + else if (flag === "--output") options.output = value(); + else if (flag === "--format") options.format = parseFormat(value()); + else if (flag === "--target-kind") options.targetKind = parseTarget(value()); + else if (flag === "--cleanup-plan") options.cleanupPlan = value(); + else { + throwUnknownOption(arg, "state-space"); + continue; + } + if (inline === undefined) index += 1; + } + return options; +} + +function splitArg(arg: string): [string, string | undefined] { + const index = arg.indexOf("="); + return index < 0 ? [arg, undefined] : [arg.slice(0, index), arg.slice(index + 1)]; +} + +function parseFormat(value: string): StateSpaceOptions["format"] { + if (value === "json" || value === "markdown") return value; + throw new Error(`Invalid state-space format "${value}". Expected json or markdown.`); +} + +function parseTarget(value: string): NonNullable { + if ( + value === "unspecified" || + value === "fixture-local" || + value === "disposable-local" || + value === "remote-unapproved" || + value === "production-like" + ) { + return value; + } + throw new Error(`Invalid state-space target kind "${value}".`); +} diff --git a/packages/cli/src/types/index.ts b/packages/cli/src/types/index.ts index 7e29a8b..dec9968 100644 --- a/packages/cli/src/types/index.ts +++ b/packages/cli/src/types/index.ts @@ -13,6 +13,7 @@ export * from "./llm-review"; export * from "./loop"; export * from "./maintenance"; export * from "./concurrency"; +export * from "./state-space"; export * from "./migration"; export * from "./mcp"; export * from "./memory"; diff --git a/packages/cli/src/types/state-space.ts b/packages/cli/src/types/state-space.ts new file mode 100644 index 0000000..a443937 --- /dev/null +++ b/packages/cli/src/types/state-space.ts @@ -0,0 +1,12 @@ +import type { ConfigFormat } from "./common"; +import type { StateSpaceTargetKind } from "@submuxhq/codedecay-knowledge"; + +export interface StateSpaceOptions { + cwd?: string | undefined; + experimentFile?: string | undefined; + surfaceFiles: string[]; + targetKind?: StateSpaceTargetKind | undefined; + cleanupPlan?: string | undefined; + format: ConfigFormat; + output?: string | undefined; +} diff --git a/packages/cli/test/state-space.test.ts b/packages/cli/test/state-space.test.ts new file mode 100644 index 0000000..729bd95 --- /dev/null +++ b/packages/cli/test/state-space.test.ts @@ -0,0 +1,69 @@ +import { execFileSync } from "node:child_process"; +import { copyFileSync, mkdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { runCli } from "../src/index"; + +const fixtures = join(dirname(fileURLToPath(import.meta.url)), "../../knowledge/test/fixtures/state-space"); +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("codedecay state-space CLI", () => { + it("evaluates a stale-cache fixture in a child repository", async () => { + const root = createRepo(); + mkdirSync(join(root, "experiments"), { recursive: true }); + copyFileSync(join(fixtures, "stale-cache.json"), join(root, "experiments", "stale-cache.json")); + const result = await run([ + "state-space", + "--cwd", + root, + "--experiment", + "experiments/stale-cache.json", + "--format", + "json", + "--output", + "reports/state-space.json" + ]); + const report = JSON.parse(readFileSync(join(root, "reports", "state-space.json"), "utf8")) as { + verdict: string; + fullyVerified: boolean; + coverage: { exhaustive: boolean }; + }; + expect(result).toEqual({ exitCode: 0, stdout: "", stderr: "" }); + expect(report.verdict).toBe("confirmed-regression"); + expect(report.fullyVerified).toBe(false); + expect(report.coverage.exhaustive).toBe(false); + }); + + it("exposes state-space help", async () => { + const result = await run(["state-space", "--help"]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("CodeDecay state-space"); + }); +}); + +async function run(args: string[]): Promise<{ exitCode: number; stdout: string; stderr: string }> { + let stdout = ""; + let stderr = ""; + const exitCode = await runCli(args, { + stdout: (text) => { + stdout += text; + }, + stderr: (text) => { + stderr += text; + } + }); + return { exitCode, stdout, stderr }; +} + +function createRepo(): string { + const root = join(tmpdir(), `codedecay-state-space-cli-${Date.now()}-${Math.random().toString(16).slice(2)}`); + mkdirSync(root, { recursive: true }); + execFileSync("git", ["init", "-q"], { cwd: root }); + roots.push(root); + return root; +} diff --git a/packages/knowledge/src/index.ts b/packages/knowledge/src/index.ts index 4aad425..7710eaf 100644 --- a/packages/knowledge/src/index.ts +++ b/packages/knowledge/src/index.ts @@ -110,6 +110,28 @@ export type { ConcurrencyTargetKind, ConcurrencyVerdict } from "./concurrency/types"; +export { analyzeStateSpaceSafety } from "./state-space/analyze"; +export type { AnalyzeStateSpaceSafetyOptions } from "./state-space/analyze"; +export { gateStateSpaceBounds } from "./state-space/bounds"; +export { detectStateSpaceCandidates } from "./state-space/detect"; +export { evaluateStateSpaceOracle, generateStateSpaceCombinations } from "./state-space/oracles"; +export { renderStateSpaceSafetyMarkdown } from "./state-space/render"; +export { + STATE_SPACE_DEFAULT_BOUNDS, + STATE_SPACE_EVIDENCE_SCHEMA_VERSION +} from "./state-space/types"; +export type { + StateSpaceBounds, + StateSpaceCandidate, + StateSpaceCombination, + StateSpaceCoverageReport, + StateSpaceDimension, + StateSpaceExperimentInput, + StateSpaceExperimentKind, + StateSpaceSafetyReport, + StateSpaceTargetKind, + StateSpaceVerdict +} from "./state-space/types"; export type { IngestRuntimeEvidenceOptions } from "./runtime/ingest"; export { renderRuntimeEvidenceMarkdown } from "./runtime/render"; export { RUNTIME_EVIDENCE_SCHEMA_VERSION } from "./runtime/types"; diff --git a/packages/knowledge/src/state-space/analyze.ts b/packages/knowledge/src/state-space/analyze.ts new file mode 100644 index 0000000..4021167 --- /dev/null +++ b/packages/knowledge/src/state-space/analyze.ts @@ -0,0 +1,345 @@ +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { resolve } from "node:path"; +import { gateStateSpaceBounds } from "./bounds"; +import { detectStateSpaceCandidates } from "./detect"; +import { evaluateStateSpaceOracle, generateStateSpaceCombinations } from "./oracles"; +import { + STATE_SPACE_DEFAULT_BOUNDS, + STATE_SPACE_EVIDENCE_SCHEMA_VERSION, + type StateSpaceCleanupEvidence, + type StateSpaceCoverageReport, + type StateSpaceExperimentInput, + type StateSpaceRepairTask, + type StateSpaceSafetyReport, + type StateSpaceTargetKind, + type StateSpaceVerdict +} from "./types"; + +const TOOL_VERSION = "codedecay-state-space-oracle/1"; + +export interface AnalyzeStateSpaceSafetyOptions { + rootDir: string; + experimentFile?: string | undefined; + experiment?: StateSpaceExperimentInput | undefined; + surfaceFiles?: string[] | undefined; + cleanupPlan?: string | undefined; + targetKind?: StateSpaceTargetKind | undefined; + generatedAt?: string | undefined; +} + +export function analyzeStateSpaceSafety(options: AnalyzeStateSpaceSafetyOptions): StateSpaceSafetyReport { + const rootDir = realpathSync(options.rootDir); + const experiment = options.experiment ?? loadExperiment(rootDir, options.experimentFile); + const candidates = detectStateSpaceCandidates(rootDir, options.surfaceFiles ?? []); + const limitations = [ + "Bounded state-space matrices do not prove exhaustive flag/cache coverage.", + "No production cache flush, flag mutation, or remote provider write was performed.", + "Keyword candidate detection is not proof; attach explicit dimensions and oracles.", + "Pruned combinations remain untested and must be reported as such." + ]; + const blockers: string[] = []; + const investigationTasks: string[] = []; + const repairTasks: StateSpaceRepairTask[] = []; + + if (!experiment) { + limitations.unshift("No state-space experiment fixture was supplied."); + for (const candidate of candidates) { + investigationTasks.push( + `Promote ${candidate.kind} candidate ${candidate.surface} into an explicit dimension with bounded values.` + ); + } + return baseReport({ + generatedAt: options.generatedAt, + verdict: candidates.length ? "plan-ready" : "needs-human", + bounds: { + maxDimensions: STATE_SPACE_DEFAULT_BOUNDS.maxDimensions, + maxCombinations: STATE_SPACE_DEFAULT_BOUNDS.maxCombinations, + timeoutMs: STATE_SPACE_DEFAULT_BOUNDS.timeoutMs, + targetKind: options.targetKind ?? "unspecified" + }, + boundsBlocked: false, + candidates, + dimensions: [], + combinations: [], + coverage: emptyCoverage(), + cleanup: createCleanup(options.cleanupPlan, options.targetKind ?? "unspecified"), + repairTasks: [], + treeStatus: "unverified", + blockers: [], + investigationTasks, + limitations, + remoteFlagProviderContacted: false + }); + } + + const bounds = { + ...experiment.bounds, + targetKind: options.targetKind ?? experiment.bounds.targetKind + }; + const combinations = generateStateSpaceCombinations( + experiment.dimensions, + experiment.seed, + bounds.maxCombinations, + experiment.combinations + ); + const gate = gateStateSpaceBounds(bounds, experiment.dimensions.length, combinations.filter((c) => c.selected).length); + const cleanup = createCleanup(options.cleanupPlan ?? experiment.cleanup?.plan, bounds.targetKind); + const remoteContacted = experiment.remoteFlagProvider?.contacted === true; + const remoteConfigured = experiment.remoteFlagProvider?.configured === true; + + if (remoteContacted && !remoteConfigured) { + blockers.push("Remote flag provider contact requires explicit configuration and command intent."); + return baseReport({ + generatedAt: options.generatedAt, + experimentId: experiment.id, + experimentKind: experiment.kind, + verdict: "provider-blocked", + bounds: gate.effective, + boundsBlocked: false, + candidates, + dimensions: experiment.dimensions, + combinations, + coverage: coverageFrom(combinations, []), + cleanup, + repairTasks: [], + treeStatus: "unverified", + blockers, + investigationTasks: ["Configure a local/disposable flag adapter before contacting any remote provider."], + limitations, + remoteFlagProviderContacted: true + }); + } + + if (gate.blocked) { + blockers.push(...gate.reasons); + return baseReport({ + generatedAt: options.generatedAt, + experimentId: experiment.id, + experimentKind: experiment.kind, + verdict: "bounds-blocked", + bounds: gate.effective, + boundsBlocked: true, + candidates, + dimensions: experiment.dimensions, + combinations, + coverage: coverageFrom(combinations, []), + cleanup, + repairTasks: [], + treeStatus: "unverified", + blockers, + investigationTasks: ["Reduce dimensions/combinations/timeout to configured disposable bounds."], + limitations, + remoteFlagProviderContacted: remoteContacted && remoteConfigured + }); + } + + const oracleEval = evaluateStateSpaceOracle({ ...experiment, bounds: gate.effective }, combinations); + let verdict: StateSpaceVerdict = oracleEval.verdict; + const coverage = coverageFrom(combinations, oracleEval.combinationResults); + + if (verdict === "confirmed-regression") { + investigationTasks.push( + `Confirmed state-space regression for ${experiment.id}; preserve seed ${experiment.seed} and failing combinations.` + ); + repairTasks.push({ + id: `repair:${experiment.id}`, + title: `Fix confirmed state-space defect ${experiment.id}`, + detail: oracleEval.failures.join(" ") + }); + } + + if (verdict === "passed-oracle") { + investigationTasks.push( + `Oracle passed for ${experiment.id}; keep the selected matrix as a regression fixture. Coverage is not exhaustive.` + ); + } + + let treeStatus: StateSpaceSafetyReport["treeStatus"] = "unverified"; + if (experiment.repair?.durableRegressionTestId) { + repairTasks.push({ + id: `regression:${experiment.id}`, + title: "Add durable state-space regression test", + detail: `Attach ${experiment.repair.durableRegressionTestId} for the confirmed matrix failure.`, + durableRegressionTestId: experiment.repair.durableRegressionTestId + }); + } + if (experiment.implementation.mode === "repaired" && experiment.repair?.revalidated === true) { + treeStatus = "revalidated-fixture"; + verdict = "passed-oracle"; + investigationTasks.push( + `Revalidated state matrix on the current tree using ${experiment.repair.durableRegressionTestId ?? "fixture oracle"}.` + ); + } + + if (cleanup.required && !cleanup.plan) { + blockers.push("Cleanup plan is required for disposable state-space targets."); + verdict = "needs-human"; + } + + investigationTasks.push( + `Coverage: tested=${coverage.testedCount}, failed=${coverage.failedCount}, untested=${coverage.untestedCount}, pruned=${coverage.prunedCount}.` + ); + + return baseReport({ + generatedAt: options.generatedAt, + experimentId: experiment.id, + experimentKind: experiment.kind, + verdict, + bounds: gate.effective, + boundsBlocked: false, + candidates, + dimensions: experiment.dimensions, + combinations, + coverage, + oracle: { + verdict, + seed: experiment.seed, + toolVersion: TOOL_VERSION, + combinationResults: oracleEval.combinationResults, + failures: oracleEval.failures + }, + cleanup, + repairTasks, + treeStatus, + blockers, + investigationTasks, + limitations, + remoteFlagProviderContacted: remoteContacted && remoteConfigured + }); +} + +function loadExperiment(rootDir: string, file?: string): StateSpaceExperimentInput | undefined { + if (!file) return undefined; + const absolute = resolve(rootDir, file); + if (!existsSync(absolute) || !statSync(absolute).isFile()) { + throw new Error(`State-space experiment file not found: ${file}`); + } + const parsed = JSON.parse(readFileSync(absolute, "utf8")) as StateSpaceExperimentInput; + if (!parsed?.id || !parsed.kind || !parsed.dimensions || !parsed.bounds || parsed.seed === undefined) { + throw new Error(`State-space experiment file is missing required fields: ${file}`); + } + return parsed; +} + +function createCleanup(plan: string | undefined, targetKind: StateSpaceTargetKind): StateSpaceCleanupEvidence { + const required = targetKind === "fixture-local" || targetKind === "disposable-local"; + return { + plan, + required, + proven: false, + requiredOnFailure: true, + limitations: [ + "Cleanup plans are recorded but not executed in this deterministic oracle slice.", + "No production cache flush or flag mutation is allowed." + ] + }; +} + +function emptyCoverage(): StateSpaceCoverageReport { + return { + selectedCount: 0, + testedCount: 0, + failedCount: 0, + skippedCount: 0, + untestedCount: 0, + prunedCount: 0, + exhaustive: false, + limitations: ["No combinations were generated."] + }; +} + +function coverageFrom( + combinations: StateSpaceSafetyReport["combinations"], + results: NonNullable["combinationResults"] +): StateSpaceCoverageReport { + const byId = new Map(results.map((item) => [item.combinationId, item])); + let testedCount = 0; + let failedCount = 0; + let skippedCount = 0; + let untestedCount = 0; + let prunedCount = 0; + for (const combination of combinations) { + if (!combination.selected) { + prunedCount += 1; + untestedCount += 1; + continue; + } + const result = byId.get(combination.id); + if (!result || result.status === "untested") { + untestedCount += 1; + continue; + } + if (result.status === "skipped") skippedCount += 1; + else { + testedCount += 1; + if (result.status === "failed") failedCount += 1; + } + } + return { + selectedCount: combinations.filter((item) => item.selected).length, + testedCount, + failedCount, + skippedCount, + untestedCount, + prunedCount, + exhaustive: false, + limitations: ["Generated coverage is bounded and never implies exhaustive state-space proof."] + }; +} + +function baseReport(input: { + generatedAt?: string | undefined; + experimentId?: string | undefined; + experimentKind?: StateSpaceSafetyReport["experimentKind"]; + verdict: StateSpaceVerdict; + bounds: StateSpaceSafetyReport["bounds"]; + boundsBlocked: boolean; + candidates: StateSpaceSafetyReport["candidates"]; + dimensions: StateSpaceSafetyReport["dimensions"]; + combinations: StateSpaceSafetyReport["combinations"]; + coverage: StateSpaceCoverageReport; + oracle?: StateSpaceSafetyReport["oracle"]; + cleanup: StateSpaceCleanupEvidence; + repairTasks: StateSpaceRepairTask[]; + treeStatus: StateSpaceSafetyReport["treeStatus"]; + blockers: string[]; + investigationTasks: string[]; + limitations: string[]; + remoteFlagProviderContacted: boolean; +}): StateSpaceSafetyReport { + return { + tool: "CodeDecay", + schemaVersion: STATE_SPACE_EVIDENCE_SCHEMA_VERSION, + generatedAt: input.generatedAt ?? new Date().toISOString(), + experimentId: input.experimentId, + experimentKind: input.experimentKind, + verdict: input.verdict, + fullyVerified: false, + bounds: input.bounds, + boundsBlocked: input.boundsBlocked, + candidates: input.candidates, + dimensions: input.dimensions, + combinations: input.combinations, + coverage: input.coverage, + oracle: input.oracle, + cleanup: input.cleanup, + repairTasks: input.repairTasks, + treeStatus: input.treeStatus, + extensionBoundaries: [ + { id: "launchdarkly", status: "planned", detail: "LaunchDarkly adapter boundary without hidden network access." }, + { id: "unleash", status: "planned", detail: "Unleash adapter boundary for local fixtures." }, + { id: "redis-cache", status: "planned", detail: "Disposable Redis/cache testcontainer adapter." }, + { id: "config-flags", status: "planned", detail: "Repo-local config flag files remain first-class." } + ], + blockers: input.blockers, + investigationTasks: input.investigationTasks, + limitations: input.limitations, + safety: { + commandsExecuted: false, + productionTargetAllowed: false, + networkCalled: false, + remoteFlagProviderContacted: input.remoteFlagProviderContacted, + secretsRead: false + } + }; +} diff --git a/packages/knowledge/src/state-space/bounds.ts b/packages/knowledge/src/state-space/bounds.ts new file mode 100644 index 0000000..12d5a3a --- /dev/null +++ b/packages/knowledge/src/state-space/bounds.ts @@ -0,0 +1,48 @@ +import { + STATE_SPACE_DEFAULT_BOUNDS, + type StateSpaceBounds, + type StateSpaceTargetKind +} from "./types"; + +export interface StateSpaceBoundsGate { + blocked: boolean; + reasons: string[]; + effective: StateSpaceBounds; +} + +export function gateStateSpaceBounds( + bounds: StateSpaceBounds, + dimensionCount: number, + combinationCount: number +): StateSpaceBoundsGate { + const reasons: string[] = []; + if (bounds.maxDimensions < 1 || bounds.maxDimensions > STATE_SPACE_DEFAULT_BOUNDS.maxDimensions) { + reasons.push( + `maxDimensions ${bounds.maxDimensions} is outside configured bound 1..${STATE_SPACE_DEFAULT_BOUNDS.maxDimensions}.` + ); + } + if (bounds.maxCombinations < 1 || bounds.maxCombinations > STATE_SPACE_DEFAULT_BOUNDS.maxCombinations) { + reasons.push( + `maxCombinations ${bounds.maxCombinations} is outside configured bound 1..${STATE_SPACE_DEFAULT_BOUNDS.maxCombinations}.` + ); + } + if (bounds.timeoutMs < 1 || bounds.timeoutMs > STATE_SPACE_DEFAULT_BOUNDS.timeoutMs) { + reasons.push( + `timeoutMs ${bounds.timeoutMs} is outside configured bound 1..${STATE_SPACE_DEFAULT_BOUNDS.timeoutMs}.` + ); + } + if (!isAllowedTarget(bounds.targetKind)) { + reasons.push(`Target kind "${bounds.targetKind}" is not allowed for state-space experiments.`); + } + if (dimensionCount > bounds.maxDimensions) { + reasons.push(`Dimension count ${dimensionCount} exceeds maxDimensions ${bounds.maxDimensions}.`); + } + if (combinationCount > bounds.maxCombinations) { + reasons.push(`Combination count ${combinationCount} exceeds maxCombinations ${bounds.maxCombinations}.`); + } + return { blocked: reasons.length > 0, reasons, effective: { ...bounds } }; +} + +function isAllowedTarget(kind: StateSpaceTargetKind): boolean { + return kind === "fixture-local" || kind === "disposable-local"; +} diff --git a/packages/knowledge/src/state-space/detect.ts b/packages/knowledge/src/state-space/detect.ts new file mode 100644 index 0000000..5b2f8bb --- /dev/null +++ b/packages/knowledge/src/state-space/detect.ts @@ -0,0 +1,65 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; +import { relative, resolve } from "node:path"; +import type { StateSpaceCandidate, StateSpaceDimensionKind } from "./types"; + +const MAX_FILES = 50; +const MAX_FILE_BYTES = 1024 * 1024; + +interface Detector { + kind: StateSpaceDimensionKind; + pattern: RegExp; + note: string; +} + +const DETECTORS: Detector[] = [ + { + kind: "feature-flag", + pattern: /\b(featureFlag|feature_flag|launchDarkly|unleash|flags?\.[a-zA-Z]|isEnabled\()\b/i, + note: "Feature-flag API mentioned; keyword match is a candidate dimension, not proof." + }, + { + kind: "cache-state", + pattern: /\b(redis|cache\.(get|set|del)|invalidate|memoize|lru)\b/i, + note: "Cache API mentioned; model cold/warm/stale states explicitly." + }, + { + kind: "config-value", + pattern: /\b(process\.env\.|config\.[a-zA-Z]|getConfig\()\b/i, + note: "Config value mentioned; treat as a state dimension only with cited requirements." + }, + { + kind: "actor-tenant", + pattern: /\b(tenantId|orgId|workspaceId|actorId)\b/i, + note: "Tenant/actor identity mentioned; multi-tenant cache keys may diverge." + }, + { + kind: "rollout-cohort", + pattern: /\b(rollout|cohort|canary|percentage)\b/i, + note: "Rollout/cohort mention; pairwise with flags when requirements cite it." + } +]; + +export function detectStateSpaceCandidates(rootDir: string, files: string[]): StateSpaceCandidate[] { + const root = realpathSync(rootDir); + const candidates: StateSpaceCandidate[] = []; + for (const file of files.slice(0, MAX_FILES)) { + const absolute = resolve(root, file); + if (!existsSync(absolute) || !statSync(absolute).isFile()) continue; + if (statSync(absolute).size > MAX_FILE_BYTES) continue; + const content = readFileSync(absolute, "utf8"); + const relativePath = relative(root, absolute).replaceAll("\\", "/"); + for (const detector of DETECTORS) { + if (!detector.pattern.test(content)) continue; + candidates.push({ + id: createHash("sha256").update(`${detector.kind}:${relativePath}`).digest("hex").slice(0, 12), + kind: detector.kind, + surface: relativePath, + sourceRef: relativePath, + citedEvidence: [`keyword:${detector.kind}`], + note: detector.note + }); + } + } + return candidates; +} diff --git a/packages/knowledge/src/state-space/oracles.ts b/packages/knowledge/src/state-space/oracles.ts new file mode 100644 index 0000000..c00d8a4 --- /dev/null +++ b/packages/knowledge/src/state-space/oracles.ts @@ -0,0 +1,200 @@ +import type { + StateSpaceCombination, + StateSpaceDimension, + StateSpaceExperimentInput +} from "./types"; + +/** Deterministic pairwise-ish generator: cover each adjacent dimension pair, then prune to maxCombinations. */ +export function generateStateSpaceCombinations( + dimensions: StateSpaceDimension[], + seed: number, + maxCombinations: number, + explicit?: StateSpaceCombination[] +): StateSpaceCombination[] { + if (explicit?.length) { + return explicit.map((item, index) => ({ + ...item, + selected: index < maxCombinations, + exclusionReason: index < maxCombinations ? undefined : "Pruned by maxCombinations bound." + })); + } + + if (!dimensions.length) return []; + + const pairs: StateSpaceCombination[] = []; + for (let i = 0; i < dimensions.length; i += 1) { + for (let j = i + 1; j < dimensions.length; j += 1) { + const left = dimensions[i]!; + const right = dimensions[j]!; + for (const lv of left.values) { + for (const rv of right.values) { + const values: Record = {}; + for (const dim of dimensions) { + if (dim.id === left.id) values[dim.id] = lv; + else if (dim.id === right.id) values[dim.id] = rv; + else values[dim.id] = dim.values[seed % dim.values.length]!; + } + pairs.push({ + id: `pair:${left.id}=${lv}|${right.id}=${rv}`, + values, + selected: true + }); + } + } + } + } + + // Also include the all-defaults row. + const defaults: Record = {}; + for (const dim of dimensions) defaults[dim.id] = dim.values[0]!; + pairs.unshift({ id: "defaults", values: defaults, selected: true }); + + const deduped = dedupe(pairs); + return deduped.map((item, index) => ({ + ...item, + selected: index < maxCombinations, + exclusionReason: index < maxCombinations ? undefined : "Pruned by maxCombinations bound; coverage is not exhaustive." + })); +} + +function dedupe(items: StateSpaceCombination[]): StateSpaceCombination[] { + const seen = new Set(); + const out: StateSpaceCombination[] = []; + for (const item of items) { + const key = JSON.stringify(item.values); + if (seen.has(key)) continue; + seen.add(key); + out.push(item); + } + return out; +} + +export function evaluateStateSpaceOracle( + experiment: StateSpaceExperimentInput, + combinations: StateSpaceCombination[] +): { + verdict: import("./types").StateSpaceVerdict; + combinationResults: import("./types").StateSpaceCombinationResult[]; + failures: string[]; +} { + const failures: string[] = []; + const combinationResults: import("./types").StateSpaceCombinationResult[] = []; + const selected = combinations.filter((item) => item.selected); + + for (const combination of combinations) { + if (!combination.selected) { + combinationResults.push({ + combinationId: combination.id, + values: combination.values, + status: "untested", + detail: combination.exclusionReason ?? "Not selected." + }); + continue; + } + + const result = evaluateOne(experiment, combination); + combinationResults.push(result); + if (result.status === "failed") failures.push(`${combination.id}: ${result.detail}`); + } + + if (experiment.remoteFlagProvider?.contacted && !experiment.remoteFlagProvider.configured) { + return { + verdict: "provider-blocked", + combinationResults, + failures: ["Remote flag provider was contacted without explicit configuration."] + }; + } + + const failed = combinationResults.filter((item) => item.status === "failed"); + if (!selected.length) { + return { verdict: "insufficient-state-model", combinationResults, failures: ["No combinations were selected."] }; + } + if (failed.length) { + return { verdict: "confirmed-regression", combinationResults, failures }; + } + return { verdict: "passed-oracle", combinationResults, failures }; +} + +function evaluateOne( + experiment: StateSpaceExperimentInput, + combination: StateSpaceCombination +): import("./types").StateSpaceCombinationResult { + const mode = experiment.implementation.mode; + const cacheState = combination.values.cache ?? combination.values["cache-state"]; + const flags = Object.entries(combination.values).filter(([key]) => key.startsWith("flag:") || key.includes("flag")); + + if (mode === "stale-cache") { + const writeValue = experiment.implementation.writeValue ?? "new"; + const cachedValue = experiment.implementation.cachedValue ?? "old"; + if (cacheState === "warm" || cacheState === "stale") { + const read = cachedValue; + if (experiment.oracle.expectedReadValue && read !== experiment.oracle.expectedReadValue) { + return { + combinationId: combination.id, + values: combination.values, + status: "failed", + detail: `Stale cache read returned "${read}" after write "${writeValue}" under ${cacheState} state.` + }; + } + } + if (cacheState === "cold" || cacheState === "invalidated" || cacheState === "missing") { + return { + combinationId: combination.id, + values: combination.values, + status: "passed", + detail: `Cold/invalidated path returned fresh value under ${cacheState}.` + }; + } + } + + if (mode === "flag-interaction-bug") { + const required = experiment.oracle.requiredFlagsOn ?? []; + const allOn = required.every((flagId) => combination.values[flagId] === "on"); + const anyOn = required.some((flagId) => combination.values[flagId] === "on"); + if (allOn) { + return { + combinationId: combination.id, + values: combination.values, + status: "failed", + detail: `Pairwise flag combination ${required.join("+")}=on fails while independent ons pass.` + }; + } + if (anyOn || required.every((flagId) => combination.values[flagId] === "off")) { + return { + combinationId: combination.id, + values: combination.values, + status: "passed", + detail: "Independent flag states pass." + }; + } + } + + if (mode === "clean" || mode === "repaired") { + const forbidden = experiment.oracle.forbidden ?? []; + for (const rule of forbidden) { + const [flagId, value] = rule.split("="); + if (flagId && value && combination.values[flagId] === value) { + return { + combinationId: combination.id, + values: combination.values, + status: "skipped", + detail: `Forbidden combination ${rule} excluded as expected flag-specific behavior.` + }; + } + } + return { + combinationId: combination.id, + values: combination.values, + status: "passed", + detail: "Oracle passed for selected state combination." + }; + } + + void flags; + return { + combinationId: combination.id, + values: combination.values, + status: "passed", + detail: "No failing oracle condition matched." + }; +} diff --git a/packages/knowledge/src/state-space/render.ts b/packages/knowledge/src/state-space/render.ts new file mode 100644 index 0000000..97d5b17 --- /dev/null +++ b/packages/knowledge/src/state-space/render.ts @@ -0,0 +1,74 @@ +import type { StateSpaceSafetyReport } from "./types"; + +export function renderStateSpaceSafetyMarkdown(report: StateSpaceSafetyReport): string { + const lines = [ + "## CodeDecay State-Space Safety", + "", + `Verdict: \`${report.verdict}\`; fullyVerified: \`${report.fullyVerified}\`; tree: \`${report.treeStatus}\`.`, + report.experimentId + ? `Experiment: \`${report.experimentId}\` (${report.experimentKind ?? "unknown"}).` + : "Experiment: none supplied.", + `Bounds: dimensions≤${report.bounds.maxDimensions}, combinations≤${report.bounds.maxCombinations}, timeoutMs=${report.bounds.timeoutMs}, target=${report.bounds.targetKind}.`, + `Coverage: tested=${report.coverage.testedCount}, failed=${report.coverage.failedCount}, skipped=${report.coverage.skippedCount}, untested=${report.coverage.untestedCount}, pruned=${report.coverage.prunedCount}; exhaustive=\`${report.coverage.exhaustive}\`.`, + "Commands executed: no. Remote flag provider contacted: " + + (report.safety.remoteFlagProviderContacted ? "yes (configured)." : "no."), + "", + "### Dimensions", + "" + ]; + if (!report.dimensions.length) lines.push("No explicit dimensions were supplied."); + for (const dim of report.dimensions) { + lines.push(`- \`${dim.id}\` (${dim.kind}): ${dim.values.map((value) => `\`${value}\``).join(", ")} — ${dim.note}`); + } + + lines.push("", "### Combinations", ""); + if (!report.combinations.length) lines.push("No combinations were generated."); + for (const combination of report.combinations) { + const values = Object.entries(combination.values) + .map(([key, value]) => `${key}=${value}`) + .join(", "); + lines.push( + `- \`${combination.id}\` ${combination.selected ? "selected" : "pruned"}: ${values}${combination.exclusionReason ? ` (${combination.exclusionReason})` : ""}` + ); + } + + lines.push("", "### Oracle", ""); + if (!report.oracle) lines.push("No oracle was evaluated."); + else { + lines.push(`Verdict \`${report.oracle.verdict}\`; seed=${report.oracle.seed}; tool=${report.oracle.toolVersion}.`); + for (const result of report.oracle.combinationResults) { + lines.push(`- ${result.status} \`${result.combinationId}\`: ${result.detail}`); + } + } + + lines.push("", "### Candidates", ""); + if (!report.candidates.length) lines.push("No keyword candidates were detected."); + for (const candidate of report.candidates) { + lines.push(`- \`${candidate.kind}\` \`${candidate.surface}\` — ${candidate.note}`); + } + + lines.push("", "### Repair Tasks", ""); + if (!report.repairTasks.length) lines.push("No repair task was generated."); + for (const task of report.repairTasks) lines.push(`- **${task.title}**: ${task.detail}`); + + lines.push("", "### Blockers", ""); + if (!report.blockers.length) lines.push("No blocker."); + for (const blocker of report.blockers) lines.push(`- ${blocker}`); + + lines.push("", "### Cleanup", ""); + lines.push(`- Required: \`${report.cleanup.required}\``); + lines.push(`- Plan: ${report.cleanup.plan ? `\`${report.cleanup.plan}\`` : "missing"}`); + lines.push(`- Proven: \`${report.cleanup.proven}\``); + + lines.push("", "### Extension Boundaries", ""); + for (const boundary of report.extensionBoundaries) { + lines.push(`- \`${boundary.id}\` (${boundary.status}): ${boundary.detail}`); + } + + lines.push("", "### Investigation Tasks", ""); + for (const task of report.investigationTasks) lines.push(`- ${task}`); + + lines.push("", "### Limitations", ""); + for (const limitation of report.limitations) lines.push(`- ${limitation}`); + return `${lines.join("\n")}\n`; +} diff --git a/packages/knowledge/src/state-space/types.ts b/packages/knowledge/src/state-space/types.ts new file mode 100644 index 0000000..c18727a --- /dev/null +++ b/packages/knowledge/src/state-space/types.ts @@ -0,0 +1,199 @@ +export const STATE_SPACE_EVIDENCE_SCHEMA_VERSION = 1 as const; + +export const STATE_SPACE_DEFAULT_BOUNDS = { + maxDimensions: 8, + maxCombinations: 32, + timeoutMs: 30_000 +} as const; + +export type StateSpaceTargetKind = + | "unspecified" + | "fixture-local" + | "disposable-local" + | "remote-unapproved" + | "production-like"; + +export type StateSpaceCacheState = + | "cold" + | "warm" + | "stale" + | "missing" + | "malformed" + | "expired" + | "invalidated"; + +export type StateSpaceVerdict = + | "confirmed-regression" + | "expected-flag-behavior" + | "passed-oracle" + | "insufficient-state-model" + | "flaky-suspicion" + | "setup-failure" + | "untested-combination" + | "bounds-blocked" + | "provider-blocked" + | "needs-human" + | "plan-ready"; + +export type StateSpaceDimensionKind = + | "feature-flag" + | "config-value" + | "cache-state" + | "cache-version" + | "actor-tenant" + | "rollout-cohort" + | "revision"; + +export type StateSpaceExperimentKind = "pairwise-matrix" | "impact-guided" | "explicit-matrix"; + +export interface StateSpaceBounds { + maxDimensions: number; + maxCombinations: number; + timeoutMs: number; + targetKind: StateSpaceTargetKind; + allowRemoteFlagProvider?: boolean | undefined; +} + +export interface StateSpaceDimension { + id: string; + kind: StateSpaceDimensionKind; + values: string[]; + sourceRef: string; + note: string; +} + +export interface StateSpaceCombination { + id: string; + values: Record; + selected: boolean; + exclusionReason?: string | undefined; +} + +export interface StateSpaceOracleExpectation { + /** Cache key that should be invalidated after write. */ + cacheKey?: string | undefined; + /** Expected readable value after the write under the selected cache state. */ + expectedReadValue?: string | undefined; + /** Flag ids that must all be on for the path to succeed. */ + requiredFlagsOn?: string[] | undefined; + /** Forbidden combinations described as flagId=value pairs. */ + forbidden?: string[] | undefined; +} + +export interface StateSpaceImplementation { + /** Declared buggy/clean behavior for fixture evaluation. */ + mode: "stale-cache" | "flag-interaction-bug" | "clean" | "repaired"; + writeValue?: string | undefined; + cachedValue?: string | undefined; + flagEffects?: Record | undefined; +} + +export interface StateSpaceRepairEvidence { + durableRegressionTestId?: string | undefined; + revalidated?: boolean | undefined; +} + +export interface StateSpaceExperimentInput { + id: string; + kind: StateSpaceExperimentKind; + seed: number; + dimensions: StateSpaceDimension[]; + /** Optional explicit combinations; otherwise pairwise generated. */ + combinations?: StateSpaceCombination[] | undefined; + oracle: StateSpaceOracleExpectation; + implementation: StateSpaceImplementation; + bounds: StateSpaceBounds; + cleanup?: { plan?: string | undefined } | undefined; + repair?: StateSpaceRepairEvidence | undefined; + remoteFlagProvider?: { + configured: boolean; + contacted?: boolean | undefined; + } | undefined; +} + +export interface StateSpaceCandidate { + id: string; + kind: StateSpaceDimensionKind; + surface: string; + sourceRef: string; + citedEvidence: string[]; + note: string; +} + +export interface StateSpaceCombinationResult { + combinationId: string; + values: Record; + status: "passed" | "failed" | "skipped" | "untested"; + detail: string; +} + +export interface StateSpaceCoverageReport { + selectedCount: number; + testedCount: number; + failedCount: number; + skippedCount: number; + untestedCount: number; + prunedCount: number; + exhaustive: false; + limitations: string[]; +} + +export interface StateSpaceOracleResult { + verdict: StateSpaceVerdict; + seed: number; + toolVersion: string; + combinationResults: StateSpaceCombinationResult[]; + failures: string[]; +} + +export interface StateSpaceCleanupEvidence { + plan?: string | undefined; + required: boolean; + proven: false; + requiredOnFailure: true; + limitations: string[]; +} + +export interface StateSpaceRepairTask { + id: string; + title: string; + detail: string; + durableRegressionTestId?: string | undefined; +} + +export interface StateSpaceExtensionBoundary { + id: string; + status: "planned"; + detail: string; +} + +export interface StateSpaceSafetyReport { + tool: "CodeDecay"; + schemaVersion: typeof STATE_SPACE_EVIDENCE_SCHEMA_VERSION; + generatedAt: string; + experimentId?: string | undefined; + experimentKind?: StateSpaceExperimentKind | undefined; + verdict: StateSpaceVerdict; + fullyVerified: false; + bounds: StateSpaceBounds; + boundsBlocked: boolean; + candidates: StateSpaceCandidate[]; + dimensions: StateSpaceDimension[]; + combinations: StateSpaceCombination[]; + coverage: StateSpaceCoverageReport; + oracle?: StateSpaceOracleResult | undefined; + cleanup: StateSpaceCleanupEvidence; + repairTasks: StateSpaceRepairTask[]; + treeStatus: "unverified" | "revalidated-fixture"; + extensionBoundaries: StateSpaceExtensionBoundary[]; + blockers: string[]; + investigationTasks: string[]; + limitations: string[]; + safety: { + commandsExecuted: false; + productionTargetAllowed: false; + networkCalled: false; + remoteFlagProviderContacted: boolean; + secretsRead: false; + }; +} diff --git a/packages/knowledge/test/fixtures/state-space/clean-matrix.json b/packages/knowledge/test/fixtures/state-space/clean-matrix.json new file mode 100644 index 0000000..f47be13 --- /dev/null +++ b/packages/knowledge/test/fixtures/state-space/clean-matrix.json @@ -0,0 +1,27 @@ +{ + "id": "uat-state-3-clean-matrix", + "kind": "explicit-matrix", + "seed": 33, + "dimensions": [ + { + "id": "flag:gamma", + "kind": "feature-flag", + "values": ["off", "on"], + "sourceRef": "fixture", + "note": "Gamma flag" + } + ], + "combinations": [ + { "id": "off", "values": { "flag:gamma": "off" }, "selected": true }, + { "id": "on", "values": { "flag:gamma": "on" }, "selected": true } + ], + "oracle": {}, + "implementation": { "mode": "clean" }, + "bounds": { + "maxDimensions": 4, + "maxCombinations": 8, + "timeoutMs": 5000, + "targetKind": "fixture-local" + }, + "cleanup": { "plan": "reset fixture flags" } +} diff --git a/packages/knowledge/test/fixtures/state-space/coverage.json b/packages/knowledge/test/fixtures/state-space/coverage.json new file mode 100644 index 0000000..0da9045 --- /dev/null +++ b/packages/knowledge/test/fixtures/state-space/coverage.json @@ -0,0 +1,37 @@ +{ + "id": "uat-state-4-coverage", + "kind": "pairwise-matrix", + "seed": 44, + "dimensions": [ + { + "id": "flag:a", + "kind": "feature-flag", + "values": ["off", "on"], + "sourceRef": "fixture", + "note": "A" + }, + { + "id": "flag:b", + "kind": "feature-flag", + "values": ["off", "on"], + "sourceRef": "fixture", + "note": "B" + }, + { + "id": "cache", + "kind": "cache-state", + "values": ["cold", "warm"], + "sourceRef": "fixture", + "note": "cache" + } + ], + "oracle": {}, + "implementation": { "mode": "clean" }, + "bounds": { + "maxDimensions": 8, + "maxCombinations": 3, + "timeoutMs": 5000, + "targetKind": "fixture-local" + }, + "cleanup": { "plan": "reset fixture" } +} diff --git a/packages/knowledge/test/fixtures/state-space/flag-pair.json b/packages/knowledge/test/fixtures/state-space/flag-pair.json new file mode 100644 index 0000000..cacf66f --- /dev/null +++ b/packages/knowledge/test/fixtures/state-space/flag-pair.json @@ -0,0 +1,36 @@ +{ + "id": "uat-state-2-flag-pair", + "kind": "explicit-matrix", + "seed": 22, + "dimensions": [ + { + "id": "flag:alpha", + "kind": "feature-flag", + "values": ["off", "on"], + "sourceRef": "fixture", + "note": "Alpha flag" + }, + { + "id": "flag:beta", + "kind": "feature-flag", + "values": ["off", "on"], + "sourceRef": "fixture", + "note": "Beta flag" + } + ], + "combinations": [ + { "id": "alpha-on", "values": { "flag:alpha": "on", "flag:beta": "off" }, "selected": true }, + { "id": "beta-on", "values": { "flag:alpha": "off", "flag:beta": "on" }, "selected": true }, + { "id": "both-on", "values": { "flag:alpha": "on", "flag:beta": "on" }, "selected": true }, + { "id": "both-off", "values": { "flag:alpha": "off", "flag:beta": "off" }, "selected": true } + ], + "oracle": { "requiredFlagsOn": ["flag:alpha", "flag:beta"] }, + "implementation": { "mode": "flag-interaction-bug" }, + "bounds": { + "maxDimensions": 4, + "maxCombinations": 8, + "timeoutMs": 5000, + "targetKind": "fixture-local" + }, + "cleanup": { "plan": "reset fixture flags" } +} diff --git a/packages/knowledge/test/fixtures/state-space/remote-provider.json b/packages/knowledge/test/fixtures/state-space/remote-provider.json new file mode 100644 index 0000000..656eba6 --- /dev/null +++ b/packages/knowledge/test/fixtures/state-space/remote-provider.json @@ -0,0 +1,28 @@ +{ + "id": "uat-state-5-remote-provider", + "kind": "explicit-matrix", + "seed": 55, + "dimensions": [ + { + "id": "flag:remote", + "kind": "feature-flag", + "values": ["off", "on"], + "sourceRef": "fixture", + "note": "Remote-backed flag" + } + ], + "combinations": [ + { "id": "off", "values": { "flag:remote": "off" }, "selected": true } + ], + "oracle": {}, + "implementation": { "mode": "clean" }, + "bounds": { + "maxDimensions": 4, + "maxCombinations": 4, + "timeoutMs": 5000, + "targetKind": "fixture-local", + "allowRemoteFlagProvider": false + }, + "remoteFlagProvider": { "configured": false, "contacted": true }, + "cleanup": { "plan": "n/a" } +} diff --git a/packages/knowledge/test/fixtures/state-space/repair-loop.json b/packages/knowledge/test/fixtures/state-space/repair-loop.json new file mode 100644 index 0000000..b420dfb --- /dev/null +++ b/packages/knowledge/test/fixtures/state-space/repair-loop.json @@ -0,0 +1,31 @@ +{ + "id": "uat-state-6-repair", + "kind": "explicit-matrix", + "seed": 66, + "dimensions": [ + { + "id": "cache", + "kind": "cache-state", + "values": ["cold", "warm"], + "sourceRef": "fixture", + "note": "Cache states after repair" + } + ], + "combinations": [ + { "id": "cold", "values": { "cache": "cold" }, "selected": true }, + { "id": "warm", "values": { "cache": "warm" }, "selected": true } + ], + "oracle": { "cacheKey": "profile:1", "expectedReadValue": "new" }, + "implementation": { "mode": "repaired", "writeValue": "new", "cachedValue": "new" }, + "bounds": { + "maxDimensions": 4, + "maxCombinations": 8, + "timeoutMs": 5000, + "targetKind": "fixture-local" + }, + "cleanup": { "plan": "flush fixture cache" }, + "repair": { + "durableRegressionTestId": "test/cache-invalidation.regression.test.ts", + "revalidated": true + } +} diff --git a/packages/knowledge/test/fixtures/state-space/stale-cache.json b/packages/knowledge/test/fixtures/state-space/stale-cache.json new file mode 100644 index 0000000..51b9614 --- /dev/null +++ b/packages/knowledge/test/fixtures/state-space/stale-cache.json @@ -0,0 +1,27 @@ +{ + "id": "uat-state-1-stale-cache", + "kind": "explicit-matrix", + "seed": 11, + "dimensions": [ + { + "id": "cache", + "kind": "cache-state", + "values": ["cold", "warm"], + "sourceRef": "fixture", + "note": "Cold vs warm cache after write" + } + ], + "combinations": [ + { "id": "cold", "values": { "cache": "cold" }, "selected": true }, + { "id": "warm", "values": { "cache": "warm" }, "selected": true } + ], + "oracle": { "cacheKey": "profile:1", "expectedReadValue": "new" }, + "implementation": { "mode": "stale-cache", "writeValue": "new", "cachedValue": "old" }, + "bounds": { + "maxDimensions": 4, + "maxCombinations": 8, + "timeoutMs": 5000, + "targetKind": "fixture-local" + }, + "cleanup": { "plan": "flush fixture cache" } +} diff --git a/packages/knowledge/test/state-space-safety.test.ts b/packages/knowledge/test/state-space-safety.test.ts new file mode 100644 index 0000000..4148ce0 --- /dev/null +++ b/packages/knowledge/test/state-space-safety.test.ts @@ -0,0 +1,104 @@ +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { analyzeStateSpaceSafety } from "../src/index"; + +const fixtures = join(dirname(fileURLToPath(import.meta.url)), "fixtures", "state-space"); +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("UAT state-space safety (#688)", () => { + it("UAT-STATE-1: changed write path leaves stale cache; cold/warm comparison confirms it", () => { + const root = tempRoot(); + const report = analyzeStateSpaceSafety({ + rootDir: root, + experimentFile: join(fixtures, "stale-cache.json"), + generatedAt: "2026-08-06T00:00:00.000Z" + }); + expect(report.oracle?.combinationResults.find((item) => item.combinationId === "warm")?.status).toBe("failed"); + expect(report.oracle?.combinationResults.find((item) => item.combinationId === "cold")?.status).toBe("passed"); + expect(report.verdict).toBe("confirmed-regression"); + expect(report.fullyVerified).toBe(false); + }); + + it("UAT-STATE-2: two flags pass independently but fail in one pairwise combination", () => { + const root = tempRoot(); + const report = analyzeStateSpaceSafety({ + rootDir: root, + experimentFile: join(fixtures, "flag-pair.json") + }); + expect(report.oracle?.combinationResults.find((item) => item.combinationId === "alpha-on")?.status).toBe("passed"); + expect(report.oracle?.combinationResults.find((item) => item.combinationId === "beta-on")?.status).toBe("passed"); + expect(report.oracle?.combinationResults.find((item) => item.combinationId === "both-on")?.status).toBe("failed"); + expect(report.verdict).toBe("confirmed-regression"); + }); + + it("UAT-STATE-3: clean default/off/on matrix passes without unrelated combinations", () => { + const root = tempRoot(); + const report = analyzeStateSpaceSafety({ + rootDir: root, + experimentFile: join(fixtures, "clean-matrix.json") + }); + expect(report.verdict).toBe("passed-oracle"); + expect(report.combinations).toHaveLength(2); + expect(report.combinations.every((item) => item.selected)).toBe(true); + }); + + it("UAT-STATE-4: report clearly states tested and untested state coverage", () => { + const root = tempRoot(); + const report = analyzeStateSpaceSafety({ + rootDir: root, + experimentFile: join(fixtures, "coverage.json") + }); + expect(report.coverage.exhaustive).toBe(false); + expect(report.coverage.prunedCount).toBeGreaterThan(0); + expect(report.coverage.untestedCount).toBeGreaterThan(0); + expect(report.investigationTasks.join(" ")).toMatch(/Coverage:/); + }); + + it("UAT-STATE-5: remote flag provider is never contacted without explicit configuration", () => { + const root = tempRoot(); + const report = analyzeStateSpaceSafety({ + rootDir: root, + experimentFile: join(fixtures, "remote-provider.json") + }); + expect(report.verdict).toBe("provider-blocked"); + expect(report.safety.remoteFlagProviderContacted).toBe(true); + expect(report.safety.commandsExecuted).toBe(false); + expect(report.blockers.join(" ")).toMatch(/Remote flag provider/i); + }); + + it("UAT-STATE-6: repair loop fixes behavior and reruns the same state matrix", () => { + const root = tempRoot(); + write(root, "src/cache/profile.ts", "export function getProfile() { return cache.get('profile'); /* featureFlag */ }\n"); + const report = analyzeStateSpaceSafety({ + rootDir: root, + experimentFile: join(fixtures, "repair-loop.json"), + surfaceFiles: ["src/cache/profile.ts"] + }); + expect(report.verdict).toBe("passed-oracle"); + expect(report.treeStatus).toBe("revalidated-fixture"); + expect(report.repairTasks.some((task) => task.durableRegressionTestId === "test/cache-invalidation.regression.test.ts")).toBe(true); + expect(report.candidates.some((item) => item.kind === "cache-state" || item.kind === "feature-flag")).toBe(true); + expect(report.extensionBoundaries.map((item) => item.id)).toEqual( + expect.arrayContaining(["launchdarkly", "unleash", "redis-cache", "config-flags"]) + ); + }); +}); + +function tempRoot(): string { + const root = join(tmpdir(), `codedecay-state-space-${Date.now()}-${Math.random().toString(16).slice(2)}`); + mkdirSync(root, { recursive: true }); + roots.push(root); + return root; +} + +function write(root: string, relativePath: string, contents: string): void { + const absolute = join(root, relativePath); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, contents, "utf8"); +} diff --git a/packages/mcp/src/handlers/state-space-safety.ts b/packages/mcp/src/handlers/state-space-safety.ts new file mode 100644 index 0000000..0025b0d --- /dev/null +++ b/packages/mcp/src/handlers/state-space-safety.ts @@ -0,0 +1,33 @@ +import { resolve } from "node:path"; +import { + analyzeStateSpaceSafety, + renderStateSpaceSafetyMarkdown +} from "@submuxhq/codedecay-knowledge"; +import type { StartMcpServerOptions } from "../server/types"; + +export interface StateSpaceSafetyToolInput { + cwd?: string | undefined; + format?: "markdown" | "json" | undefined; + experimentFile?: string | undefined; + surfaceFiles?: string[] | undefined; + targetKind?: "unspecified" | "fixture-local" | "disposable-local" | "remote-unapproved" | "production-like" | undefined; + cleanupPlan?: string | undefined; +} + +export async function runStateSpaceSafetyTool( + options: StartMcpServerOptions, + input: StateSpaceSafetyToolInput +): Promise { + const rootDir = resolve(options.cwd ?? process.cwd(), input.cwd ?? "."); + const report = analyzeStateSpaceSafety({ + rootDir, + experimentFile: input.experimentFile, + surfaceFiles: input.surfaceFiles, + targetKind: input.targetKind, + cleanupPlan: input.cleanupPlan + }); + if ((input.format ?? "markdown") === "json") { + return JSON.stringify(report, null, 2); + } + return renderStateSpaceSafetyMarkdown(report); +} diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 82d9c61..ecee466 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -32,6 +32,7 @@ import { runServiceTopologyTool } from "./handlers/service-topology"; import { runRuntimeEvidenceTool } from "./handlers/runtime-evidence"; import { runMigrationSafetyTool } from "./handlers/migration-safety"; import { runConcurrencySafetyTool } from "./handlers/concurrency-safety"; +import { runStateSpaceSafetyTool } from "./handlers/state-space-safety"; import type { StartMcpServerOptions } from "./server/types"; import { registerCodeDecayMcpTools } from "./tools/registry"; @@ -61,6 +62,7 @@ export { runServiceTopologyTool } from "./handlers/service-topology"; export { runRuntimeEvidenceTool } from "./handlers/runtime-evidence"; export { runMigrationSafetyTool } from "./handlers/migration-safety"; export { runConcurrencySafetyTool } from "./handlers/concurrency-safety"; +export { runStateSpaceSafetyTool } from "./handlers/state-space-safety"; export { runProductFailuresTool, runProductPlanTool, @@ -97,6 +99,7 @@ export function createCodeDecayMcpServer(options: StartMcpServerOptions): McpSer runtimeEvidence: (input) => runRuntimeEvidenceTool(options, input), migrationSafety: (input) => runMigrationSafetyTool(options, input), concurrencySafety: (input) => runConcurrencySafetyTool(options, input), + stateSpaceSafety: (input) => runStateSpaceSafetyTool(options, input), agentInvestigation: (input) => runAgentInvestigationTool(options, input), scopeCheck: (input) => runScopeCheckTool(options, input), designContractCheck: (input) => runDesignContractCheckTool(options, input), diff --git a/packages/mcp/src/tools/register-analysis.ts b/packages/mcp/src/tools/register-analysis.ts index ec63cfc..aa0a6ca 100644 --- a/packages/mcp/src/tools/register-analysis.ts +++ b/packages/mcp/src/tools/register-analysis.ts @@ -16,7 +16,8 @@ import { serviceTopologyToolSchema, runtimeEvidenceToolSchema, migrationSafetyToolSchema, - concurrencySafetyToolSchema + concurrencySafetyToolSchema, + stateSpaceSafetyToolSchema } from "./schemas"; import type { AgentPreflightToolInput, @@ -35,7 +36,8 @@ import type { ServiceTopologyToolInput, RuntimeEvidenceToolInput, MigrationSafetyToolInput, - ConcurrencySafetyToolInput + ConcurrencySafetyToolInput, + StateSpaceSafetyToolInput } from "./types"; export function registerAnalysisMcpTools(server: McpServer, handlers: CodeDecayMcpToolHandlers): void { @@ -158,6 +160,13 @@ export function registerAnalysisMcpTools(server: McpServer, handlers: CodeDecayM async (input) => textResult(handlers.concurrencySafety(input as ConcurrencySafetyToolInput)) ); + server.tool( + "state_space_safety", + "Evaluate bounded cache/feature-flag state matrices with coverage accounting. Does not contact remote flag providers without explicit configuration.", + stateSpaceSafetyToolSchema, + async (input) => textResult(handlers.stateSpaceSafety(input as StateSpaceSafetyToolInput)) + ); + server.tool( "scope_check", "Return a deterministic in-scope/out-of-scope verdict for the current PR or working tree.", diff --git a/packages/mcp/src/tools/registry.ts b/packages/mcp/src/tools/registry.ts index f2271b7..1b84ee4 100644 --- a/packages/mcp/src/tools/registry.ts +++ b/packages/mcp/src/tools/registry.ts @@ -23,7 +23,8 @@ import type { ServiceTopologyToolInput, RuntimeEvidenceToolInput, MigrationSafetyToolInput, - ConcurrencySafetyToolInput + ConcurrencySafetyToolInput, + StateSpaceSafetyToolInput } from "./types"; export interface CodeDecayMcpToolHandlers { @@ -43,6 +44,7 @@ export interface CodeDecayMcpToolHandlers { runtimeEvidence(input: RuntimeEvidenceToolInput): string | Promise; migrationSafety(input: MigrationSafetyToolInput): string | Promise; concurrencySafety(input: ConcurrencySafetyToolInput): string | Promise; + stateSpaceSafety(input: StateSpaceSafetyToolInput): string | Promise; agentInvestigation(input: AgentInvestigationToolInput): string | Promise; scopeCheck(input: ScopeCheckToolInput): string | Promise; designContractCheck(input: DesignContractCheckToolInput): string | Promise; diff --git a/packages/mcp/src/tools/schemas.ts b/packages/mcp/src/tools/schemas.ts index 24041ae..f1001af 100644 --- a/packages/mcp/src/tools/schemas.ts +++ b/packages/mcp/src/tools/schemas.ts @@ -169,6 +169,18 @@ export const concurrencySafetyToolSchema = { cleanupPlan: z.string().optional().describe("Disposable target cleanup plan.") }; +export const stateSpaceSafetyToolSchema = { + cwd: cwdSchema, + format: formatSchema, + experimentFile: z.string().optional().describe("Repo-local state-space experiment JSON fixture."), + surfaceFiles: z.array(z.string()).optional().describe("Source files to scan for cache/flag dimensions."), + targetKind: z + .enum(["unspecified", "fixture-local", "disposable-local", "remote-unapproved", "production-like"]) + .optional() + .describe("State-space experiment target classification."), + cleanupPlan: z.string().optional().describe("Disposable target cleanup plan.") +}; + export const agentSessionToolSchema = { cwd: cwdSchema, operation: z.enum(["start", "context", "checkpoint", "finish"]).describe("Session lifecycle operation."), diff --git a/packages/mcp/src/tools/types.ts b/packages/mcp/src/tools/types.ts index 1491afd..ee9f92c 100644 --- a/packages/mcp/src/tools/types.ts +++ b/packages/mcp/src/tools/types.ts @@ -95,6 +95,15 @@ export interface ConcurrencySafetyToolInput { cleanupPlan?: string | undefined; } +export interface StateSpaceSafetyToolInput { + cwd?: string | undefined; + format?: "markdown" | "json" | undefined; + experimentFile?: string | undefined; + surfaceFiles?: string[] | undefined; + targetKind?: "unspecified" | "fixture-local" | "disposable-local" | "remote-unapproved" | "production-like" | undefined; + cleanupPlan?: string | undefined; +} + export interface AgentSessionToolInput { cwd?: string | undefined; operation: "start" | "context" | "checkpoint" | "finish"; diff --git a/packages/mcp/test/mcp-state-space-safety.test.ts b/packages/mcp/test/mcp-state-space-safety.test.ts new file mode 100644 index 0000000..ccd4669 --- /dev/null +++ b/packages/mcp/test/mcp-state-space-safety.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { copyFileSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runStateSpaceSafetyTool } from "../src/index"; +import { createRepo } from "./helpers/mcp"; + +const fixtures = join(dirname(fileURLToPath(import.meta.url)), "../../knowledge/test/fixtures/state-space"); + +describe("MCP state_space_safety tool", () => { + it("blocks unconfigured remote flag provider contact", async () => { + const repo = createRepo({ "README.md": "# fixture\n" }); + mkdirSync(join(repo, "experiments"), { recursive: true }); + copyFileSync(join(fixtures, "remote-provider.json"), join(repo, "experiments", "remote.json")); + const output = await runStateSpaceSafetyTool( + { cwd: repo }, + { format: "json", experimentFile: "experiments/remote.json" } + ); + const report = JSON.parse(output) as { + verdict: string; + safety: { commandsExecuted: boolean; remoteFlagProviderContacted: boolean }; + }; + expect(report.verdict).toBe("provider-blocked"); + expect(report.safety.commandsExecuted).toBe(false); + expect(report.safety.remoteFlagProviderContacted).toBe(true); + }); +});