From c5e67673878ad977866cbf3ea30b2f19d61204d9 Mon Sep 17 00:00:00 2001 From: kunaldhongade Date: Thu, 6 Aug 2026 20:39:04 +0530 Subject: [PATCH] feat(runtime): close local telemetry evidence with UAT-RUNTIME coverage for #685 Add downstream/budget correlation, cited investigation tasks, historical non-proof semantics, MCP runtime_evidence, and local-only provider defaults without network calls. --- docs/runtime.md | 29 ++ .../cli/src/docs/command-docs/analysis.ts | 5 +- packages/knowledge/src/index.ts | 8 +- packages/knowledge/src/runtime/ingest.ts | 417 +++++++++++++++--- packages/knowledge/src/runtime/render.ts | 43 +- packages/knowledge/src/runtime/types.ts | 44 +- .../knowledge/test/runtime-evidence.test.ts | 201 ++++++++- packages/mcp/src/handlers/runtime-evidence.ts | 39 ++ packages/mcp/src/index.ts | 3 + packages/mcp/src/tools/register-analysis.ts | 13 +- packages/mcp/src/tools/registry.ts | 4 +- packages/mcp/src/tools/schemas.ts | 10 + packages/mcp/src/tools/types.ts | 10 + .../mcp/test/mcp-runtime-evidence.test.ts | 55 +++ 14 files changed, 807 insertions(+), 74 deletions(-) create mode 100644 docs/runtime.md create mode 100644 packages/mcp/src/handlers/runtime-evidence.ts create mode 100644 packages/mcp/test/mcp-runtime-evidence.test.ts diff --git a/docs/runtime.md b/docs/runtime.md new file mode 100644 index 0000000..1d69f71 --- /dev/null +++ b/docs/runtime.md @@ -0,0 +1,29 @@ +# Runtime evidence + +CodeDecay can ingest **local** OpenTelemetry JSON exports and structured +error/deployment event files as read-only engineering evidence. + +## What it can establish + +- Which services/routes appear in a supplied export window +- Observed latency/error counts for those operations +- Declared downstream topology neighbors (`calls`/`consumes`) and latency budgets +- Historical vs current-revision trust labels +- Cited investigation tasks for agents (never merge-safe proof by themselves) + +## What it cannot establish + +- That the current head revision is safe +- Absence of failures when the export is sampled +- Production state without an explicit future provider adapter and command intent +- Unredacted secret/PII payload contents (those are stripped before persistence) + +## Defaults + +- Provider kind: `local-artifact` only +- Zero network calls when no remote provider is configured +- Artifact: `.codedecay/local/runtime-evidence.json` +- CLI: `codedecay runtime --telemetry ... --errors ... --topology ...` +- MCP: `runtime_evidence` + +Remote SaaS providers are intentionally out of this slice. diff --git a/packages/cli/src/docs/command-docs/analysis.ts b/packages/cli/src/docs/command-docs/analysis.ts index 4ecea7d..a8a7f30 100644 --- a/packages/cli/src/docs/command-docs/analysis.ts +++ b/packages/cli/src/docs/command-docs/analysis.ts @@ -36,7 +36,10 @@ export const ANALYSIS_COMMAND_DOCS: Record = { "codedecay runtime --telemetry .codedecay/runtime/traces.json --head-revision $(git rev-parse HEAD)", "codedecay runtime --errors .codedecay/runtime/errors.json --format json" ], - notes: ["Inputs must resolve inside the repository. The command performs no network calls or project command execution."] + notes: [ + "Inputs must resolve inside the repository. The command performs no network calls or project command execution.", + "Historical or sampled runtime evidence cannot prove the current tree safe. See docs/runtime.md." + ] }, topology: { name: "topology", diff --git a/packages/knowledge/src/index.ts b/packages/knowledge/src/index.ts index ebf57a0..86f839f 100644 --- a/packages/knowledge/src/index.ts +++ b/packages/knowledge/src/index.ts @@ -67,7 +67,7 @@ export { SERVICE_TOPOLOGY_NODE_KINDS, SERVICE_TOPOLOGY_SCHEMA_VERSION } from "./topology/types"; -export { ingestRuntimeEvidence } from "./runtime/ingest"; +export { ingestRuntimeEvidence, persistRuntimeEvidenceArtifact, RUNTIME_EVIDENCE_ARTIFACT_PATH } from "./runtime/ingest"; export { analyzeMigrationSafety } from "./migration/analyze"; export type { AnalyzeMigrationSafetyOptions } from "./migration/analyze"; export { renderMigrationSafetyMarkdown } from "./migration/render"; @@ -84,11 +84,15 @@ export type { IngestRuntimeEvidenceOptions } from "./runtime/ingest"; export { renderRuntimeEvidenceMarkdown } from "./runtime/render"; export { RUNTIME_EVIDENCE_SCHEMA_VERSION } from "./runtime/types"; export type { + RuntimeDeploymentEvidence, RuntimeErrorEvidence, RuntimeEvidenceReport, RuntimeEvidenceSource, RuntimeEvidenceTrust, - RuntimeOperationEvidence + RuntimeInvestigationTask, + RuntimeOperationEvidence, + RuntimeProviderConfig, + RuntimeProviderKind } from "./runtime/types"; export type { ServiceTopologyConfidence, diff --git a/packages/knowledge/src/runtime/ingest.ts b/packages/knowledge/src/runtime/ingest.ts index ab88f83..18c4723 100644 --- a/packages/knowledge/src/runtime/ingest.ts +++ b/packages/knowledge/src/runtime/ingest.ts @@ -1,21 +1,25 @@ import { createHash } from "node:crypto"; -import { existsSync, readFileSync, realpathSync, statSync } from "node:fs"; -import { resolve } from "node:path"; -import type { ServiceTopologyGraph } from "../topology/types"; +import { existsSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import type { ServiceTopologyGraph, ServiceTopologyNode } from "../topology/types"; import { RUNTIME_EVIDENCE_SCHEMA_VERSION, + type RuntimeDeploymentEvidence, type RuntimeErrorEvidence, type RuntimeEvidenceReport, type RuntimeEvidenceSource, type RuntimeEvidenceTrust, - type RuntimeOperationEvidence + type RuntimeInvestigationTask, + type RuntimeOperationEvidence, + type RuntimeProviderConfig } from "./types"; -const SENSITIVE_KEY = /authorization|cookie|password|secret|token|api[-_]?key|request\.body|request_body|user\.email|client\.address/i; +const SENSITIVE_KEY = /authorization|cookie|password|secret|token|api[-_]?key|request\.body|request_body|user\.email|client\.address|ssn|phone/i; const SENSITIVE_VALUE = /(?:bearer\s+[a-z0-9._~+/-]+=*|\b(?:sk|ghp|github_pat)_[a-z0-9_-]+|[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,})/gi; const DEFAULT_MAX_SPANS = 5_000; const DEFAULT_MAX_OPERATIONS = 200; const DEFAULT_MAX_INPUT_BYTES = 10 * 1024 * 1024; +export const RUNTIME_EVIDENCE_ARTIFACT_PATH = ".codedecay/local/runtime-evidence.json"; export interface IngestRuntimeEvidenceOptions { rootDir: string; @@ -24,10 +28,12 @@ export interface IngestRuntimeEvidenceOptions { headRevision?: string | undefined; environment?: string | undefined; topology?: ServiceTopologyGraph | undefined; + provider?: RuntimeProviderConfig | undefined; maxSpans?: number | undefined; maxOperations?: number | undefined; maxInputBytes?: number | undefined; generatedAt?: string | undefined; + persist?: boolean | undefined; } interface MutableStats { @@ -43,6 +49,7 @@ interface SpanRecord { route?: string | undefined; environment?: string | undefined; revision?: string | undefined; + peerService?: string | undefined; latencyMs: number; error: boolean; sampled: boolean; @@ -51,28 +58,59 @@ interface SpanRecord { export function ingestRuntimeEvidence(options: IngestRuntimeEvidenceOptions): RuntimeEvidenceReport { const rootDir = realpathSync(options.rootDir); + const provider = normalizeProvider(options.provider); const stats: MutableStats = { spansRead: 0, spansDroppedByBounds: 0, malformedRecords: 0, redactedValues: 0 }; const sources: RuntimeEvidenceSource[] = []; - const limitations: string[] = []; + const limitations: string[] = [ + "Runtime evidence is read-only local artifact ingestion by default; it never proves a head revision safe by itself." + ]; const maxInputBytes = options.maxInputBytes ?? DEFAULT_MAX_INPUT_BYTES; - const spans = options.otlpPath ? loadOtlp(rootDir, options.otlpPath, options.environment, options.maxSpans ?? DEFAULT_MAX_SPANS, maxInputBytes, stats, sources) : []; - const errors = options.errorsPath ? loadErrors(rootDir, options.errorsPath, options.headRevision, options.environment, maxInputBytes, stats, sources) : []; - if (!options.otlpPath) limitations.push("No local OpenTelemetry export was configured; runtime path exposure is unavailable."); - if (!options.errorsPath) limitations.push("No structured error export was configured; deployment-correlated errors are unavailable."); - const operations = aggregateOperations(spans, options.headRevision, options.topology, options.maxOperations ?? DEFAULT_MAX_OPERATIONS, stats); - const investigationTasks = createTasks(operations, errors); - if (stats.malformedRecords > 0) limitations.push(`${stats.malformedRecords} malformed runtime record(s) were ignored; the report may be incomplete.`); - if (stats.spansDroppedByBounds > 0) limitations.push(`${stats.spansDroppedByBounds} runtime record(s) were omitted by cardinality bounds.`); + const spans = options.otlpPath + ? loadOtlp(rootDir, options.otlpPath, options.environment, options.maxSpans ?? DEFAULT_MAX_SPANS, maxInputBytes, stats, sources) + : []; + const loadedErrors = options.errorsPath + ? loadErrorsAndDeployments(rootDir, options.errorsPath, options.headRevision, options.environment, maxInputBytes, stats, sources) + : { errors: [] as RuntimeErrorEvidence[], deployments: [] as RuntimeDeploymentEvidence[] }; + if (!options.otlpPath) { + limitations.push("No local OpenTelemetry export was configured; runtime path exposure is unavailable."); + } + if (!options.errorsPath) { + limitations.push("No structured error export was configured; deployment-correlated errors are unavailable."); + } + limitations.push("No remote observability provider is configured; zero network calls were made."); - return { + const operations = aggregateOperations( + spans, + options.headRevision, + options.topology, + options.maxOperations ?? DEFAULT_MAX_OPERATIONS, + stats + ); + const errors = annotateMatchingDeployments(loadedErrors.errors, loadedErrors.deployments); + const investigationTasks = createTasks(operations, errors, loadedErrors.deployments); + if (stats.malformedRecords > 0) { + limitations.push(`${stats.malformedRecords} malformed runtime record(s) were ignored; the report may be incomplete.`); + } + if (stats.spansDroppedByBounds > 0) { + limitations.push(`${stats.spansDroppedByBounds} runtime record(s) were omitted by cardinality bounds.`); + } + if (operations.every((item) => item.trust !== "current-revision") && operations.length > 0) { + limitations.push("Only historical or unmatched runtime operations were ingested; they cannot prove the current tree."); + } + + const report: RuntimeEvidenceReport = { tool: "CodeDecay", schemaVersion: RUNTIME_EVIDENCE_SCHEMA_VERSION, generatedAt: options.generatedAt ?? new Date().toISOString(), headRevision: options.headRevision, + provider, sources, operations, errors, + deployments: loadedErrors.deployments, investigationTasks, + investigationTaskTitles: investigationTasks.map((task) => task.title), + canProveCurrentTree: false, limitations, stats, safety: { @@ -83,39 +121,96 @@ export function ingestRuntimeEvidence(options: IngestRuntimeEvidenceOptions): Ru secretsPersisted: false } }; + + if (options.persist !== false) { + persistRuntimeEvidenceArtifact(rootDir, report); + } + return report; +} + +export function persistRuntimeEvidenceArtifact(rootDir: string, report: RuntimeEvidenceReport): string { + const outputPath = resolve(rootDir, RUNTIME_EVIDENCE_ARTIFACT_PATH); + mkdirSync(dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + return RUNTIME_EVIDENCE_ARTIFACT_PATH; +} + +function normalizeProvider(provider: RuntimeProviderConfig | undefined): RuntimeProviderConfig { + if (!provider) { + return { kind: "local-artifact" }; + } + if (provider.kind !== "local-artifact") { + throw new Error(`Unsupported runtime provider kind "${String((provider as { kind?: string }).kind)}". Only local-artifact is enabled.`); + } + return { + kind: "local-artifact", + endpointOrFile: provider.endpointOrFile, + environmentAllowlist: provider.environmentAllowlist, + queryBudgetMs: provider.queryBudgetMs, + secretEnvNames: provider.secretEnvNames ?? [] + }; } -function loadOtlp(rootDir: string, path: string, environment: string | undefined, maxSpans: number, maxInputBytes: number, stats: MutableStats, sources: RuntimeEvidenceSource[]): SpanRecord[] { +function loadOtlp( + rootDir: string, + path: string, + environment: string | undefined, + maxSpans: number, + maxInputBytes: number, + stats: MutableStats, + sources: RuntimeEvidenceSource[] +): SpanRecord[] { const sourcePath = resolveInput(rootDir, path); const value = parseLocalJson(sourcePath, maxInputBytes, stats); const resourceSpans = recordArray(value, "resourceSpans", stats); const records: SpanRecord[] = []; let sampled = false; + let collectionStart: string | undefined; + let collectionEnd: string | undefined; for (const resourceItem of resourceSpans) { const resourceSpan = asRecord(resourceItem); - if (!resourceSpan) { stats.malformedRecords += 1; continue; } + if (!resourceSpan) { + stats.malformedRecords += 1; + continue; + } const resourceAttributes = attributes(asRecord(resourceSpan.resource)?.attributes, stats); const service = stringAttribute(resourceAttributes, "service.name") ?? "unknown-service"; - const revision = stringAttribute(resourceAttributes, "service.version") ?? stringAttribute(resourceAttributes, "vcs.ref.head.revision"); + const revision = + stringAttribute(resourceAttributes, "service.version") ?? + stringAttribute(resourceAttributes, "vcs.ref.head.revision"); const spanEnvironment = stringAttribute(resourceAttributes, "deployment.environment.name") ?? environment; for (const scopeItem of recordArray(resourceSpan, "scopeSpans", stats)) { const scopeSpan = asRecord(scopeItem); - if (!scopeSpan) { stats.malformedRecords += 1; continue; } + if (!scopeSpan) { + stats.malformedRecords += 1; + continue; + } for (const spanItem of recordArray(scopeSpan, "spans", stats)) { stats.spansRead += 1; - if (records.length >= maxSpans) { stats.spansDroppedByBounds += 1; continue; } + if (records.length >= maxSpans) { + stats.spansDroppedByBounds += 1; + continue; + } const span = asRecord(spanItem); - if (!span || typeof span.name !== "string") { stats.malformedRecords += 1; continue; } + if (!span || typeof span.name !== "string") { + stats.malformedRecords += 1; + continue; + } const spanAttributes = attributes(span.attributes, stats); const spanFlags = numberValue(span.flags); const spanSampled = spanFlags !== undefined && (spanFlags & 1) === 1; sampled ||= spanSampled; + const startMs = unixNanoToIso(span.startTimeUnixNano); + const endMs = unixNanoToIso(span.endTimeUnixNano); + if (startMs && (!collectionStart || startMs < collectionStart)) collectionStart = startMs; + if (endMs && (!collectionEnd || endMs > collectionEnd)) collectionEnd = endMs; records.push({ service, operation: redactText(span.name, stats), route: stripQuery(stringAttribute(spanAttributes, "http.route") ?? stringAttribute(spanAttributes, "url.path")), environment: spanEnvironment, revision, + peerService: stringAttribute(spanAttributes, "peer.service") ?? stringAttribute(spanAttributes, "net.peer.name"), latencyMs: durationMs(span.startTimeUnixNano, span.endTimeUnixNano), error: asRecord(span.status)?.code === 2 || Boolean(stringAttribute(spanAttributes, "error.type")), sampled: spanSampled, @@ -124,18 +219,42 @@ function loadOtlp(rootDir: string, path: string, environment: string | undefined } } } - sources.push({ kind: "otlp-json", path, environment, sampled, redacted: true, limitations: sampled ? ["Trace export is sampled and cannot prove absence of unobserved paths."] : [] }); + sources.push({ + kind: "otlp-json", + path, + environment, + collectionStart, + collectionEnd, + sampled, + redacted: true, + limitations: sampled ? ["Trace export is sampled and cannot prove absence of unobserved paths."] : [] + }); return records; } -function loadErrors(rootDir: string, path: string, headRevision: string | undefined, environment: string | undefined, maxInputBytes: number, stats: MutableStats, sources: RuntimeEvidenceSource[]): RuntimeErrorEvidence[] { +function loadErrorsAndDeployments( + rootDir: string, + path: string, + headRevision: string | undefined, + environment: string | undefined, + maxInputBytes: number, + stats: MutableStats, + sources: RuntimeEvidenceSource[] +): { errors: RuntimeErrorEvidence[]; deployments: RuntimeDeploymentEvidence[] } { const value = parseLocalJson(resolveInput(rootDir, path), maxInputBytes, stats); const records = recordArray(value, "errors", stats); + const deploymentRecords = recordArray(value, "deployments", stats); if (records.length > 500) stats.spansDroppedByBounds += records.length - 500; + if (deploymentRecords.length > 100) stats.spansDroppedByBounds += deploymentRecords.length - 100; + const errors = records.slice(0, 500).flatMap((item, index) => { const error = asRecord(item); - if (!error || typeof error.service !== "string" || typeof error.message !== "string") { stats.malformedRecords += 1; return []; } + if (!error || typeof error.service !== "string" || typeof error.message !== "string") { + stats.malformedRecords += 1; + return []; + } const revision = optionalString(error.revision); + const trust = revisionTrust(revision, headRevision); const group = optionalString(error.group) ?? `error-${index + 1}`; return [{ evidenceId: evidenceId(["error", path, group, revision ?? "unknown"]), @@ -148,27 +267,89 @@ function loadErrors(rootDir: string, path: string, headRevision: string | undefi revision, firstSeen: validTimestamp(error.firstSeen), lastSeen: validTimestamp(error.lastSeen), - trust: revisionTrust(revision, headRevision), + trust, + provesCurrentTree: false as const, sourceRef: `${path}#error:${index + 1}`, - limitations: revision ? [] : ["Error export does not identify a deployment revision."] + limitations: [ + ...(revision ? [] : ["Error export does not identify a deployment revision."]), + ...(trust !== "current-revision" ? ["Historical or unmatched errors cannot prove the current tree."] : []) + ] } satisfies RuntimeErrorEvidence]; }); + + const deployments = deploymentRecords.slice(0, 100).flatMap((item, index) => { + const deployment = asRecord(item); + if (!deployment || typeof deployment.service !== "string" || typeof deployment.revision !== "string") { + stats.malformedRecords += 1; + return []; + } + const trust = revisionTrust(deployment.revision, headRevision); + return [{ + evidenceId: evidenceId(["deployment", path, deployment.service, deployment.revision]), + service: redactText(deployment.service, stats), + revision: deployment.revision, + environment: optionalString(deployment.environment) ?? environment, + deployedAt: validTimestamp(deployment.deployedAt), + trust, + sourceRef: `${path}#deployment:${index + 1}`, + limitations: trust !== "current-revision" + ? ["Deployment event does not match the current head revision."] + : [] + } satisfies RuntimeDeploymentEvidence]; + }); + sources.push({ kind: "structured-errors", path, environment, sampled: false, redacted: true, limitations: [] }); - return errors; + if (deployments.length > 0) { + sources.push({ + kind: "deployment-events", + path, + environment, + sampled: false, + redacted: true, + limitations: ["Deployment events are untrusted context until corroborated against the current tree."] + }); + } + return { errors, deployments }; +} + +function annotateMatchingDeployments( + errors: RuntimeErrorEvidence[], + deployments: RuntimeDeploymentEvidence[] +): RuntimeErrorEvidence[] { + return errors.map((error) => { + const match = deployments.find( + (deployment) => + deployment.service === error.service && + Boolean(error.revision) && + deployment.revision === error.revision + ); + return match ? { ...error, matchingDeploymentId: match.evidenceId } : error; + }); } -function aggregateOperations(spans: SpanRecord[], headRevision: string | undefined, topology: ServiceTopologyGraph | undefined, maxOperations: number, stats: MutableStats): RuntimeOperationEvidence[] { +function aggregateOperations( + spans: SpanRecord[], + headRevision: string | undefined, + topology: ServiceTopologyGraph | undefined, + maxOperations: number, + stats: MutableStats +): RuntimeOperationEvidence[] { const groups = new Map(); for (const span of spans) { const key = [span.service, span.operation, span.route ?? "", span.environment ?? "", span.revision ?? ""].join("\0"); const existing = groups.get(key); - if (existing) existing.push(span); else if (groups.size < maxOperations) groups.set(key, [span]); else stats.spansDroppedByBounds += 1; + if (existing) existing.push(span); + else if (groups.size < maxOperations) groups.set(key, [span]); + else stats.spansDroppedByBounds += 1; } return [...groups.values()].map((items) => { const first = items[0] as SpanRecord; - const topologyNodeIds = correlateTopology(topology, first.service, first.route); + const correlation = correlateTopology(topology, first.service, first.route, first.peerService); const trust = revisionTrust(first.revision, headRevision); const totalLatency = items.reduce((sum, item) => sum + item.latencyMs, 0); + const maxLatencyMs = Math.max(...items.map((item) => item.latencyMs)); + const latencyBudgetMs = correlation.latencyBudgetMs; + const budgetBreached = latencyBudgetMs !== undefined && maxLatencyMs > latencyBudgetMs; return { evidenceId: evidenceId(["operation", first.service, first.operation, first.route ?? "", first.revision ?? "unknown"]), service: first.service, @@ -178,51 +359,143 @@ function aggregateOperations(spans: SpanRecord[], headRevision: string | undefin revision: first.revision, spanCount: items.length, errorCount: items.filter((item) => item.error).length, - maxLatencyMs: Math.max(...items.map((item) => item.latencyMs)), + maxLatencyMs, averageLatencyMs: Math.round((totalLatency / items.length) * 100) / 100, + latencyBudgetMs, + budgetBreached, sampled: items.some((item) => item.sampled), trust, - topologyNodeIds, + provesCurrentTree: false as const, + topologyNodeIds: correlation.topologyNodeIds, + downstreamServiceIds: correlation.downstreamServiceIds, sourceRefs: items.slice(0, 20).map((item) => item.sourceRef), limitations: [ ...(items.some((item) => item.sampled) ? ["Sampled traces cannot prove absence of failures."] : []), - ...(trust !== "current-revision" ? ["Runtime evidence does not exactly match the current head revision."] : []) + ...(trust !== "current-revision" ? ["Runtime evidence does not exactly match the current head revision and cannot prove the current tree."] : []), + ...(budgetBreached ? [`Observed max latency ${maxLatencyMs}ms exceeds declared budget ${latencyBudgetMs}ms.`] : []) ] }; }).sort((left, right) => right.errorCount - left.errorCount || right.maxLatencyMs - left.maxLatencyMs || left.evidenceId.localeCompare(right.evidenceId)); } -function correlateTopology(topology: ServiceTopologyGraph | undefined, service: string, route: string | undefined): string[] { - if (!topology) return []; +function correlateTopology( + topology: ServiceTopologyGraph | undefined, + service: string, + route: string | undefined, + peerService: string | undefined +): { topologyNodeIds: string[]; downstreamServiceIds: string[]; latencyBudgetMs?: number | undefined } { + if (!topology) { + return { + topologyNodeIds: [], + downstreamServiceIds: peerService ? [`service:${peerService}`] : [] + }; + } const normalizedService = service.toLowerCase(); const normalizedRoute = route?.toLowerCase(); - return topology.nodes.filter((node) => { + const matched = topology.nodes.filter((node) => { const metadataRoute = typeof node.metadata?.route === "string" ? node.metadata.route.toLowerCase() : undefined; - return node.id.toLowerCase() === `service:${normalizedService}` || node.label.toLowerCase() === normalizedService || Boolean(normalizedRoute && metadataRoute === normalizedRoute); - }).map((node) => node.id).sort(); + return ( + node.id.toLowerCase() === `service:${normalizedService}` || + node.label.toLowerCase() === normalizedService || + Boolean(normalizedRoute && metadataRoute === normalizedRoute) + ); + }); + const topologyNodeIds = matched.map((node) => node.id).sort(); + const matchedIds = new Set(topologyNodeIds); + const downstream = new Set(); + if (peerService) downstream.add(`service:${peerService}`); + for (const edge of topology.edges) { + if (!matchedIds.has(edge.from)) continue; + if (edge.kind !== "calls" && edge.kind !== "consumes") continue; + const target = topology.nodes.find((node) => node.id === edge.to); + if (target && (target.kind === "service" || target.kind === "api" || target.kind === "deployment-unit")) { + downstream.add(target.id); + } + } + const latencyBudgetMs = firstLatencyBudget(matched); + return { + topologyNodeIds, + downstreamServiceIds: [...downstream].sort(), + latencyBudgetMs + }; } -function createTasks(operations: RuntimeOperationEvidence[], errors: RuntimeErrorEvidence[]): string[] { - return [ - ...operations.filter((item) => item.errorCount > 0).map((item) => `Reproduce ${item.errorCount} observed error span(s) for ${item.service} ${item.route ?? item.operation} against the current tree.`), - ...operations.filter((item) => item.maxLatencyMs >= 1_000).map((item) => `Verify the ${item.maxLatencyMs}ms runtime hotspot for ${item.service} ${item.route ?? item.operation} with a bounded local performance check.`), - ...errors.map((item) => `Investigate runtime error group ${item.group} (${item.count} event(s)) for ${item.service}; do not treat the export as current-tree proof.`) - ]; +function firstLatencyBudget(nodes: ServiceTopologyNode[]): number | undefined { + for (const node of nodes) { + const value = node.metadata?.latencyBudgetMs; + if (typeof value === "number" && Number.isFinite(value) && value > 0) return value; + } + return undefined; +} + +function createTasks( + operations: RuntimeOperationEvidence[], + errors: RuntimeErrorEvidence[], + deployments: RuntimeDeploymentEvidence[] +): RuntimeInvestigationTask[] { + const tasks: RuntimeInvestigationTask[] = []; + for (const item of operations.filter((entry) => entry.errorCount > 0)) { + tasks.push({ + evidenceId: evidenceId(["task", "error-spans", item.evidenceId]), + title: `Reproduce ${item.errorCount} observed error span(s) for ${item.service} ${item.route ?? item.operation}`, + detail: `Use trusted local execution against the current tree. Cited runtime evidence: ${item.evidenceId}. Downstream: ${item.downstreamServiceIds.join(", ") || "none declared"}.`, + citedEvidenceIds: [item.evidenceId], + priority: "high", + provesCurrentTree: false + }); + } + for (const item of operations.filter((entry) => entry.budgetBreached || entry.maxLatencyMs >= 1_000)) { + const budgetText = item.latencyBudgetMs + ? `budget ${item.latencyBudgetMs}ms (observed max ${item.maxLatencyMs}ms)` + : `${item.maxLatencyMs}ms hotspot`; + tasks.push({ + evidenceId: evidenceId(["task", "latency", item.evidenceId]), + title: `Verify latency ${budgetText} for ${item.service} ${item.route ?? item.operation}`, + detail: `Run a bounded local performance check. Downstream services: ${item.downstreamServiceIds.join(", ") || "none declared"}. Cited: ${item.evidenceId}.`, + citedEvidenceIds: [item.evidenceId, ...item.downstreamServiceIds.map((id) => evidenceId(["topology", id]))], + priority: item.budgetBreached ? "high" : "medium", + provesCurrentTree: false + }); + } + for (const item of errors) { + const cited = [item.evidenceId, ...(item.matchingDeploymentId ? [item.matchingDeploymentId] : [])]; + const deployment = deployments.find((entry) => entry.evidenceId === item.matchingDeploymentId); + tasks.push({ + evidenceId: evidenceId(["task", "error-group", item.evidenceId]), + title: `Investigate runtime error group ${item.group} for ${item.service}`, + detail: deployment + ? `Matching deployment ${deployment.evidenceId} at revision ${deployment.revision} correlates with this error export. Do not treat the export as current-tree proof. Cited: ${cited.join(", ")}.` + : `Do not treat the export as current-tree proof. Cited: ${item.evidenceId}.`, + citedEvidenceIds: cited, + priority: item.matchingDeploymentId ? "high" : "medium", + provesCurrentTree: false + }); + } + return uniqueBy(tasks, (task) => task.evidenceId).sort((left, right) => left.evidenceId.localeCompare(right.evidenceId)); } function resolveInput(rootDir: string, path: string): string { const lexical = resolve(rootDir, path); - if (lexical !== rootDir && !lexical.startsWith(`${rootDir}/`)) throw new Error(`Runtime evidence path must stay inside repository: ${path}`); + if (lexical !== rootDir && !lexical.startsWith(`${rootDir}/`)) { + throw new Error(`Runtime evidence path must stay inside repository: ${path}`); + } if (!existsSync(lexical)) throw new Error(`Runtime evidence file not found: ${path}`); const real = realpathSync(lexical); - if (real !== rootDir && !real.startsWith(`${rootDir}/`)) throw new Error(`Runtime evidence path must stay inside repository: ${path}`); + if (real !== rootDir && !real.startsWith(`${rootDir}/`)) { + throw new Error(`Runtime evidence path must stay inside repository: ${path}`); + } return real; } function parseLocalJson(path: string, maxInputBytes: number, stats: MutableStats): unknown { const size = statSync(path).size; if (size > maxInputBytes) throw new Error(`Runtime evidence file exceeds ${maxInputBytes} byte limit: ${path}`); - try { return JSON.parse(readFileSync(path, "utf8")) as unknown; } catch { stats.malformedRecords += 1; return {}; } + try { + return JSON.parse(readFileSync(path, "utf8")) as unknown; + } catch { + stats.malformedRecords += 1; + return {}; + } } function attributes(value: unknown, stats: MutableStats): Map { @@ -230,11 +503,20 @@ function attributes(value: unknown, stats: MutableStats): Map, key: st function redactText(value: string, stats: MutableStats): string { let redacted = stripQuery(value) ?? ""; - redacted = redacted.replace(SENSITIVE_VALUE, () => { stats.redactedValues += 1; return "[REDACTED]"; }); + redacted = redacted.replace(SENSITIVE_VALUE, () => { + stats.redactedValues += 1; + return "[REDACTED]"; + }); return redacted.slice(0, 500); } @@ -265,14 +550,30 @@ function durationMs(start: unknown, end: unknown): number { try { const duration = Number(BigInt(String(end ?? 0)) - BigInt(String(start ?? 0))) / 1_000_000; return Number.isFinite(duration) && duration >= 0 ? Math.round(duration * 100) / 100 : 0; - } catch { return 0; } + } catch { + return 0; + } +} + +function unixNanoToIso(value: unknown): string | undefined { + try { + const nanos = BigInt(String(value ?? "")); + const ms = Number(nanos / 1_000_000n); + if (!Number.isFinite(ms) || ms <= 0) return undefined; + return new Date(ms).toISOString(); + } catch { + return undefined; + } } function recordArray(value: unknown, key: string, stats: MutableStats): unknown[] { const record = asRecord(value); const items = record?.[key]; if (items === undefined) return []; - if (!Array.isArray(items)) { stats.malformedRecords += 1; return []; } + if (!Array.isArray(items)) { + stats.malformedRecords += 1; + return []; + } return items; } @@ -297,3 +598,13 @@ function validTimestamp(value: unknown): string | undefined { function evidenceId(parts: string[]): string { return `runtime:${createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 20)}`; } + +function uniqueBy(items: T[], key: (item: T) => string): T[] { + const seen = new Set(); + return items.filter((item) => { + const value = key(item); + if (seen.has(value)) return false; + seen.add(value); + return true; + }); +} diff --git a/packages/knowledge/src/runtime/render.ts b/packages/knowledge/src/runtime/render.ts index fe8631f..3a85beb 100644 --- a/packages/knowledge/src/runtime/render.ts +++ b/packages/knowledge/src/runtime/render.ts @@ -5,22 +5,57 @@ export function renderRuntimeEvidenceMarkdown(report: RuntimeEvidenceReport): st "## CodeDecay Runtime Evidence", "", `Head revision: \`${report.headRevision ?? "unknown"}\``, + `Provider: \`${report.provider.kind}\``, + `Can prove current tree: \`${report.canProveCurrentTree}\``, `Sources: ${report.sources.length}; spans read: ${report.stats.spansRead}; bounded drops: ${report.stats.spansDroppedByBounds}; malformed: ${report.stats.malformedRecords}`, "", "### Runtime Operations", "" ]; if (report.operations.length === 0) lines.push("No runtime operations were ingested.", ""); - for (const item of report.operations) lines.push(`- **${item.service} ${item.route ?? item.operation}**: ${item.spanCount} span(s), ${item.errorCount} error(s), max ${item.maxLatencyMs}ms; trust \`${item.trust}\`.`); + for (const item of report.operations) { + lines.push( + `- **${item.service} ${item.route ?? item.operation}** \`${item.evidenceId}\``, + ` - ${item.spanCount} span(s), ${item.errorCount} error(s), max ${item.maxLatencyMs}ms` + + (item.latencyBudgetMs ? ` / budget ${item.latencyBudgetMs}ms` : "") + + `; trust \`${item.trust}\`; provesCurrentTree \`${item.provesCurrentTree}\``, + ` - Downstream: ${item.downstreamServiceIds.map((id) => `\`${id}\``).join(", ") || "none declared"}` + ); + } lines.push("", "### Correlated Errors", ""); if (report.errors.length === 0) lines.push("No structured error groups were ingested.", ""); - for (const item of report.errors) lines.push(`- **${item.group}**: ${item.count} event(s) for ${item.service}; trust \`${item.trust}\`; source \`${item.sourceRef}\`.`); + for (const item of report.errors) { + lines.push( + `- **${item.group}** \`${item.evidenceId}\`: ${item.count} event(s) for ${item.service}; trust \`${item.trust}\`` + + (item.matchingDeploymentId ? `; matching deployment \`${item.matchingDeploymentId}\`` : "") + + `; source \`${item.sourceRef}\`.` + ); + } + lines.push("", "### Deployments", ""); + if (report.deployments.length === 0) lines.push("No deployment events were ingested.", ""); + for (const item of report.deployments) { + lines.push(`- **${item.service}@${item.revision}** \`${item.evidenceId}\`; trust \`${item.trust}\`.`); + } lines.push("", "### Investigation Tasks", ""); if (report.investigationTasks.length === 0) lines.push("No runtime investigation task was generated."); - for (const task of report.investigationTasks) lines.push(`- ${task}`); + for (const task of report.investigationTasks) { + lines.push( + `- **${task.title}** (${task.priority}) \`${task.evidenceId}\``, + ` - ${task.detail}`, + ` - Cited: ${task.citedEvidenceIds.map((id) => `\`${id}\``).join(", ")}` + ); + } lines.push("", "### Limitations", ""); if (report.limitations.length === 0) lines.push("No ingestion limitation was reported."); for (const limitation of report.limitations) lines.push(`- ${limitation}`); - lines.push("", "### Safety", "", "- Local artifact ingestion only; no network or command execution.", "- Sensitive attributes, query strings, authorization data, request bodies, tokens, and email addresses are redacted before report assembly.", "- Historical or sampled runtime evidence cannot prove the current tree safe.", ""); + lines.push( + "", + "### Safety", + "", + "- Local artifact ingestion only; no network or command execution.", + "- Sensitive attributes, query strings, authorization data, request bodies, tokens, and email addresses are redacted before report assembly.", + "- Historical or sampled runtime evidence cannot prove the current tree safe.", + "" + ); return `${lines.join("\n")}\n`; } diff --git a/packages/knowledge/src/runtime/types.ts b/packages/knowledge/src/runtime/types.ts index 9abb7a0..dad2a60 100644 --- a/packages/knowledge/src/runtime/types.ts +++ b/packages/knowledge/src/runtime/types.ts @@ -1,9 +1,10 @@ export const RUNTIME_EVIDENCE_SCHEMA_VERSION = 1 as const; export type RuntimeEvidenceTrust = "current-revision" | "historical" | "unmatched" | "inferred"; +export type RuntimeProviderKind = "local-artifact"; export interface RuntimeEvidenceSource { - kind: "otlp-json" | "structured-errors"; + kind: "otlp-json" | "structured-errors" | "deployment-events"; path: string; collectionStart?: string | undefined; collectionEnd?: string | undefined; @@ -24,9 +25,13 @@ export interface RuntimeOperationEvidence { errorCount: number; maxLatencyMs: number; averageLatencyMs: number; + latencyBudgetMs?: number | undefined; + budgetBreached: boolean; sampled: boolean; trust: RuntimeEvidenceTrust; + provesCurrentTree: false | true; topologyNodeIds: string[]; + downstreamServiceIds: string[]; sourceRefs: string[]; limitations: string[]; } @@ -42,20 +47,55 @@ export interface RuntimeErrorEvidence { revision?: string | undefined; firstSeen?: string | undefined; lastSeen?: string | undefined; + matchingDeploymentId?: string | undefined; trust: RuntimeEvidenceTrust; + provesCurrentTree: false | true; sourceRef: string; limitations: string[]; } +export interface RuntimeDeploymentEvidence { + evidenceId: string; + service: string; + revision: string; + environment?: string | undefined; + deployedAt?: string | undefined; + trust: RuntimeEvidenceTrust; + sourceRef: string; + limitations: string[]; +} + +export interface RuntimeInvestigationTask { + evidenceId: string; + title: string; + detail: string; + citedEvidenceIds: string[]; + priority: "high" | "medium" | "low"; + provesCurrentTree: false; +} + +export interface RuntimeProviderConfig { + kind: RuntimeProviderKind; + endpointOrFile?: string | undefined; + environmentAllowlist?: string[] | undefined; + queryBudgetMs?: number | undefined; + secretEnvNames?: string[] | undefined; +} + export interface RuntimeEvidenceReport { tool: "CodeDecay"; schemaVersion: typeof RUNTIME_EVIDENCE_SCHEMA_VERSION; generatedAt: string; headRevision?: string | undefined; + provider: RuntimeProviderConfig; sources: RuntimeEvidenceSource[]; operations: RuntimeOperationEvidence[]; errors: RuntimeErrorEvidence[]; - investigationTasks: string[]; + deployments: RuntimeDeploymentEvidence[]; + investigationTasks: RuntimeInvestigationTask[]; + /** @deprecated Prefer investigationTasks; kept as title strings for older consumers. */ + investigationTaskTitles: string[]; + canProveCurrentTree: false; limitations: string[]; stats: { spansRead: number; diff --git a/packages/knowledge/test/runtime-evidence.test.ts b/packages/knowledge/test/runtime-evidence.test.ts index d8da6b0..36fb0cf 100644 --- a/packages/knowledge/test/runtime-evidence.test.ts +++ b/packages/knowledge/test/runtime-evidence.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -10,6 +10,146 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +describe("UAT runtime evidence (#685)", () => { + it("UAT-RUNTIME-1: OTEL fixture maps changed route to downstream service and latency budget", () => { + const root = tempRoot(); + writeJson(join(root, "traces.json"), otlp([ + span("GET /checkout", "route-1", 0, { + "http.route": "/checkout", + "peer.service": "payments" + }, "0", "2500000000") + ], "api", "head")); + const topology = normalizeServiceTopologyGraph({ + schemaVersion: 1, + nodes: [ + node("service:api", "service", "api", { route: "/checkout", latencyBudgetMs: 500 }), + node("service:payments", "service", "payments") + ], + edges: [ + edge("edge:api-calls-payments", "service:api", "service:payments", "calls") + ], + limitations: [] + }); + + const report = ingestRuntimeEvidence({ + rootDir: root, + otlpPath: "traces.json", + headRevision: "head", + topology, + persist: false, + generatedAt: "2026-08-06T00:00:00.000Z" + }); + + expect(report.operations[0]).toMatchObject({ + route: "/checkout", + topologyNodeIds: ["service:api"], + downstreamServiceIds: expect.arrayContaining(["service:payments"]), + latencyBudgetMs: 500, + budgetBreached: true, + provesCurrentTree: false + }); + expect(report.investigationTasks.some((task) => task.title.includes("latency") && task.citedEvidenceIds.includes(report.operations[0]!.evidenceId))).toBe(true); + }); + + it("UAT-RUNTIME-2: matching deployment/error fixture raises a cited investigation task", () => { + const root = tempRoot(); + writeJson(join(root, "errors.json"), { + deployments: [{ service: "api", revision: "head", deployedAt: "2026-08-06T00:00:00.000Z" }], + errors: [{ service: "api", group: "checkout-timeout", message: "upstream timeout", revision: "head", count: 4 }] + }); + + const report = ingestRuntimeEvidence({ + rootDir: root, + errorsPath: "errors.json", + headRevision: "head", + persist: false + }); + + expect(report.errors[0]?.matchingDeploymentId).toBe(report.deployments[0]?.evidenceId); + const task = report.investigationTasks.find((item) => item.title.includes("checkout-timeout")); + expect(task?.citedEvidenceIds).toEqual(expect.arrayContaining([ + report.errors[0]!.evidenceId, + report.deployments[0]!.evidenceId + ])); + expect(task?.provesCurrentTree).toBe(false); + }); + + it("UAT-RUNTIME-3: old revision telemetry is historical and cannot prove the current tree", () => { + const root = tempRoot(); + writeJson(join(root, "traces.json"), otlp([span("GET /users", "1", 0, { "http.route": "/users" })], "api", "old")); + const report = ingestRuntimeEvidence({ + rootDir: root, + otlpPath: "traces.json", + headRevision: "head", + persist: false + }); + + expect(report.operations[0]).toMatchObject({ trust: "historical", provesCurrentTree: false }); + expect(report.canProveCurrentTree).toBe(false); + expect(report.limitations.join(" ")).toMatch(/cannot prove the current tree/i); + }); + + it("UAT-RUNTIME-4: PII and secrets are absent from reports and persisted artifacts", () => { + const root = tempRoot(); + writeJson(join(root, "traces.json"), otlp([ + span("GET /users?token=secret", "route-1", 1, { + "http.route": "/users?authorization=Bearer abc.def", + "user.email": "person@example.com", + "error.type": "timeout" + }) + ], "api", "head")); + writeJson(join(root, "errors.json"), { + errors: [{ service: "api", group: "users?token=secret", message: "failed for person@example.com with ghp_abcdefghijklmnopqrstuvwxyz", revision: "old", count: 3 }] + }); + + const report = ingestRuntimeEvidence({ + rootDir: root, + otlpPath: "traces.json", + errorsPath: "errors.json", + headRevision: "head" + }); + const serialized = JSON.stringify(report); + const artifact = JSON.parse(readFileSync(join(root, ".codedecay/local/runtime-evidence.json"), "utf8")) as unknown; + + expect(serialized).not.toContain("person@example.com"); + expect(serialized).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz"); + expect(serialized).not.toContain("Bearer abc.def"); + expect(JSON.stringify(artifact)).not.toContain("person@example.com"); + expect(report.safety.secretsPersisted).toBe(false); + }); + + it("UAT-RUNTIME-5: no configured provider means zero network calls and a clear limitation", () => { + const report = ingestRuntimeEvidence({ rootDir: tempRoot(), persist: false, generatedAt: "2026-08-06T00:00:00.000Z" }); + expect(report.provider.kind).toBe("local-artifact"); + expect(report.safety.networkCalled).toBe(false); + expect(report.limitations.join(" ")).toMatch(/zero network calls/i); + expect(report.limitations.join(" ")).toMatch(/No local OpenTelemetry export/i); + }); + + it("UAT-RUNTIME-6: malformed or partial export does not crash and keeps coverage limitations", () => { + const root = tempRoot(); + writeJson(join(root, "traces.json"), otlp([ + span("one", "1", 0), + { spanId: "bad" } + ], "api", "head")); + writeFileSync(join(root, "errors.json"), "{not json", "utf8"); + + const report = ingestRuntimeEvidence({ + rootDir: root, + otlpPath: "traces.json", + errorsPath: "errors.json", + headRevision: "head", + maxSpans: 10, + persist: false + }); + + expect(report.operations.length).toBeGreaterThanOrEqual(1); + expect(report.errors).toEqual([]); + expect(report.stats.malformedRecords).toBeGreaterThan(0); + expect(report.limitations.join(" ")).toMatch(/malformed runtime record/i); + }); +}); + describe("runtime evidence ingestion", () => { it("correlates current-revision traces while redacting sensitive data", () => { const root = tempRoot(); @@ -30,7 +170,7 @@ describe("runtime evidence ingestion", () => { limitations: [] }); - const report = ingestRuntimeEvidence({ rootDir: root, otlpPath: "traces.json", errorsPath: "errors.json", headRevision: "head-sha", topology, generatedAt: "2026-08-02T00:00:00.000Z" }); + const report = ingestRuntimeEvidence({ rootDir: root, otlpPath: "traces.json", errorsPath: "errors.json", headRevision: "head-sha", topology, generatedAt: "2026-08-02T00:00:00.000Z", persist: false }); const serialized = JSON.stringify(report); expect(report.operations[0]).toMatchObject({ route: "/users", spanCount: 1, errorCount: 1, sampled: true, trust: "current-revision", topologyNodeIds: ["service:api"] }); @@ -51,7 +191,7 @@ describe("runtime evidence ingestion", () => { ], "api", "head")); writeFileSync(join(root, "errors.json"), "{not json", "utf8"); - const report = ingestRuntimeEvidence({ rootDir: root, otlpPath: "traces.json", errorsPath: "errors.json", headRevision: "head", maxSpans: 1 }); + const report = ingestRuntimeEvidence({ rootDir: root, otlpPath: "traces.json", errorsPath: "errors.json", headRevision: "head", maxSpans: 1, persist: false }); expect(report.operations).toHaveLength(1); expect(report.operations[0]?.sampled).toBe(false); @@ -70,14 +210,14 @@ describe("runtime evidence ingestion", () => { writeJson(join(outside, "trace.json"), otlp([], "api", "head")); symlinkSync(join(outside, "trace.json"), join(root, "linked.json")); - expect(() => ingestRuntimeEvidence({ rootDir: root, otlpPath: "large.json", maxInputBytes: 4 })).toThrow("exceeds 4 byte limit"); - expect(() => ingestRuntimeEvidence({ rootDir: root, otlpPath: "linked.json" })).toThrow("must stay inside repository"); + expect(() => ingestRuntimeEvidence({ rootDir: root, otlpPath: "large.json", maxInputBytes: 4, persist: false })).toThrow("exceeds 4 byte limit"); + expect(() => ingestRuntimeEvidence({ rootDir: root, otlpPath: "linked.json", persist: false })).toThrow("must stay inside repository"); }); it("reports explicit limitations when no providers are configured", () => { - const report = ingestRuntimeEvidence({ rootDir: tempRoot(), generatedAt: "2026-08-02T00:00:00.000Z" }); + const report = ingestRuntimeEvidence({ rootDir: tempRoot(), generatedAt: "2026-08-02T00:00:00.000Z", persist: false }); expect(report.sources).toEqual([]); - expect(report.limitations).toHaveLength(2); + expect(report.limitations.length).toBeGreaterThanOrEqual(2); expect(report.investigationTasks).toEqual([]); }); }); @@ -97,10 +237,53 @@ function otlp(spans: unknown[], service: string, revision: string): unknown { return { resourceSpans: [{ resource: { attributes: [attribute("service.name", service), attribute("service.version", revision)] }, scopeSpans: [{ spans }] }] }; } -function span(name: string, spanId: string, flags: number, values: Record = {}): unknown { - return { name, spanId, flags, startTimeUnixNano: "1000000", endTimeUnixNano: "6000000", status: { code: 0 }, attributes: Object.entries(values).map(([key, value]) => attribute(key, value)) }; +function span( + name: string, + spanId: string, + flags: number, + values: Record = {}, + start = "1000000", + end = "6000000" +): unknown { + return { + name, + spanId, + flags, + startTimeUnixNano: start, + endTimeUnixNano: end, + status: { code: 0 }, + attributes: Object.entries(values).map(([key, value]) => attribute(key, value)) + }; } function attribute(key: string, value: string): unknown { return { key, value: { stringValue: value } }; } + +function node(id: string, kind: string, label: string, metadata: Record = {}): Record { + return { + id, + kind, + label, + confidence: "declared", + freshness: "current", + trustClass: "declared-context", + sources: [{ kind: "manifest", source: "fixture", repositoryId: "repo", revision: "head", observedAt: "2026-08-06T00:00:00.000Z" }], + limitations: [], + metadata + }; +} + +function edge(id: string, from: string, to: string, kind: string): Record { + return { + id, + from, + to, + kind, + confidence: "declared", + freshness: "current", + trustClass: "declared-context", + sources: [{ kind: "manifest", source: "fixture", repositoryId: "repo", revision: "head", observedAt: "2026-08-06T00:00:00.000Z" }], + limitations: [] + }; +} diff --git a/packages/mcp/src/handlers/runtime-evidence.ts b/packages/mcp/src/handlers/runtime-evidence.ts new file mode 100644 index 0000000..74e1a30 --- /dev/null +++ b/packages/mcp/src/handlers/runtime-evidence.ts @@ -0,0 +1,39 @@ +import { resolve } from "node:path"; +import { + ingestRuntimeEvidence, + loadServiceTopologyManifest, + renderRuntimeEvidenceMarkdown +} from "@submuxhq/codedecay-knowledge"; +import type { StartMcpServerOptions } from "../server/types"; + +export interface RuntimeEvidenceToolInput { + cwd?: string | undefined; + format?: "markdown" | "json" | undefined; + telemetry?: string | undefined; + errors?: string | undefined; + topology?: string | undefined; + headRevision?: string | undefined; + environment?: string | undefined; +} + +export async function runRuntimeEvidenceTool( + options: StartMcpServerOptions, + input: RuntimeEvidenceToolInput +): Promise { + const rootDir = resolve(options.cwd ?? process.cwd(), input.cwd ?? "."); + const topology = input.topology + ? loadServiceTopologyManifest({ rootDir, path: input.topology }) + : undefined; + const report = ingestRuntimeEvidence({ + rootDir, + otlpPath: input.telemetry, + errorsPath: input.errors, + topology, + headRevision: input.headRevision, + environment: input.environment + }); + if ((input.format ?? "markdown") === "json") { + return JSON.stringify(report, null, 2); + } + return renderRuntimeEvidenceMarkdown(report); +} diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index dbae79c..8d9abb1 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -29,6 +29,7 @@ import { } from "./handlers/product"; import { runContextServiceTool } from "./handlers/context-service"; import { runServiceTopologyTool } from "./handlers/service-topology"; +import { runRuntimeEvidenceTool } from "./handlers/runtime-evidence"; import type { StartMcpServerOptions } from "./server/types"; import { registerCodeDecayMcpTools } from "./tools/registry"; @@ -55,6 +56,7 @@ export { export { runExecuteConfiguredChecksTool } from "./handlers/execution"; export { runContextServiceTool } from "./handlers/context-service"; export { runServiceTopologyTool } from "./handlers/service-topology"; +export { runRuntimeEvidenceTool } from "./handlers/runtime-evidence"; export { runProductFailuresTool, runProductPlanTool, @@ -88,6 +90,7 @@ export function createCodeDecayMcpServer(options: StartMcpServerOptions): McpSer taskContext: (input) => runTaskContextTool(options, input), contextService: (input) => runContextServiceTool(options, input), serviceTopology: (input) => runServiceTopologyTool(options, input), + runtimeEvidence: (input) => runRuntimeEvidenceTool(options, input), agentInvestigation: (input) => runAgentInvestigationTool(options, input), scopeCheck: (input) => runScopeCheckTool(options, input), designContractCheck: (input) => runDesignContractCheckTool(options, input), diff --git a/packages/mcp/src/tools/register-analysis.ts b/packages/mcp/src/tools/register-analysis.ts index e05cba0..3d8de40 100644 --- a/packages/mcp/src/tools/register-analysis.ts +++ b/packages/mcp/src/tools/register-analysis.ts @@ -13,7 +13,8 @@ import { scopeCheckToolSchema, taskContextToolSchema, contextServiceToolSchema, - serviceTopologyToolSchema + serviceTopologyToolSchema, + runtimeEvidenceToolSchema } from "./schemas"; import type { AgentPreflightToolInput, @@ -29,7 +30,8 @@ import type { TaskContextToolInput, WhatDidIMissToolInput, ContextServiceToolInput, - ServiceTopologyToolInput + ServiceTopologyToolInput, + RuntimeEvidenceToolInput } from "./types"; export function registerAnalysisMcpTools(server: McpServer, handlers: CodeDecayMcpToolHandlers): void { @@ -131,6 +133,13 @@ export function registerAnalysisMcpTools(server: McpServer, handlers: CodeDecayM async (input) => textResult(handlers.serviceTopology(input as ServiceTopologyToolInput)) ); + server.tool( + "runtime_evidence", + "Ingest local OpenTelemetry/error exports as redacted runtime evidence. Local-only; historical evidence cannot prove the current tree.", + runtimeEvidenceToolSchema, + async (input) => textResult(handlers.runtimeEvidence(input as RuntimeEvidenceToolInput)) + ); + server.tool( "scope_check", "Return a deterministic in-scope/out-of-scope verdict for the current PR or working tree.", diff --git a/packages/mcp/src/tools/registry.ts b/packages/mcp/src/tools/registry.ts index 11abb7f..e6266bf 100644 --- a/packages/mcp/src/tools/registry.ts +++ b/packages/mcp/src/tools/registry.ts @@ -20,7 +20,8 @@ import type { TaskContextToolInput, WhatDidIMissToolInput, ContextServiceToolInput, - ServiceTopologyToolInput + ServiceTopologyToolInput, + RuntimeEvidenceToolInput } from "./types"; export interface CodeDecayMcpToolHandlers { @@ -37,6 +38,7 @@ export interface CodeDecayMcpToolHandlers { taskContext(input: TaskContextToolInput): string | Promise; contextService(input: ContextServiceToolInput): string | Promise; serviceTopology(input: ServiceTopologyToolInput): string | Promise; + runtimeEvidence(input: RuntimeEvidenceToolInput): string | Promise; agentInvestigation(input: AgentInvestigationToolInput): string | Promise; scopeCheck(input: ScopeCheckToolInput): string | Promise; designContractCheck(input: DesignContractCheckToolInput): string | Promise; diff --git a/packages/mcp/src/tools/schemas.ts b/packages/mcp/src/tools/schemas.ts index db1a6e4..317b1e8 100644 --- a/packages/mcp/src/tools/schemas.ts +++ b/packages/mcp/src/tools/schemas.ts @@ -131,6 +131,16 @@ export const serviceTopologyToolSchema = { subscriberServiceId: z.string().optional().describe("Optional AsyncAPI subscriber service id.") }; +export const runtimeEvidenceToolSchema = { + cwd: cwdSchema, + format: formatSchema, + telemetry: z.string().optional().describe("Repo-local OTLP JSON trace export."), + errors: z.string().optional().describe("Repo-local structured error/deployment export."), + topology: z.string().optional().describe("Optional repo-local service topology manifest."), + headRevision: z.string().optional().describe("Current source revision for trust classification."), + environment: z.string().optional().describe("Environment label when an export omits one.") +}; + export const agentSessionToolSchema = { cwd: cwdSchema, operation: z.enum(["start", "context", "checkpoint", "finish"]).describe("Session lifecycle operation."), diff --git a/packages/mcp/src/tools/types.ts b/packages/mcp/src/tools/types.ts index d22061e..6f17ec7 100644 --- a/packages/mcp/src/tools/types.ts +++ b/packages/mcp/src/tools/types.ts @@ -63,6 +63,16 @@ export interface ServiceTopologyToolInput { subscriberServiceId?: string | undefined; } +export interface RuntimeEvidenceToolInput { + cwd?: string | undefined; + format?: "markdown" | "json" | undefined; + telemetry?: string | undefined; + errors?: string | undefined; + topology?: string | undefined; + headRevision?: string | undefined; + environment?: string | undefined; +} + export interface AgentSessionToolInput { cwd?: string | undefined; operation: "start" | "context" | "checkpoint" | "finish"; diff --git a/packages/mcp/test/mcp-runtime-evidence.test.ts b/packages/mcp/test/mcp-runtime-evidence.test.ts new file mode 100644 index 0000000..e5c84cf --- /dev/null +++ b/packages/mcp/test/mcp-runtime-evidence.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { writeFileSync, mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { runRuntimeEvidenceTool } from "../src/index"; +import { createRepo } from "./helpers/mcp"; + +describe("MCP runtime_evidence tool", () => { + it("returns redacted local evidence with zero network calls", async () => { + const repo = createRepo({ "README.md": "# fixture\n" }); + mkdirSync(join(repo, ".codedecay", "runtime"), { recursive: true }); + writeFileSync( + join(repo, ".codedecay", "runtime", "traces.json"), + JSON.stringify({ + resourceSpans: [{ + resource: { + attributes: [ + { key: "service.name", value: { stringValue: "api" } }, + { key: "service.version", value: { stringValue: "head" } } + ] + }, + scopeSpans: [{ + spans: [{ + name: "GET /users", + spanId: "1", + flags: 0, + startTimeUnixNano: "0", + endTimeUnixNano: "1000000", + attributes: [ + { key: "http.route", value: { stringValue: "/users" } }, + { key: "user.email", value: { stringValue: "person@example.com" } } + ] + }] + }] + }] + }), + "utf8" + ); + + const output = await runRuntimeEvidenceTool( + { cwd: repo }, + { format: "json", telemetry: ".codedecay/runtime/traces.json", headRevision: "head" } + ); + expect(output).not.toContain("person@example.com"); + const report = JSON.parse(output) as { + canProveCurrentTree: boolean; + provider: { kind: string }; + safety: { networkCalled: boolean }; + operations: Array<{ route?: string }>; + }; + expect(report.provider.kind).toBe("local-artifact"); + expect(report.canProveCurrentTree).toBe(false); + expect(report.safety.networkCalled).toBe(false); + expect(report.operations[0]?.route).toBe("/users"); + }); +});