diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 6a804871c..1535292da 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -43,6 +43,7 @@ jobs: # CLI before running the tests. - name: Build workspace dependencies + CLI run: | + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-agent-runtime npm run build --workspace @beyondnet/evolith-sdk @@ -140,6 +141,7 @@ jobs: - name: Build workspace dependencies run: | + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-agent-runtime npm run build --workspace @beyondnet/evolith-sdk @@ -203,6 +205,7 @@ jobs: - name: Build workspace dependencies run: | + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-infra-providers @@ -229,6 +232,7 @@ jobs: run: npm ci - name: Build workspace dependencies run: | + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-agent-runtime - name: Run sdk-client tests with coverage @@ -267,6 +271,7 @@ jobs: # core-domain + agent-runtime, which is why this job had never once # passed since it was wired. npm run build --workspace @beyondnet/evolith-contracts + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-infra-providers npm run build --workspace @beyondnet/evolith-core @@ -293,7 +298,9 @@ jobs: run: npm ci # infra-providers depends on @beyondnet/evolith-core-domain (resolved from dist). - name: Build workspace dependencies - run: npm run build --workspace @beyondnet/evolith-core-domain + run: | + npm run build --workspace @beyondnet/evolith-contracts + npm run build --workspace @beyondnet/evolith-core-domain - name: Run infra-providers tests with coverage run: npm run test:cov --workspace @beyondnet/evolith-infra-providers @@ -317,7 +324,9 @@ jobs: - name: Install dependencies run: npm ci - name: Build workspace dependencies - run: npm run build --workspace @beyondnet/evolith-core-domain + run: | + npm run build --workspace @beyondnet/evolith-contracts + npm run build --workspace @beyondnet/evolith-core-domain - name: Run agent-runtime tests run: npm run test --workspace @beyondnet/evolith-agent-runtime @@ -336,6 +345,7 @@ jobs: run: npm ci - name: Build workspace dependencies run: | + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-agent-runtime - name: Run agent-runtime-api tests @@ -360,7 +370,9 @@ jobs: # (where a dist already exists from an earlier build) and fails in CI on a # clean checkout — which is exactly what it did the first time this job ran. - name: Build workspace dependencies - run: npm run build --workspace @beyondnet/evolith-core-domain + run: | + npm run build --workspace @beyondnet/evolith-contracts + npm run build --workspace @beyondnet/evolith-core-domain - name: Run contracts tests run: npm run test --workspace @beyondnet/evolith-contracts @@ -393,6 +405,7 @@ jobs: # do with the corpus. Building first makes the count reproducible. - name: Build workspace dependencies run: | + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-infra-providers diff --git a/.github/workflows/evolith-validate-dogfood.yml b/.github/workflows/evolith-validate-dogfood.yml index e332309c0..2b15dbdcc 100644 --- a/.github/workflows/evolith-validate-dogfood.yml +++ b/.github/workflows/evolith-validate-dogfood.yml @@ -98,6 +98,7 @@ jobs: # `cli-command`, which skips the npm install. - name: Build workspace dependencies run: | + npm run build -w @beyondnet/evolith-contracts npm run build -w @beyondnet/evolith-core-domain npm run build -w @beyondnet/evolith-agent-runtime npm run build -w @beyondnet/evolith-sdk diff --git a/.github/workflows/npm-release.yml b/.github/workflows/npm-release.yml index dcdc95d0d..de01cbdf5 100644 --- a/.github/workflows/npm-release.yml +++ b/.github/workflows/npm-release.yml @@ -95,6 +95,7 @@ jobs: # against publishing an empty package. run: | rm -f src/packages/mcp-server/tsconfig.tsbuildinfo + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-mcp test -f src/packages/mcp-server/dist/main.js diff --git a/.github/workflows/reliability.yml b/.github/workflows/reliability.yml index e5f26f437..c973fb5a7 100644 --- a/.github/workflows/reliability.yml +++ b/.github/workflows/reliability.yml @@ -62,6 +62,7 @@ jobs: - name: Build core-api and its workspace chain run: | + npm run build --workspace @beyondnet/evolith-contracts npm run build --workspace @beyondnet/evolith-core-domain npm run build --workspace @beyondnet/evolith-infra-providers npm run build --workspace core-api diff --git a/.github/workflows/sdk-cli-ci.yml b/.github/workflows/sdk-cli-ci.yml index 59efa8a14..8356dee60 100644 --- a/.github/workflows/sdk-cli-ci.yml +++ b/.github/workflows/sdk-cli-ci.yml @@ -138,6 +138,7 @@ jobs: - name: Build workspace dependencies run: | + npm run build -w @beyondnet/evolith-contracts npm run build -w @beyondnet/evolith-core-domain npm run build -w @beyondnet/evolith-agent-runtime npm run build -w @beyondnet/evolith-sdk @@ -180,6 +181,7 @@ jobs: - name: Build workspace dependencies run: | + npm run build -w @beyondnet/evolith-contracts npm run build -w @beyondnet/evolith-core-domain npm run build -w @beyondnet/evolith-agent-runtime npm run build -w @beyondnet/evolith-sdk @@ -279,6 +281,7 @@ jobs: - name: Build workspace dependencies run: | + npm run build -w @beyondnet/evolith-contracts npm run build -w @beyondnet/evolith-core-domain npm run build -w @beyondnet/evolith-agent-runtime npm run build -w @beyondnet/evolith-sdk @@ -345,6 +348,7 @@ jobs: - name: Build workspace dependencies run: | + npm run build -w @beyondnet/evolith-contracts npm run build -w @beyondnet/evolith-core-domain npm run build -w @beyondnet/evolith-agent-runtime npm run build -w @beyondnet/evolith-sdk @@ -506,6 +510,7 @@ jobs: - name: Build workspace dependencies run: | + npm run build -w @beyondnet/evolith-contracts npm run build -w @beyondnet/evolith-core-domain npm run build -w @beyondnet/evolith-agent-runtime npm run build -w @beyondnet/evolith-sdk diff --git a/src/packages/infra-providers/package.json b/src/packages/infra-providers/package.json index d98c2e8d6..24dd97799 100644 --- a/src/packages/infra-providers/package.json +++ b/src/packages/infra-providers/package.json @@ -20,6 +20,7 @@ "test:cov": "jest --coverage" }, "dependencies": { + "@beyondnet/evolith-contracts": "^1.1.0", "@beyondnet/evolith-core-domain": "^1.2.0", "@nestjs/common": "11.1.27", "ajv": "8.20.0", diff --git a/src/packages/infra-providers/src/index.ts b/src/packages/infra-providers/src/index.ts index 46e16f5f7..257c0c9de 100644 --- a/src/packages/infra-providers/src/index.ts +++ b/src/packages/infra-providers/src/index.ts @@ -49,3 +49,23 @@ export type { LighthouseCategoryResult, LighthouseEvidenceProviderOptions, } from './lighthouse-evidence.provider'; + +// GT-604 — the ONE client every verdict-producing surface deposits through. Three +// surfaces each writing their own fetch is three chances to drop the rule engine, +// to rename `accountableOwner` back to `owner`, or to put a `tenantId` in the body +// — and the last is a security hole, not a typo. +export { + TrackerEvaluationIngestClient, + EvaluationIngestError, + createEvaluationIngestClientFromEnv, + depositEvaluation, + EVOLITH_TRACKER_URL_ENV, + EVOLITH_TRACKER_API_KEY_ENV, +} from './tracker/evaluation-ingest.client'; +export type { + EvaluationIngestAck, + TrackerEvaluationIngestClientOptions, + DepositOutcome, + DepositEvaluationInput, + IngestFetchLike, +} from './tracker/evaluation-ingest.client'; diff --git a/src/packages/infra-providers/src/tracker/evaluation-ingest.client.spec.ts b/src/packages/infra-providers/src/tracker/evaluation-ingest.client.spec.ts new file mode 100644 index 000000000..70d9af2f2 --- /dev/null +++ b/src/packages/infra-providers/src/tracker/evaluation-ingest.client.spec.ts @@ -0,0 +1,243 @@ +/** + * GT-604 — the shared deposit client, tested on the four properties that make it + * worth sharing at all. Everything here runs against an injected fetch: what is + * under test is WHAT GOES ON THE WIRE, not whether Node can make a request. + */ + +import { + TrackerEvaluationIngestClient, + EvaluationIngestError, + createEvaluationIngestClientFromEnv, + depositEvaluation, + EVOLITH_TRACKER_URL_ENV, + EVOLITH_TRACKER_API_KEY_ENV, + type IngestFetchLike, +} from './evaluation-ingest.client'; +import { checkEvaluationIngestPayload } from '@beyondnet/evolith-contracts/ingest'; +import type { EvaluationIngestInput } from '@beyondnet/evolith-contracts/ingest'; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const BASE_URL = 'http://tracker-api:8080/api/v1'; +const API_KEY = 'core-machine-key-for-tests'; + +/** + * A verdict shaped like what `evaluateDriftGate` produces, carrying the two things + * the gap names: a rule executed by an engine the Tracker has never heard of, and + * a requester that is NOT the accountable owner. + */ +function verdict(): EvaluationIngestInput { + return { + surface: 'cli', + producerVersion: 'evolith-cli@1.2.0', + result: { + overallVerdict: 'FAIL', + outcome: 'rejected', + evaluatedAt: '2026-07-30T10:00:00.000Z', + correlationId: 'cli-eval-2026-07-30T10:00:00.000Z', + rulesExecuted: [ + { ruleId: 'layer-boundary', rulesetRef: 'hexagonal@1', engine: 'native', verdict: 'FAIL' }, + { ruleId: 'adr-compliance', engine: 'opa', verdict: 'PASS' }, + { ruleId: 'brand-new-rule', engine: 'an-engine-nobody-has-heard-of', verdict: 'WARN' }, + ], + requester: { actorType: 'agent', actorId: 'who-asked', modelRef: 'claude-opus-5' }, + versions: { core: '1.2.0' }, + }, + violations: [ + { + ruleId: 'layer-boundary', + tool: 'dependency-cruiser', + file: 'src/app/handler.ts', + severity: 'error', + message: 'application imports infrastructure', + owner: '@team/who-fixes', + fingerprint: 'fp-001', + frozen: false, + }, + ], + }; +} + +/** Records what was sent and answers 200 with the Tracker's ack shape. */ +function recordingFetch(status = 200, body: unknown = { transactionId: 't-1', correlationId: 'c-1', created: true }) { + const calls: { url: string; init: Record }[] = []; + const impl: IngestFetchLike = async (url, init) => { + calls.push({ url, init }); + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + }; + }; + return { calls, impl }; +} + +const sentBody = (calls: { init: Record }[]): Record => + JSON.parse(String(calls[0].init.body)) as Record; + +// --------------------------------------------------------------------------- + +describe('TrackerEvaluationIngestClient', () => { + it('posts to the route the published contract declares, not to a hand-typed path', async () => { + const { calls, impl } = recordingFetch(); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await client.deposit(verdict()); + + expect(calls).toHaveLength(1); + expect(calls[0].url).toBe(`${BASE_URL}/core-evaluation-transactions`); + expect(calls[0].init.method).toBe('POST'); + }); + + it('presents the CoreMachine key in x-api-key', async () => { + const { calls, impl } = recordingFetch(); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await client.deposit(verdict()); + + expect((calls[0].init.headers as Record)['x-api-key']).toBe(API_KEY); + }); + + /** + * The one field whose PRESENCE is the defect. The Tracker derives the tenant from + * which key matched; a body-supplied tenant would let any valid key deposit into + * any tenant's ledger. + */ + it('never puts a tenant in the body', async () => { + const { calls, impl } = recordingFetch(); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await client.deposit(verdict()); + + expect(sentBody(calls)).not.toHaveProperty('tenantId'); + }); + + /** + * The engine vocabulary is OPEN. A client that quietly rewrote an unrecognised + * engine to `native` would deposit a row claiming a governance rule produced a + * finding that a policy engine produced — a substitution no later consumer could + * detect. + */ + it('carries the engine each rule actually ran on, verbatim', async () => { + const { calls, impl } = recordingFetch(); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await client.deposit(verdict()); + + const rules = sentBody(calls).rulesExecuted as { engine: string }[]; + expect(rules.map((r) => r.engine)).toEqual(['native', 'opa', 'an-engine-nobody-has-heard-of']); + }); + + it('keeps the requester and the accountable owner apart on the wire', async () => { + const { calls, impl } = recordingFetch(); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await client.deposit(verdict()); + + const body = sentBody(calls); + expect((body.requestedBy as { actorId: string }).actorId).toBe('who-asked'); + const violations = body.violations as Record[]; + expect(violations[0].accountableOwner).toBe('@team/who-fixes'); + expect(violations[0]).not.toHaveProperty('owner'); + expect(body.accountableOwners).toEqual(['@team/who-fixes']); + }); + + /** + * The oracle the Tracker's own conformance test runs. Asserting against it here + * means the client and the endpoint are checked by the SAME rule rather than by + * two hand-written lists that can drift apart. + */ + it('sends a body that satisfies the contract oracle', async () => { + const { calls, impl } = recordingFetch(); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await client.deposit(verdict()); + + expect(checkEvaluationIngestPayload(sentBody(calls))).toEqual({ ok: true, problems: [] }); + }); + + it('reports created=false on an idempotent replay so the producer can log which it got', async () => { + const { impl } = recordingFetch(200, { transactionId: 't-1', correlationId: 'c-1', created: false }); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await expect(client.deposit(verdict())).resolves.toMatchObject({ created: false, transactionId: 't-1' }); + }); + + it('throws on a refusal instead of pretending the deposit landed', async () => { + const { impl } = recordingFetch(401, { error: 'no' }); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + await expect(client.deposit(verdict())).rejects.toBeInstanceOf(EvaluationIngestError); + }); +}); + +describe('createEvaluationIngestClientFromEnv', () => { + it('returns undefined when the surface is not configured to deposit', () => { + expect(createEvaluationIngestClientFromEnv({})).toBeUndefined(); + }); + + /** + * A no-op client that answered "deposited" would make an unconfigured deployment + * indistinguishable from a working one — which is the state this gap started from. + */ + it('returns a real client when configured', () => { + const client = createEvaluationIngestClientFromEnv({ + [EVOLITH_TRACKER_URL_ENV]: BASE_URL, + [EVOLITH_TRACKER_API_KEY_ENV]: API_KEY, + }); + expect(client?.url).toBe(`${BASE_URL}/core-evaluation-transactions`); + }); + + it('refuses a URL without a key rather than 401-ing on every evaluation', () => { + expect(() => createEvaluationIngestClientFromEnv({ [EVOLITH_TRACKER_URL_ENV]: BASE_URL })).toThrow( + /machine-authenticated/, + ); + }); +}); + +describe('depositEvaluation', () => { + it('does nothing, and says nothing, when no client is configured', async () => { + const warn = jest.fn(); + await expect(depositEvaluation({ ...verdict(), client: undefined, warn })).resolves.toEqual({ + deposited: false, + }); + expect(warn).not.toHaveBeenCalled(); + }); + + /** + * The load-bearing one. A ledger outage must NOT change a verdict: if depositing + * could throw out of here, `evolith evaluate` would fail on an unrelated service + * being down, and a gate that breaks open on an outage is worse than no gate. + */ + it('never throws when the Tracker is down — the verdict stands, the row is missing', async () => { + const client = new TrackerEvaluationIngestClient({ + baseUrl: BASE_URL, + apiKey: API_KEY, + fetchImpl: async () => { + throw new Error('ECONNREFUSED'); + }, + }); + const warn = jest.fn(); + + const outcome = await depositEvaluation({ ...verdict(), client, warn }); + + expect(outcome.deposited).toBe(false); + expect(outcome.error).toContain('ECONNREFUSED'); + // Reported, never swallowed: a deposit path that fails quietly is + // indistinguishable from one that was never wired. + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toContain('NOT deposited'); + }); + + it('reports the ack when the deposit lands', async () => { + const { impl } = recordingFetch(); + const client = new TrackerEvaluationIngestClient({ baseUrl: BASE_URL, apiKey: API_KEY, fetchImpl: impl }); + + const outcome = await depositEvaluation({ ...verdict(), client }); + + expect(outcome).toMatchObject({ deposited: true, ack: { created: true } }); + }); +}); diff --git a/src/packages/infra-providers/src/tracker/evaluation-ingest.client.ts b/src/packages/infra-providers/src/tracker/evaluation-ingest.client.ts new file mode 100644 index 000000000..481ccc238 --- /dev/null +++ b/src/packages/infra-providers/src/tracker/evaluation-ingest.client.ts @@ -0,0 +1,285 @@ +/** + * GT-604 — the ONE client every verdict-producing surface deposits through. + * + * The state this closes + * --------------------- + * The evidence write path pointed only INWARD: the Tracker opened transactions + * against the Core and the Core had nowhere to put its verdicts. Every + * `evolith evaluate`, every `enforce edit` veto, every MCP `tools/call` and every + * drift-gate run produced a complete, owner-attributed, engine-attributed verdict + * and then evaporated on process exit. The Tracker side of GT-604 added the route; + * this is the caller. + * + * Why ONE client and not one per surface + * -------------------------------------- + * Three surfaces each writing their own `fetch` is three chances to drop + * `rulesExecuted[].engine`, to send `owner` where the wire says `accountableOwner`, + * or to put a `tenantId` in the body — and the third is a security hole, not a + * typo: the Tracker derives the tenant from WHICH machine key matched, so a + * body-supplied tenant would let one key deposit into another tenant's ledger. + * The payload is built exactly once, by `toEvaluationIngestPayload` from + * `@beyondnet/evolith-contracts/ingest`, which is the same module the Tracker's + * own conformance test pins. + * + * Why a failed deposit never changes a verdict + * -------------------------------------------- + * Depositing evidence is not a gate. If the Tracker is down, `evolith evaluate` + * must still print its verdict and still exit non-zero when it BLOCKS: making the + * gate depend on the ledger's availability would mean an outage silently opens the + * gate — the exact inversion a governance product cannot afford. So + * {@link depositEvaluation} reports the failure on the caller's warning channel + * and returns `{ deposited: false }`. It is reported and never swallowed: a + * deposit path that fails quietly is indistinguishable from one that was never + * wired, which is the state this gap started from. + * + * Auth, verbatim from the precedent + * --------------------------------- + * Same shape `TrackerApprovalHttpClient` already uses for `POST /runtime-approvals` + * (agent-runtime): the CoreMachine key travels in `x-api-key`, the tenant is + * derived from the key and NEVER sent. Not configured means not deposited — a + * surface without a Tracker URL is a surface running standalone, not a failure. + */ + +import { + toEvaluationIngestPayload, + EVALUATION_INGEST_ENDPOINT_CONTRACT, + type EvaluationIngestInput, + type EvaluationIngestPayload, + type EvaluationIngestSurface, +} from '@beyondnet/evolith-contracts/ingest'; + +// --------------------------------------------------------------------------- +// Transport seam +// --------------------------------------------------------------------------- + +interface FetchResponse { + readonly ok: boolean; + readonly status: number; + json(): Promise; + text?(): Promise; +} + +export type IngestFetchLike = (url: string, init: Record) => Promise; + +/** The Tracker's acknowledgement. `created` distinguishes a first deposit from a replay. */ +export interface EvaluationIngestAck { + readonly transactionId: string; + readonly correlationId: string; + readonly created: boolean; +} + +export interface TrackerEvaluationIngestClientOptions { + /** Tracker base URL INCLUDING the versioned prefix, e.g. `http://tracker-api:8080/api/v1`. */ + readonly baseUrl: string; + /** CoreMachine key the Tracker binds to a tenant; sent as `x-api-key`. */ + readonly apiKey?: string; + /** Abort the request after this many ms (default 10s). */ + readonly timeoutMs?: number; + /** Inject a fetch implementation (defaults to global fetch). */ + readonly fetchImpl?: IngestFetchLike; +} + +const DEFAULT_TIMEOUT_MS = 10_000; + +/** Raised when the Tracker refuses or fails a deposit. Carries the status for the caller's log. */ +export class EvaluationIngestError extends Error { + constructor( + message: string, + readonly status: number, + ) { + super(message); + this.name = 'EvaluationIngestError'; + } +} + +// --------------------------------------------------------------------------- +// The client +// --------------------------------------------------------------------------- + +export class TrackerEvaluationIngestClient { + private readonly endpoint: string; + private readonly apiKey?: string; + private readonly timeoutMs: number; + private readonly fetchImpl: IngestFetchLike; + + constructor(options: TrackerEvaluationIngestClientOptions) { + const globalFetch = (globalThis as { fetch?: IngestFetchLike }).fetch; + const impl = options.fetchImpl ?? globalFetch; + if (!impl) { + throw new Error( + 'TrackerEvaluationIngestClient requires fetch (Node 18+) or an injected fetchImpl.', + ); + } + this.fetchImpl = impl; + // The route is taken from the published contract rather than typed here, so a + // path change upstream moves this client with it instead of leaving a caller + // that 404s against a route nobody remembers renaming. + this.endpoint = `${options.baseUrl.replace(/\/+$/, '')}${EVALUATION_INGEST_ENDPOINT_CONTRACT.path}`; + this.apiKey = options.apiKey; + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; + } + + /** Where this client posts. Exposed so a caller can name it in a log line. */ + get url(): string { + return this.endpoint; + } + + /** + * Build the payload from a verdict and POST it. + * + * The mapping is delegated entirely to the shared contract: this method never + * reshapes a field, never coerces an engine and never adds a tenant. + */ + async deposit(input: EvaluationIngestInput): Promise { + return this.send(toEvaluationIngestPayload(input)); + } + + /** POST an already-built payload. Split out so a test can post a hand-made body. */ + async send(payload: EvaluationIngestPayload): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), this.timeoutMs); + + try { + const response = await this.fetchImpl(this.endpoint, { + method: EVALUATION_INGEST_ENDPOINT_CONTRACT.method, + headers: { + 'content-type': 'application/json', + // The tenant rides on WHICH key matched. It is never in the body — see + // the module header. + ...(this.apiKey ? { [EVALUATION_INGEST_ENDPOINT_CONTRACT.auth.header]: this.apiKey } : {}), + }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + + if (!response.ok) { + const detail = response.text ? await safeText(response) : ''; + throw new EvaluationIngestError( + `Tracker refused the evaluation deposit (${response.status})${detail ? `: ${detail}` : ''}`, + response.status, + ); + } + + const body = (await response.json()) as Partial | null; + return { + transactionId: String(body?.transactionId ?? ''), + // Fall back to what was sent rather than to an empty string: the caller + // logs this id to join its local artifact to the ledger row. + correlationId: String(body?.correlationId ?? payload.correlationId), + created: body?.created === true, + }; + } finally { + clearTimeout(timer); + } + } +} + +async function safeText(response: FetchResponse): Promise { + try { + return (await response.text!()).slice(0, 500); + } catch { + return ''; + } +} + +// --------------------------------------------------------------------------- +// Configuration — not configured is not a failure +// --------------------------------------------------------------------------- + +/** Base URL of the Tracker, INCLUDING `/api/v1`. Absent means "do not deposit". */ +export const EVOLITH_TRACKER_URL_ENV = 'EVOLITH_TRACKER_URL'; + +/** CoreMachine key. The Tracker resolves the tenant from it; it never travels in a body. */ +export const EVOLITH_TRACKER_API_KEY_ENV = 'EVOLITH_TRACKER_API_KEY'; + +export interface IngestEnv { + readonly [key: string]: string | undefined; +} + +/** + * Build a client from the environment, or `undefined` when the surface is not + * configured to deposit. + * + * Returning `undefined` rather than a no-op client is deliberate: a no-op that + * answers "deposited" would make an unconfigured deployment indistinguishable + * from a working one, and this whole gap exists because nobody noticed evidence + * going nowhere. + * + * A URL WITHOUT a key throws. That combination is always a misconfiguration — the + * endpoint is machine-authenticated, so every request would 401 — and failing at + * construction says so once instead of once per evaluation. Same posture as the + * agent-runtime factory takes for `AGENT_RUNTIME_APPROVAL_TRACKER_URL`. + */ +export function createEvaluationIngestClientFromEnv( + env: IngestEnv, + options?: Omit, +): TrackerEvaluationIngestClient | undefined { + const baseUrl = trimmed(env[EVOLITH_TRACKER_URL_ENV]); + if (!baseUrl) return undefined; + + const apiKey = trimmed(env[EVOLITH_TRACKER_API_KEY_ENV]); + if (!apiKey) { + throw new Error( + `${EVOLITH_TRACKER_URL_ENV} is set without ${EVOLITH_TRACKER_API_KEY_ENV}. The Tracker ` + + 'evaluation-ingest endpoint is machine-authenticated (CoreMachine key in x-api-key), so ' + + 'every deposit would be rejected with 401 and every verdict would still be lost.', + ); + } + + return new TrackerEvaluationIngestClient({ ...options, baseUrl, apiKey }); +} + +function trimmed(value: string | undefined): string | undefined { + const v = value?.trim(); + return v ? v : undefined; +} + +// --------------------------------------------------------------------------- +// The surface-facing helper +// --------------------------------------------------------------------------- + +export interface DepositOutcome { + readonly deposited: boolean; + readonly ack?: EvaluationIngestAck; + /** Why it did not land. Present only when `deposited` is false AND it was attempted. */ + readonly error?: string; +} + +export interface DepositEvaluationInput extends EvaluationIngestInput { + /** `undefined` when the surface is not configured — then nothing is attempted. */ + readonly client?: TrackerEvaluationIngestClient; + /** Where a failure is reported. Defaults to `console.warn` (stderr), never stdout. */ + readonly warn?: (message: string) => void; +} + +/** + * Deposit a verdict without ever letting the deposit change the verdict. + * + * Every surface calls THIS rather than `client.deposit` directly, so that the + * "a ledger outage must not open a gate" rule is implemented once instead of + * three times with two of them subtly different. + * + * The warning goes to `console.warn` — stderr — and never to stdout, because + * `--format json` and `--format sarif` promise a parseable document on stdout and + * a warning printed there would corrupt every consumer that pipes it. + */ +export async function depositEvaluation(input: DepositEvaluationInput): Promise { + const { client, warn, ...ingest } = input; + if (!client) return { deposited: false }; + + const report = warn ?? ((message: string) => console.warn(message)); + + try { + const ack = await client.deposit(ingest); + return { deposited: true, ack }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + report( + `[evolith] evaluation evidence was NOT deposited in the Tracker (${client.url}): ${message}. ` + + 'The verdict above stands; only the ledger row is missing.', + ); + return { deposited: false, error: message }; + } +} + +export type { EvaluationIngestInput, EvaluationIngestPayload, EvaluationIngestSurface }; diff --git a/src/packages/infra-providers/src/tracker/ingest-surface-parity.spec.ts b/src/packages/infra-providers/src/tracker/ingest-surface-parity.spec.ts new file mode 100644 index 000000000..a3e5e460c --- /dev/null +++ b/src/packages/infra-providers/src/tracker/ingest-surface-parity.spec.ts @@ -0,0 +1,124 @@ +/** + * GT-604, acceptance criterion 2 — **one shared client, used by all three + * surfaces.** + * + * The criterion is not "a client exists"; it is that the CLI, the MCP server and + * the drift gate deposit through the SAME one. A client that only its own unit + * test calls would pass every assertion in `evaluation-ingest.client.spec.ts` + * forever while every verdict kept evaporating — which is the exact shape of the + * defect this gap describes. + * + * So this guard asserts the CALL SITES, not the client. Two properties: + * + * 1. each surface calls `depositEvaluation` and declares the right + * `producer.surface`. A surface that deposited under the wrong label would + * make the Tracker's per-surface filter — the whole reason that field is a + * closed vocabulary — quietly wrong; + * 2. no surface builds its own request to the ingest route. Three hand-written + * `fetch`es are three chances to drop `rulesExecuted[].engine`, to send + * `owner` where the wire says `accountableOwner`, or to put a `tenantId` in + * the body — and the last one is a security hole, not a typo, because the + * Tracker derives the tenant from which machine key matched. + * + * It reads source, in the same family as `api-catalog-parity.spec.ts`. That is a + * real limitation and it is bounded: it can prove a surface is WIRED, not that + * the wiring runs on every path. What it defends against is the regression that + * actually happens — someone deleting the call, or adding a fourth surface with + * its own copy of the payload mapping. + */ + +import { readFileSync, existsSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +/** Walk up to the monorepo root so this works from src/ and from dist/. */ +function repoRoot(): string { + let dir = resolve(__dirname); + for (let i = 0; i < 8; i++) { + if (existsSync(join(dir, 'package.json')) && existsSync(join(dir, 'src', 'packages'))) return dir; + dir = resolve(dir, '..'); + } + throw new Error('monorepo root not found from ' + __dirname); +} + +const ROOT = repoRoot(); + +interface Surface { + readonly label: string; + readonly file: string; + /** The `surface` literals this file must declare when depositing. */ + readonly declares: readonly string[]; +} + +const SURFACES: readonly Surface[] = [ + { + label: 'CLI `evolith evaluate` (and its --format drift gate)', + file: 'src/sdk/cli/src/commands/evaluate/evaluate.command.ts', + // ONE file, TWO surfaces: the drift gate is the `--format drift` path of the + // same command, and the gap names them separately because they are separate + // producers in the ledger. + declares: ['cli', 'drift-gate'], + }, + { + label: 'MCP `evolith-evaluate`', + file: 'src/packages/mcp-server/src/tools/evaluate.tool.ts', + declares: ['mcp'], + }, +]; + +function source(relative: string): string { + const full = join(ROOT, relative); + const text = existsSync(full) ? readFileSync(full, 'utf-8') : ''; + // A path that stopped resolving would turn every assertion below into "I checked + // nothing" — the failure mode this repository has already been bitten by. + expect(text.length).toBeGreaterThan(0); + return text; +} + +describe('GT-604 — every verdict-producing surface deposits through the shared client', () => { + it.each(SURFACES.map((s) => [s.label, s] as const))('%s calls depositEvaluation', (_label, surface) => { + const text = source(surface.file); + + expect(text).toContain('depositEvaluation'); + expect(text).toMatch(/from '@beyondnet\/evolith-infra-providers'/); + }); + + it.each(SURFACES.map((s) => [s.label, s] as const))( + '%s declares the surface it deposits as', + (_label, surface) => { + const text = source(surface.file); + + for (const declared of surface.declares) { + expect(text).toContain(`surface: '${declared}'`); + } + }, + ); + + /** + * The invariant that makes the shared client shared. `EVALUATION_INGEST_ENDPOINT_CONTRACT.path` + * appears exactly once in the whole repository outside the contract and the + * client's own tests: inside the client. + */ + it.each(SURFACES.map((s) => [s.label, s] as const))( + '%s does NOT build its own request to the ingest route', + (_label, surface) => { + const text = source(surface.file); + + expect(text).not.toContain('core-evaluation-transactions'); + expect(text).not.toContain('x-api-key'); + }, + ); + + it('the CLI deposits BEFORE it exits on a blocking verdict', () => { + const text = source(SURFACES[0].file); + + const deposit = text.indexOf('await this.depositEvidence'); + const blockingExit = text.indexOf('if (published.exitCode !== 0) exitWith'); + + expect(deposit).toBeGreaterThan(-1); + expect(blockingExit).toBeGreaterThan(-1); + expect(deposit).toBeLessThan(blockingExit); + // A run that BLOCKS is precisely the run whose evidence matters. Exiting first + // would lose exactly those and keep the passing ones, which is worse than + // losing all of them: the ledger would read as if nothing ever failed. + }); +}); diff --git a/src/packages/mcp-server/src/tools/evaluate.tool.ts b/src/packages/mcp-server/src/tools/evaluate.tool.ts index a6cc05d0a..63f00f837 100644 --- a/src/packages/mcp-server/src/tools/evaluate.tool.ts +++ b/src/packages/mcp-server/src/tools/evaluate.tool.ts @@ -3,10 +3,15 @@ import { Injectable, Inject } from '@nestjs/common'; import { RulesetValidatorService } from '@beyondnet/evolith-core'; import { createSuccessEnvelope } from '@beyondnet/evolith-core-domain'; import type { IFileSystem } from '@beyondnet/evolith-core-domain/domain/interfaces'; -import { NestLoggerProvider } from '@beyondnet/evolith-infra-providers'; +import { + NestLoggerProvider, + createEvaluationIngestClientFromEnv, + depositEvaluation, +} from '@beyondnet/evolith-infra-providers'; import { EvaluationOrchestrator, createDefaultKindEvaluators, + evaluateDriftGate, type EvaluationContext, type IEvaluationPipeline, type IWorkspaceReferenceResolver, @@ -117,11 +122,30 @@ export class EvaluateTool implements McpTool { const orchestrator = new EvaluationOrchestrator(pipeline, resolver, CORE_VERSION, evaluators); const result = await orchestrator.evaluate(ctx); + const correlationId = result.correlationId ?? `mcp-eval-${result.evaluatedAt}`; + + // GT-604: deposit this verdict in the Tracker's evidence ledger through the SAME + // shared client the CLI and the drift gate use, so the three surfaces cannot + // disagree about the wire shape. `undefined` when this installation is not + // configured to talk to a Tracker — a standalone run, not a failure — and a + // failed deposit never changes what this tool returns: an agent must not see a + // different verdict because a ledger was down. + await depositEvaluation({ + client: createEvaluationIngestClientFromEnv(process.env), + result, + // The canonical, owner-enriched violation set — the same one the drift gate + // blocks on, so the ledger's accountable owner is the one a PR comment names. + violations: evaluateDriftGate({ result }).evidence.violations, + surface: 'mcp', + producerVersion: `evolith-mcp@${CORE_VERSION}`, + correlationId, + }); + return createSuccessEnvelope(result, { command: 'evolith-evaluate', executedAt: result.evaluatedAt, durationMs: 0, - correlationId: result.correlationId ?? `mcp-eval-${result.evaluatedAt}`, + correlationId, schemaVersion: result.schemaVersion, }); } diff --git a/src/sdk/cli/src/commands/evaluate/evaluate.command.ts b/src/sdk/cli/src/commands/evaluate/evaluate.command.ts index bde793bf1..67218bbeb 100644 --- a/src/sdk/cli/src/commands/evaluate/evaluate.command.ts +++ b/src/sdk/cli/src/commands/evaluate/evaluate.command.ts @@ -20,6 +20,11 @@ import { type IEvaluationPipeline, type IWorkspaceReferenceResolver, } from '@beyondnet/evolith-core-domain/evaluation'; +import { + createEvaluationIngestClientFromEnv, + depositEvaluation, + type TrackerEvaluationIngestClient, +} from '@beyondnet/evolith-infra-providers'; import { BaseEvolithCommand } from '../../infrastructure/cli/base-command'; import { PromptService } from '../../infrastructure/prompts/prompt.service'; import { ConfigService } from '../../infrastructure/config/config.service'; @@ -151,6 +156,10 @@ export class EvaluateCommand extends BaseEvolithCommand { await fs.writeFile(path.resolve(options.evidence), JSON.stringify(manifest, null, 2), 'utf-8'); } + // GT-604: the deposit path. `undefined` when this installation is not configured + // to talk to a Tracker, which is a standalone run and not a failure. + const ingestClient = createEvaluationIngestClientFromEnv(process.env); + const format = options?.format || 'json'; if (format === 'sarif') { // GT-518: emit a SARIF 2.1.0 log (gaps/risks as `result`s) for the CI/PR @@ -165,6 +174,15 @@ export class EvaluateCommand extends BaseEvolithCommand { const decision = evaluateDriftGate({ result, codeowners }); const published = await new PrCommentFallbackPublisher().publish(decision); console.log(published.body); + // GT-604: deposit BEFORE the exit. A run that blocks is precisely the run whose + // evidence matters, and exiting first would lose exactly those. + await this.depositEvidence({ + client: ingestClient, + result, + violations: decision.evidence.violations, + surface: 'drift-gate', + correlationId: envelope.meta.correlationId, + }); // The drift gate's non-zero exit means the gate BLOCKS, not that the tool broke. if (published.exitCode !== 0) exitWith(CLI_EXIT_CODES.BLOCKED); return; @@ -178,6 +196,19 @@ export class EvaluateCommand extends BaseEvolithCommand { console.log(JSON.stringify(envelope, null, 2)); } + // GT-604: same deposit for the non-drift formats, tagged as the `cli` surface. + // Before the exit, for the same reason as above. + await this.depositEvidence({ + client: ingestClient, + result, + violations: evaluateDriftGate({ + result, + codeowners: await this.loadCodeowners(ctx.workspaceRef ?? process.cwd()), + }).evidence.violations, + surface: 'cli', + correlationId: envelope.meta.correlationId, + }); + // A negative evaluation verdict must exit non-zero so CI can gate on it, // mirroring `gate`/`phase` (envelope success=command-ran, exit=verdict). if (result.overallVerdict === 'FAIL' || result.outcome === 'rejected') { @@ -185,6 +216,41 @@ export class EvaluateCommand extends BaseEvolithCommand { } } + /** + * GT-604 — deposit this run's verdict in the Tracker's evidence ledger. + * + *

Every field the ledger row carries is mapped by the SHARED contract + * (`toEvaluationIngestPayload`, inside the shared client), never here: this + * command decides only WHICH surface it is and WHICH violations belong to the + * run. The canonical violations come from `evaluateDriftGate`, the same + * CODEOWNERS-enriched set the drift gate blocks on, so the ledger's accountable + * owner is the one the PR comment names.

+ * + *

A failed deposit is reported on stderr and changes NOTHING about the + * verdict or the exit code. Making the gate depend on the ledger's availability + * would mean a Tracker outage silently opens the gate.

+ */ + private async depositEvidence(input: { + client?: TrackerEvaluationIngestClient; + result: Parameters[0]['result']; + violations: Parameters[0]['violations']; + surface: 'cli' | 'drift-gate'; + correlationId: string; + }): Promise { + if (!input.client) return; + + await depositEvaluation({ + client: input.client, + result: input.result, + violations: input.violations, + surface: input.surface, + producerVersion: `evolith-cli@${CORE_VERSION}`, + // The SAME id the success envelope carries, so the local artifact and the + // ledger row join on one key instead of on two that look alike. + correlationId: input.correlationId, + }); + } + /** * Load and parse the repo's CODEOWNERS (for drift-gate owner enrichment) from the * conventional locations under the workspace. Returns `[]` when none exists — owner