diff --git a/src/lab/fabric/observe.ts b/src/lab/fabric/observe.ts index 7e93d13fd3..079b10057c 100644 --- a/src/lab/fabric/observe.ts +++ b/src/lab/fabric/observe.ts @@ -8,7 +8,7 @@ import { OUTCOMES, } from "../constants"; import { FAILURE_CLASSIFICATIONS } from "../conformance/types"; -import { fixtureDigest, isSha256Hex, jcsStringify } from "../digest"; +import { fixtureDigest, isSha256Hex, jcsStringify, subjectIdForSubject } from "../digest"; import type { ObservationEvent, RouteSubjectV1, TaskSubjectV1 } from "../events/types"; import { LabValidationError } from "../events/errors"; import { assignEventId, validateSubject } from "../events/validate"; @@ -172,6 +172,11 @@ function validateFabricVerifier(raw: Record): void { /** Validate non-negative usage counters on a producer outcome. */ function validateFabricUsage(raw: Record): void { + for (const key of Object.keys(raw)) { + if (!(USAGE_KEYS as readonly string[]).includes(key)) { + throw new FabricTaskError(`unknown usage field ${key}`, "malformed_producer_outcome", "harness"); + } + } for (const key of USAGE_KEYS) { assertNonNegativeIntegerField(raw, key); } @@ -179,6 +184,11 @@ function validateFabricUsage(raw: Record): void { /** Validate non-negative limit fields on a producer outcome. */ function validateFabricLimits(raw: Record): void { + for (const key of Object.keys(raw)) { + if (!(LIMIT_KEYS as readonly string[]).includes(key)) { + throw new FabricTaskError(`unknown limit field ${key}`, "malformed_producer_outcome", "harness"); + } + } for (const key of LIMIT_KEYS) { assertNonNegativeIntegerField(raw, key); } @@ -262,12 +272,18 @@ export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { } catch (error) { wrapValidationError(error); } + if (jcsStringify(taskSubjectObj) !== jcsStringify(taskSubject)) { + throw new FabricTaskError("taskSubject contains undeclared fields", "malformed_producer_outcome", "harness"); + } + if (jcsStringify(routeSubjectObj) !== jcsStringify(routeSubject)) { + throw new FabricTaskError("routeSubject contains undeclared fields", "malformed_producer_outcome", "harness"); + } if (!routeSubjectsMatch(routeSubject, taskSubject.routeSubject)) { throw new FabricTaskError("contradictory route subjects", "layer_subject_mismatch", "harness"); } - assertStringField(obj, "taskClassId"); - assertStringField(obj, "taskClassVersion"); + const taskClassId = assertStringField(obj, "taskClassId"); + const taskClassVersion = assertStringField(obj, "taskClassVersion"); const subjectId = assertStringField(obj, "subjectId"); if (!isSha256Hex(subjectId)) { throw new FabricTaskError("malformed producer outcome: subjectId", "malformed_producer_outcome", "harness"); @@ -277,16 +293,36 @@ export function assertFabricOutcomeV1(raw: unknown): FabricTaskOutcomeV1 { if (!isSha256Hex(taskFixtureDigest) || !isSha256Hex(verifierManifestDigest)) { throw new FabricTaskError("malformed producer outcome: digest field", "malformed_producer_outcome", "harness"); } - assertStringField(obj, "fabricCompatibilityVersion"); + const fabricCompatibilityVersion = assertStringField(obj, "fabricCompatibilityVersion"); const sandboxProfileDigest = assertStringField(obj, "sandboxProfileDigest"); if (!isSha256Hex(sandboxProfileDigest)) { throw new FabricTaskError("malformed producer outcome: sandboxProfileDigest", "malformed_producer_outcome", "harness"); } - assertIntegerField(obj, "startedAt"); - assertIntegerField(obj, "completedAt"); + if (subjectId !== subjectIdForSubject(taskSubject)) { + throw new FabricTaskError("subjectId does not match taskSubject", "malformed_producer_outcome", "harness"); + } + if ( + taskClassId !== taskSubject.taskClassId + || taskClassVersion !== taskSubject.taskClassVersion + || taskFixtureDigest !== taskSubject.taskFixtureDigest + || verifierManifestDigest !== taskSubject.verifierManifestDigest + || fabricCompatibilityVersion !== taskSubject.fabricCompatibilityVersion + || sandboxProfileDigest !== taskSubject.sandboxProfileDigest + ) { + throw new FabricTaskError("task identity fields do not match taskSubject", "malformed_producer_outcome", "harness"); + } + const startedAt = assertNonNegativeIntegerField(obj, "startedAt"); + const completedAt = assertNonNegativeIntegerField(obj, "completedAt"); + if (completedAt < startedAt) { + throw new FabricTaskError("invalid execution timestamps", "malformed_producer_outcome", "harness"); + } validateFabricLimits(assertPlainObject(obj.limits, "limits")); validateFabricUsage(assertPlainObject(obj.usage, "usage")); - validateFabricVerifier(assertPlainObject(obj.verifier, "verifier")); + const verifier = assertPlainObject(obj.verifier, "verifier"); + validateFabricVerifier(verifier); + if (verifier.manifestDigest !== verifierManifestDigest) { + throw new FabricTaskError("verifier manifest digest does not match outcome", "malformed_producer_outcome", "harness"); + } if (!OUTCOMES.includes(obj.outcome as typeof OUTCOMES[number])) { throw new FabricTaskError("malformed producer outcome: outcome", "malformed_producer_outcome", "harness"); } @@ -324,9 +360,6 @@ export function observationFromFabricOutcome( if (routeSubject.subjectKind !== "route") { throw new FabricTaskError("nested route subject required", "layer_subject_mismatch", "harness"); } - if (!Number.isInteger(outcome.startedAt) || !Number.isInteger(outcome.completedAt) || outcome.completedAt < outcome.startedAt) { - throw new FabricTaskError("invalid execution timestamps", "malformed_producer_outcome", "harness"); - } const paths = ensureLabDirs(opts.configDir); const ownsStore = !opts.artifactStore; diff --git a/tests/lab-fabric-outcome-validation.test.ts b/tests/lab-fabric-outcome-validation.test.ts new file mode 100644 index 0000000000..d5d0ee0bb7 --- /dev/null +++ b/tests/lab-fabric-outcome-validation.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, test } from "bun:test"; +import { + assertFabricOutcomeV1, + buildTaskSubjectV1, + FABRIC_LIMITS, + FABRIC_VERIFIER_ID, + subjectIdForSubject, + type FabricTaskOutcomeV1, + type RouteSubjectV1, +} from "../src/lab"; +import { FabricTaskError } from "../src/lab/fabric/types"; + +function routeSubject(): RouteSubjectV1 { + return { + subjectSchemaVersion: 1, + subjectKind: "route", + providerId: "provider-a", + providerInstanceFingerprint: "a".repeat(64), + clientModelId: "model-a", + upstreamModelId: "model-a", + effectiveAdapter: "openai-responses", + inboundProtocol: "openai-responses", + upstreamProtocol: "openai-responses", + surface: "responses-http", + opencodexCompatibilityVersion: "b".repeat(64), + behaviorFingerprint: "c".repeat(64), + endpointFingerprint: "d".repeat(64), + dependencies: [], + }; +} + +function validOutcome(): FabricTaskOutcomeV1 { + const taskSubject = buildTaskSubjectV1({ routeSubject: routeSubject() }); + return { + schemaVersion: 1, + taskClassId: taskSubject.taskClassId, + taskClassVersion: taskSubject.taskClassVersion, + routeSubject: taskSubject.routeSubject, + taskSubject, + subjectId: subjectIdForSubject(taskSubject), + taskFixtureDigest: taskSubject.taskFixtureDigest, + verifierManifestDigest: taskSubject.verifierManifestDigest, + fabricCompatibilityVersion: taskSubject.fabricCompatibilityVersion, + sandboxProfileDigest: taskSubject.sandboxProfileDigest, + startedAt: 100, + completedAt: 200, + limits: { ...FABRIC_LIMITS }, + usage: { + inputBytes: 7, + outputBytes: 6, + patchOperations: 1, + filesTouched: 1, + artifactBytes: 0, + elapsedMs: 100, + inactiveMs: 0, + }, + outcome: "pass", + verifier: { + verifierId: FABRIC_VERIFIER_ID, + manifestDigest: taskSubject.verifierManifestDigest, + passed: true, + pathSummaries: [], + }, + artifactDigests: [], + }; +} + +function changedDigest(value: string): string { + return `${value[0] === "0" ? "1" : "0"}${value.slice(1)}`; +} + +describe("CL-07 fabric outcome validation", () => { + test("accepts a canonical outcome", () => { + const outcome = validOutcome(); + expect(assertFabricOutcomeV1(outcome)).toBe(outcome); + }); + + test("rejects undeclared nested producer fields", () => { + const outcome = validOutcome(); + const malformed = [ + { ...outcome, usage: { ...outcome.usage, credential: "secret" } }, + { ...outcome, limits: { ...outcome.limits, unexpectedLimit: 1 } }, + { ...outcome, taskSubject: { ...outcome.taskSubject, credential: "secret" } }, + { + ...outcome, + taskSubject: { + ...outcome.taskSubject, + routeSubject: { ...outcome.taskSubject.routeSubject, credential: "secret" }, + }, + }, + { ...outcome, routeSubject: { ...outcome.routeSubject, credential: "secret" } }, + ]; + + for (const candidate of malformed) { + expect(() => assertFabricOutcomeV1(candidate)).toThrow(FabricTaskError); + } + }); + + test("rejects negative or reversed execution timestamps", () => { + const outcome = validOutcome(); + expect(() => assertFabricOutcomeV1({ ...outcome, startedAt: -1 })).toThrow(FabricTaskError); + expect(() => assertFabricOutcomeV1({ ...outcome, startedAt: 201 })).toThrow(FabricTaskError); + expect(() => assertFabricOutcomeV1({ ...outcome, startedAt: 100.5 })).toThrow(FabricTaskError); + expect(() => assertFabricOutcomeV1({ ...outcome, completedAt: 200.5 })).toThrow(FabricTaskError); + }); + + test("rejects contradictory task identity fields", () => { + const outcome = validOutcome(); + const malformed = [ + { ...outcome, subjectId: changedDigest(outcome.subjectId) }, + { ...outcome, taskClassId: `${outcome.taskClassId}-other` }, + { ...outcome, taskFixtureDigest: changedDigest(outcome.taskFixtureDigest) }, + { + ...outcome, + verifier: { + ...outcome.verifier, + manifestDigest: changedDigest(outcome.verifier.manifestDigest), + }, + }, + ]; + + for (const candidate of malformed) { + expect(() => assertFabricOutcomeV1(candidate)).toThrow(FabricTaskError); + } + }); +});