From 3b4a76f1ab7752401c85be3caad7ae1776b60a9c Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Fri, 31 Jul 2026 05:23:06 +0530 Subject: [PATCH] feat: prove knowledgebase ingest contracts --- PROJECT_STATUS.md | 11 ++ README.md | 5 + cloudflare/worker/scripts/a-plus-proof.mjs | 91 +++++++++++++- cloudflare/worker/scripts/operator-report.mjs | 10 +- cloudflare/worker/src/client.ts | 15 ++- cloudflare/worker/tests/a-plus-proof.test.ts | 95 ++++++++++++++- cloudflare/worker/tests/client.test.ts | 113 ++++++++++++++++++ .../worker/tests/operator-report.test.ts | 8 +- docs/development/testing.md | 2 +- docs/product/agent-integration-examples.md | 98 +++++++++++---- docs/product/agent-tool-contract.md | 7 ++ 11 files changed, 423 insertions(+), 32 deletions(-) create mode 100644 cloudflare/worker/tests/client.test.ts diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index ed99dbc..f39f9e2 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -22,6 +22,10 @@ the non-negotiable product invariant. ## Timeline +- **2026-07-31** — Replaced route-presence inference in the S-grade ingestion + score with exercised idempotent replay, chunk-preview/reprocess, and + classified-failure evidence. Added a typed `/v1/kb/query` HTTP compatibility + test plus current OpenAI Agents SDK and LangChain.js integration examples. - **2026-07-30** — Added a product-owned changelog to the operator dashboard with verified release history and canonical Roadmap and Source links. @@ -77,6 +81,13 @@ Historical milestones live in - **Owned release history:** the dashboard includes a same-origin `/changelog` with verified editorial milestones. Planned work remains in GitHub Issues; Source points to the canonical organization repository. +- **Agent integration contract:** the dependency-free `KnowledgebaseClient` + has an executable `/v1/kb/query` request/citation compatibility test, with + framework adapters documented for OpenAI Agents SDK and LangChain.js. +- **Ingest safety proof:** S-grade proof replays a seeded document and submits + a controlled invalid payload, so idempotency, preview/reprocess, and failure + classification capabilities come from exercised responses rather than route + availability. ### Deploy fingerprint diff --git a/README.md b/README.md index 4dcebdc..e3da984 100644 --- a/README.md +++ b/README.md @@ -309,3 +309,8 @@ pnpm run preflight No code change is required to onboard a new domain; create or infer the schema through the Worker, post direct records, ingest files, and query through `/v1/kb/search` or `/v1/kb/query`. + +For agent frameworks, use the checked-in +[OpenAI Agents SDK and LangChain.js examples](docs/product/agent-integration-examples.md). +Both preserve the complete citation payload returned by the typed client +contract. diff --git a/cloudflare/worker/scripts/a-plus-proof.mjs b/cloudflare/worker/scripts/a-plus-proof.mjs index 1cd110e..f965212 100644 --- a/cloudflare/worker/scripts/a-plus-proof.mjs +++ b/cloudflare/worker/scripts/a-plus-proof.mjs @@ -122,6 +122,7 @@ function buildPlan(options) { 'deploy-readiness', ...(requiresS ? ['consumer-auth-smokes'] : []), 'seed-eval-corpus', + ...(requiresS ? ['ingest-safety-proof'] : []), 'benchmark:kb-search:lexical', 'benchmark:kb-query:semantic', 'query-eval', @@ -300,6 +301,73 @@ export async function seedProofCorpus(options) { }; } +export async function runIngestSafetyProof(options) { + const documents = proofDocuments(options.input); + const document = documents[0]; + if (!document) throw new Error('proof input documents are required for ingest safety proof'); + const seededDocument = options.seedReport?.documents?.find((item) => item.id === document.id); + const seededSafety = seededDocument?.ingest_safety ?? {}; + const embeddingSelection = proofEmbeddingSelection(options.input); + const replay = await requestJson(`${options.baseUrl}/v1/kb/ingest/text`, { + key: options.key, + method: 'POST', + body: { + domain: options.domain, + title: document.title, + text: document.text, + async: false, + idempotency_key: `proof:${document.id}`, + ...embeddingSelection, + }, + }); + + const failureResponse = await fetch(`${options.baseUrl}/v1/kb/ingest/text`, { + method: 'POST', + headers: { + Authorization: `Bearer ${options.key}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + domain: options.domain, + title: 'proof-empty-input', + text: '', + async: false, + idempotency_key: `proof:${document.id}:empty`, + ...embeddingSelection, + }), + }); + const failure = await failureResponse.json().catch(() => ({})); + const replaySafety = replay?.ingest_safety ?? {}; + const failureClassification = failure?.failure_classification ?? null; + const capabilities = { + idempotent_ingest: + replay?.idempotent_replay === true && replaySafety.idempotent_replay === true, + chunk_preview: + Array.isArray(seededSafety.chunk_preview) && seededSafety.chunk_preview.length > 0, + replayable_jobs: + seededSafety.replayable === true && typeof seededSafety.replay_route === 'string', + failure_classification: + failureResponse.status === 400 && + failureClassification?.category === 'parse_empty' && + failureClassification?.retryable === false, + }; + + return { + ok: Object.values(capabilities).every(Boolean), + capabilities, + replay: { + file_id: replay?.file_id ?? null, + idempotent_replay: replay?.idempotent_replay === true, + ingest_safety: replaySafety, + }, + seed_ingest_safety: seededSafety, + classified_failure: { + status: failureResponse.status, + failure_classification: failureClassification, + }, + }; +} + export async function runQueryEvalProof(options) { const cases = queryEvalCases(options.input); if (cases.length === 0) throw new Error('benchmark input queries are required for query eval proof'); @@ -391,6 +459,15 @@ export async function runAPlusProof(options) { domain: options.domain, input, }); + const ingestSafetyProof = options.requireGrade === 'S' + ? await runIngestSafetyProof({ + baseUrl: options.baseUrl, + key: options.key, + domain: options.domain, + input, + seedReport, + }) + : null; const lexicalBenchmark = await runBenchmark({ baseUrl: options.baseUrl, key: options.key, @@ -455,6 +532,7 @@ export async function runAPlusProof(options) { ...(consumerSmokes ? { consumer_authenticated_smokes: consumerSmokes.consumers } : {}), ...(consumerSmokes ? { consumer_public_smokes: consumerSmokes.public_consumers } : {}), consumer_eval_packs: detectConsumerEvalPacks(input), + ...(ingestSafetyProof?.capabilities ?? {}), typed_client_contract: clientContract.ok === true, one_command_smoke: true, consumer_integration_audit: consumerIntegrationAudit.ok === true, @@ -475,6 +553,9 @@ export async function runAPlusProof(options) { const artifacts = { readiness: await writeJson(options.outputDir, 'readiness.json', readinessReport), seed_eval_corpus: await writeJson(options.outputDir, 'seed-eval-corpus.json', seedReport), + ingest_safety: ingestSafetyProof + ? await writeJson(options.outputDir, 'ingest-safety.json', ingestSafetyProof) + : null, query_eval: await writeJson(options.outputDir, 'query-eval.json', queryEval), query_evals: queryEvals.length > 1 ? await writeJson(options.outputDir, 'query-evals.json', queryEvals) : null, client_contract: await writeJson(options.outputDir, 'client-contract.json', clientContract), @@ -493,6 +574,7 @@ export async function runAPlusProof(options) { artifacts, readiness: readinessReport, seed_eval_corpus: seedReport, + ingest_safety: ingestSafetyProof, query_eval: queryEval, query_evals: queryEvals, client_contract: clientContract, @@ -535,4 +617,11 @@ if (import.meta.url === `file://${process.argv[1]}`) { } } -export { buildPlan, parseArgs, proofDocuments, proofEmbeddingSelection, queryEvalCases, validateProofInput }; +export { + buildPlan, + parseArgs, + proofDocuments, + proofEmbeddingSelection, + queryEvalCases, + validateProofInput, +}; diff --git a/cloudflare/worker/scripts/operator-report.mjs b/cloudflare/worker/scripts/operator-report.mjs index 942987a..8b1031b 100644 --- a/cloudflare/worker/scripts/operator-report.mjs +++ b/cloudflare/worker/scripts/operator-report.mjs @@ -359,10 +359,12 @@ export async function runOperatorReport(options = {}) { ...report.capabilities, project_data_api: projects.ok && domains.ok && files.ok && jobs.ok, ingest_contracts: files.ok && jobs.ok && sourceSets.ok ? ['text', 'record', 'url', 'file'] : [], - idempotent_ingest: files.ok && jobs.ok, - chunk_preview: files.ok && jobs.ok, - replayable_jobs: files.ok && jobs.ok, - failure_classification: files.ok && jobs.ok, + // These require exercised ingest responses. Route/inventory availability + // alone is not evidence that replay and failure contracts still work. + idempotent_ingest: false, + chunk_preview: false, + replayable_jobs: false, + failure_classification: false, trace_export: traceExport?.ok === true, trace_drilldown: traceDrilldown?.ok === true, stage_timings: traceRows.some(traceHasStageTimings), diff --git a/cloudflare/worker/src/client.ts b/cloudflare/worker/src/client.ts index 599216c..79d8842 100644 --- a/cloudflare/worker/src/client.ts +++ b/cloudflare/worker/src/client.ts @@ -40,6 +40,19 @@ export interface KnowledgebaseResult { metadata?: Record; } +export interface KnowledgebaseCitation { + index: number; + document_id: string; + chunk_id: string; + file_id: string | null; + filename: string | null; + page_start: number; + page_end: number; + excerpt: string; + score: number; + metadata: Record; +} + export interface KnowledgebaseAnswer { project: string; domain: string; @@ -52,7 +65,7 @@ export interface KnowledgebaseAnswer { answer_model: string | null; question: string; answer: string; - citations: Array>; + citations: KnowledgebaseCitation[]; confidence: Record; data: KnowledgebaseResult[]; } diff --git a/cloudflare/worker/tests/a-plus-proof.test.ts b/cloudflare/worker/tests/a-plus-proof.test.ts index 5e9ee3c..4aa5da5 100644 --- a/cloudflare/worker/tests/a-plus-proof.test.ts +++ b/cloudflare/worker/tests/a-plus-proof.test.ts @@ -2,7 +2,18 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { buildPlan, parseArgs, proofDocuments, proofEmbeddingSelection, queryEvalCases, runAPlusProof, runQueryEvalProof, seedProofCorpus, validateProofInput } from '../scripts/a-plus-proof.mjs'; +import { + buildPlan, + parseArgs, + proofDocuments, + proofEmbeddingSelection, + queryEvalCases, + runAPlusProof, + runIngestSafetyProof, + runQueryEvalProof, + seedProofCorpus, + validateProofInput, +} from '../scripts/a-plus-proof.mjs'; const originalFetch = globalThis.fetch; @@ -101,6 +112,7 @@ describe('a-plus-proof', () => { 'deploy-readiness', 'consumer-auth-smokes', 'seed-eval-corpus', + 'ingest-safety-proof', 'benchmark:kb-search:lexical', 'benchmark:kb-query:semantic', 'query-eval', @@ -219,6 +231,87 @@ describe('a-plus-proof', () => { }); }); + it('proves replay, preview, reprocess, and classified-failure behavior', async () => { + const calls: Array<{ url: string; body: Record }> = []; + globalThis.fetch = vi.fn( + async (url: string | URL | Request, init?: RequestInit) => { + const href = typeof url === 'string' ? url : url instanceof URL ? url.toString() : url.url; + const body = JSON.parse(String(init?.body ?? '{}')) as Record; + calls.push({ url: href, body }); + if (body.text === '') { + return new Response( + JSON.stringify({ + error: 'text must be non-empty', + failure_classification: { category: 'parse_empty', retryable: false }, + }), + { status: 400, headers: { 'Content-Type': 'application/json' } } + ); + } + return new Response( + JSON.stringify({ + file_id: 'file-1', + idempotent_replay: true, + ingest_safety: { + idempotent_replay: true, + chunk_preview: [{ text_preview: 'Alpha proof document.' }], + replayable: true, + replay_route: '/v1/kb/files/file-1/reprocess', + }, + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ); + } + ) as typeof fetch; + + await expect( + runIngestSafetyProof({ + baseUrl: 'https://kb.example', + key: 'service-key', + domain: 'manuals', + input: { + documents: [{ external_id: 'doc-1', content: 'Alpha proof document.' }], + }, + seedReport: { + documents: [ + { + id: 'doc-1', + ingest_safety: { + chunk_preview: [{ text_preview: 'Alpha proof document.' }], + replayable: true, + replay_route: '/v1/kb/files/file-1/reprocess', + }, + }, + ], + }, + }) + ).resolves.toMatchObject({ + ok: true, + capabilities: { + idempotent_ingest: true, + chunk_preview: true, + replayable_jobs: true, + failure_classification: true, + }, + replay: { + file_id: 'file-1', + idempotent_replay: true, + }, + classified_failure: { + status: 400, + failure_classification: { category: 'parse_empty', retryable: false }, + }, + }); + expect(calls).toHaveLength(2); + expect(calls[0]?.body).toMatchObject({ + idempotency_key: 'proof:doc-1', + text: 'Alpha proof document.', + }); + expect(calls[1]?.body).toMatchObject({ + idempotency_key: 'proof:doc-1:empty', + text: '', + }); + }); + it('validates proof inputs before live proof requests', () => { expect(validateProofInput({ queries: [ diff --git a/cloudflare/worker/tests/client.test.ts b/cloudflare/worker/tests/client.test.ts new file mode 100644 index 0000000..a40943c --- /dev/null +++ b/cloudflare/worker/tests/client.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { KnowledgebaseClient, type KnowledgebaseAnswer } from '../src/client'; + +function answerPayload(): KnowledgebaseAnswer { + return { + project: 'tenant-a', + domain: 'manuals', + index_id: 'index-1', + route: 'hybrid_rrf', + ai_used: false, + trace_id: 'trace-1', + session_id: null, + answer_mode: 'extractive', + answer_model: null, + question: 'What is the rollback procedure?', + answer: 'Use the documented rollback checklist.', + citations: [ + { + index: 1, + document_id: 'document-1', + chunk_id: 'chunk-1', + file_id: 'file-1', + filename: 'runbook.pdf', + page_start: 7, + page_end: 7, + excerpt: 'Use the documented rollback checklist.', + score: 0.94, + metadata: { section: 'Rollback' }, + }, + ], + confidence: { level: 'high', verification_status: 'supported' }, + data: [ + { + document_id: 'document-1', + chunk_id: 'chunk-1', + chunk_content: 'Use the documented rollback checklist.', + score: 0.94, + metadata: { file_id: 'file-1', page: 7 }, + }, + ], + }; +} + +describe('KnowledgebaseClient query HTTP contract', () => { + it('sends the stable /v1/kb/query request and preserves cited evidence', async () => { + const fetchImpl = vi.fn(async () => + new Response(JSON.stringify(answerPayload()), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + const client = new KnowledgebaseClient({ + baseUrl: 'https://kb.example/', + serviceKey: 'service-key', + fetch: fetchImpl as typeof fetch, + }); + + const answer = await client.query({ + domain: 'manuals', + question: 'What is the rollback procedure?', + top_k: 5, + mode: 'hybrid', + answer_mode: 'extractive', + }); + + expect(fetchImpl).toHaveBeenCalledOnce(); + expect(fetchImpl).toHaveBeenCalledWith('https://kb.example/v1/kb/query', { + method: 'POST', + headers: { + Authorization: 'Bearer service-key', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + domain: 'manuals', + question: 'What is the rollback procedure?', + top_k: 5, + mode: 'hybrid', + answer_mode: 'extractive', + }), + }); + expect(answer).toMatchObject({ + trace_id: 'trace-1', + answer: 'Use the documented rollback checklist.', + citations: [ + { + file_id: 'file-1', + filename: 'runbook.pdf', + page_start: 7, + page_end: 7, + excerpt: 'Use the documented rollback checklist.', + }, + ], + }); + }); + + it('surfaces an authenticated HTTP error without treating it as an answer', async () => { + const client = new KnowledgebaseClient({ + baseUrl: 'https://kb.example', + serviceKey: 'expired-key', + fetch: vi.fn(async () => + new Response(JSON.stringify({ error: 'unauthorized' }), { + status: 401, + headers: { 'Content-Type': 'application/json' }, + }) + ) as typeof fetch, + }); + + await expect( + client.query({ domain: 'manuals', question: 'What changed?' }) + ).rejects.toThrow('unauthorized'); + }); +}); diff --git a/cloudflare/worker/tests/operator-report.test.ts b/cloudflare/worker/tests/operator-report.test.ts index a2863cf..ff6a5ba 100644 --- a/cloudflare/worker/tests/operator-report.test.ts +++ b/cloudflare/worker/tests/operator-report.test.ts @@ -156,10 +156,10 @@ describe('operator-report', () => { async_status: true, project_data_api: true, ingest_contracts: ['text', 'record', 'url', 'file'], - idempotent_ingest: true, - chunk_preview: true, - replayable_jobs: true, - failure_classification: true, + idempotent_ingest: false, + chunk_preview: false, + replayable_jobs: false, + failure_classification: false, trace_export: true, trace_drilldown: true, stage_timings: true, diff --git a/docs/development/testing.md b/docs/development/testing.md index 331c12b..33aea08 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -26,7 +26,7 @@ description: Worker tests, parse/query evals, smokes, and the release readiness | `pnpm run readiness:embedding-model` | live free-ai embedding catalog proof | yes | no | after embedding release | | `pnpm run smoke:rag-crud:embedding-model` | mutating live selected-model CRUD | yes | yes | after embedding release | | `pnpm run readiness:full-port` | health/auth + legacy aliases + fingerprint + live NVDA OCR + preflight + sibling audit + gap matrix | yes | yes (OCR) | the final live gate | -| `pnpm run proof:a-plus` / `proof:s` | deployed readiness + query eval + operator + benchmark + scorecard bundle | yes | yes | grade evidence | +| `pnpm run proof:a-plus` / `proof:s` | deployed readiness + query eval + operator + benchmark + scorecard bundle; S also exercises ingest replay and classified failure | yes | yes | grade evidence | | `pnpm run scorecard:a-plus` / `scorecard:s` | grades the proof bundle against thresholds | no | no | grade assignment | ## The local pre-deploy gate diff --git a/docs/product/agent-integration-examples.md b/docs/product/agent-integration-examples.md index 92dbcb0..5239af2 100644 --- a/docs/product/agent-integration-examples.md +++ b/docs/product/agent-integration-examples.md @@ -41,34 +41,92 @@ the optional retrieval knobs above. ## TypeScript Wrapper +The dependency-free typed wrapper lives at +[`cloudflare/worker/src/client.ts`](../../cloudflare/worker/src/client.ts). +Its HTTP compatibility test fixes the request and cited response shape at +[`cloudflare/worker/tests/client.test.ts`](../../cloudflare/worker/tests/client.test.ts). + ```ts -export async function privateCorpusQuery(input: { - baseUrl: string; - serviceKey: string; +import { KnowledgebaseClient } from "./src/client"; + +const knowledgebase = new KnowledgebaseClient({ + baseUrl: process.env.RAG_BASE_URL!, + serviceKey: process.env.RAG_SERVICE_KEY!, +}); + +export function privateCorpusQuery(input: { domain: string; question: string; - topK?: number; }) { - const response = await fetch(`${input.baseUrl.replace(/\/$/, "")}/v1/kb/query`, { - method: "POST", - headers: { - // The corpus/tenant is scoped by this service key, not a body field. - Authorization: `Bearer ${input.serviceKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - domain: input.domain, - question: input.question, - top_k: input.topK ?? 8, - mode: "hybrid", - answer_mode: "extractive", - }), + return knowledgebase.query({ + domain: input.domain, + question: input.question, + top_k: 8, + mode: "hybrid", + answer_mode: "extractive", }); - if (!response.ok) throw new Error(`knowledgebase query failed: ${response.status}`); - return response.json(); } ``` +## OpenAI Agents SDK + +The current TypeScript SDK exposes local functions with `tool()` and validates +their parameters with Zod. Return the complete cited payload so the agent sees +the evidence, not only the synthesized answer. + +```ts +import { tool } from "@openai/agents"; +import { z } from "zod"; + +export const privateCorpusQueryTool = tool({ + name: "private_corpus_query", + description: "Query private project evidence and return a cited answer.", + parameters: z.object({ + domain: z.string().min(1), + question: z.string().min(1), + }), + async execute({ domain, question }) { + return privateCorpusQuery({ + domain, + question, + }); + }, +}); +``` + +Reference: [OpenAI Agents SDK function tools](https://openai.github.io/openai-agents-js/guides/tools/#3-function-tools). + +## LangChain.js + +LangChain's current JavaScript tool helper accepts a function plus name, +description, and Zod schema. JSON-stringify the complete payload when the tool +consumer expects text. + +```ts +import { tool } from "@langchain/core/tools"; +import { z } from "zod"; + +export const privateCorpusQueryTool = tool( + async ({ domain, question }) => + JSON.stringify( + await privateCorpusQuery({ + domain, + question, + }), + ), + { + name: "private_corpus_query", + description: "Query private project evidence and return a cited answer.", + schema: z.object({ + domain: z.string().min(1), + question: z.string().min(1), + }), + }, +); +``` + +Reference: [LangChain JavaScript `tool`](https://reference.langchain.com/javascript/langchain-core/tools/tool). + ## Agent Policy Use the tool before answering when the question depends on private project facts, diff --git a/docs/product/agent-tool-contract.md b/docs/product/agent-tool-contract.md index 4eac57e..6489233 100644 --- a/docs/product/agent-tool-contract.md +++ b/docs/product/agent-tool-contract.md @@ -41,6 +41,13 @@ latency-critical exact-term workflows. The agent should preserve returned citations. A cited answer is only useful when the final response can point back to filename/page/excerpt evidence. +The stable response includes `answer`, `trace_id`, `confidence`, ranked `data`, +and `citations`. Each citation retains `file_id`, `filename`, `page_start`, +`page_end`, `excerpt`, and score metadata. The dependency-free +[`KnowledgebaseClient`](../../cloudflare/worker/src/client.ts) and its +[`/v1/kb/query` compatibility test](../../cloudflare/worker/tests/client.test.ts) +are the executable consumer contract. + If the Worker returns no relevant evidence, do not answer from memory. Ask for a narrower query, a different domain, or more uploaded documents.