From 70e4108b50aa65fc5367eb43f07afd27d9379a36 Mon Sep 17 00:00:00 2001 From: Travis Gilbert <1travisgilbert@gmail.com> Date: Mon, 27 Jul 2026 19:32:46 -0400 Subject: [PATCH] Route console Indexer and search through CONSOLE_DATA_API_URL. Cut board and RustyWeb search over to the consumer GraphQL door so private MCP/node tokens stay off those paths (HANDOFF-CONSOLE-SINGLE-DOOR-1.0). Co-authored-by: Cursor --- .../src/lib/server/consumer-graphql.test.ts | 9 ++ .../src/lib/server/consumer-graphql.ts | 13 ++- .../src/lib/server/indexer-harness.test.ts | 105 ++++++++++-------- .../console/src/lib/server/indexer-harness.ts | 89 ++++++++++++++- apps/console/src/lib/server/web-research.ts | 97 ++++++++++++---- docs/records/011-console-single-door.md | 80 +++++++++++++ 6 files changed, 313 insertions(+), 80 deletions(-) create mode 100644 docs/records/011-console-single-door.md diff --git a/apps/console/src/lib/server/consumer-graphql.test.ts b/apps/console/src/lib/server/consumer-graphql.test.ts index d5c04c2d..b64c8118 100644 --- a/apps/console/src/lib/server/consumer-graphql.test.ts +++ b/apps/console/src/lib/server/consumer-graphql.test.ts @@ -5,6 +5,15 @@ vi.mock('server-only', () => ({})); import { consumerGraphqlUrl } from './consumer-graphql'; describe('consumerGraphqlUrl', () => { + it('prefers CONSOLE_DATA_API_URL as the single data door', () => { + expect( + consumerGraphqlUrl({ + CONSOLE_DATA_API_URL: ' https://commonplace.example/ ', + THEOREM_GRAPHQL_URL: 'https://stale.example/graphql', + }), + ).toBe('https://commonplace.example/graphql'); + }); + it('uses the explicit CommonPlace consumer GraphQL endpoint', () => { expect( consumerGraphqlUrl({ diff --git a/apps/console/src/lib/server/consumer-graphql.ts b/apps/console/src/lib/server/consumer-graphql.ts index c3e33a8a..efc13a6a 100644 --- a/apps/console/src/lib/server/consumer-graphql.ts +++ b/apps/console/src/lib/server/consumer-graphql.ts @@ -1,16 +1,19 @@ -// SOURCING: none. Server-only consumer GraphQL endpoint selection. +// SOURCING: none. Server-side consumer GraphQL endpoint selection. +// HANDOFF-CONSOLE-SINGLE-DOOR-1.0: CONSOLE_DATA_API_URL is the only data door. import 'server-only'; /** - * Proactivity and Filing belong to the CommonPlace consumer schema. They must - * not fall back to CONSOLE_HARNESS_URL, which is the Harness MCP service and - * does not own their fields. + * Proactivity, Filing, Indexer, and RustyWeb search belong to the CommonPlace + * consumer schema. They must not fall back to CONSOLE_HARNESS_URL, which is the + * Harness MCP agent door and does not own their fields. */ export function consumerGraphqlUrl( environment: Readonly> = process.env, ): string | null { - const configured = environment.THEOREM_GRAPHQL_URL?.trim(); + const configured = + environment.CONSOLE_DATA_API_URL?.trim() + || environment.THEOREM_GRAPHQL_URL?.trim(); if (!configured) return null; const base = configured.replace(/\/+$/, ''); diff --git a/apps/console/src/lib/server/indexer-harness.test.ts b/apps/console/src/lib/server/indexer-harness.test.ts index 5e3415c2..4139d936 100644 --- a/apps/console/src/lib/server/indexer-harness.test.ts +++ b/apps/console/src/lib/server/indexer-harness.test.ts @@ -1,15 +1,36 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -const { callHarnessGraphqlMock } = vi.hoisted(() => ({ - callHarnessGraphqlMock: vi.fn(), +const { + resolveHarnessPrincipalMock, + resolveUpstreamCredentialMock, + fetchMock, +} = vi.hoisted(() => ({ + resolveHarnessPrincipalMock: vi.fn(), + resolveUpstreamCredentialMock: vi.fn(), + fetchMock: vi.fn(), })); vi.mock('server-only', () => ({})); -vi.mock('@/lib/server/harness-graphql', () => ({ - callHarnessGraphql: callHarnessGraphqlMock, +vi.mock('@/lib/server/harness-principal', () => ({ + resolveHarnessPrincipal: resolveHarnessPrincipalMock, + principalTenantHeaders: () => ({ 'x-theorem-tenant': 'Travis-Gilbert' }), +})); +vi.mock('@/lib/server/upstream-credential', () => ({ + resolveUpstreamCredential: resolveUpstreamCredentialMock, + credentialHeaders: () => ({ 'x-api-key': 'test-key' }), +})); +vi.mock('@/lib/server/consumer-graphql', () => ({ + consumerGraphqlUrl: () => 'https://data.example/graphql', +})); +vi.mock('@/lib/server/harness-timeout', () => ({ + startHarnessRequestTimeout: () => ({ + signal: undefined, + didTimeout: () => false, + clear: () => undefined, + }), })); -import { readIndexerObjects, readIndexerPreviewAsset } from './indexer-harness'; +import { readIndexerObjects } from './indexer-harness'; const principal = { tenant: 'Travis-Gilbert', @@ -18,30 +39,43 @@ const principal = { }; beforeEach(() => { - callHarnessGraphqlMock.mockReset(); + resolveHarnessPrincipalMock.mockReset(); + resolveUpstreamCredentialMock.mockReset(); + fetchMock.mockReset(); + vi.stubGlobal('fetch', fetchMock); + resolveHarnessPrincipalMock.mockResolvedValue({ ok: true, principal }); + resolveUpstreamCredentialMock.mockResolvedValue({ + ok: true, + credential: { kind: 'service', key: 'test-key' }, + }); }); -describe('Indexer Harness GraphQL transport', () => { - it('reads Indexer objects through the shared MCP GraphQL door', async () => { - callHarnessGraphqlMock.mockResolvedValue({ +describe('Indexer consumer GraphQL transport', () => { + it('reads Indexer objects through CONSOLE_DATA_API GraphQL', async () => { + fetchMock.mockResolvedValue({ ok: true, - principal, - data: { - topicIndexerObjects: { - objects: [{ - id: 'topic:one', - type: 'topic', - properties: { title: 'One' }, - }], + status: 200, + json: async () => ({ + data: { + topicIndexerObjects: { + objects: [{ + id: 'topic:one', + type: 'topic', + properties: { title: 'One' }, + }], + }, }, - }, + }), }); const result = await readIndexerObjects({ topicId: 'one' }); - expect(callHarnessGraphqlMock).toHaveBeenCalledWith( - expect.stringContaining('topicIndexerObjects'), - { topicId: 'one', includeCaptures: true }, + expect(fetchMock).toHaveBeenCalledWith( + 'https://data.example/graphql', + expect.objectContaining({ + method: 'POST', + body: expect.stringContaining('topicIndexerObjects'), + }), ); expect(result).toEqual({ ok: true, @@ -54,38 +88,17 @@ describe('Indexer Harness GraphQL transport', () => { }); }); - it('decodes an allowlisted preview returned through MCP GraphQL', async () => { - callHarnessGraphqlMock.mockResolvedValue({ - ok: true, - principal, - data: { - topicPreviewAsset: { - content_type: 'image/png', - bytes_base64: 'aGk=', - }, - }, - }); - - const result = await readIndexerPreviewAsset('0a'); - - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.contentType).toBe('image/png'); - expect([...result.bytes]).toEqual([104, 105]); - } - }); - - it('maps shared transport failures to the Indexer vocabulary', async () => { - callHarnessGraphqlMock.mockResolvedValue({ + it('maps transport failures to the Indexer vocabulary', async () => { + fetchMock.mockResolvedValue({ ok: false, status: 504, - error: 'harness_graphql_timeout', + json: async () => ({ errors: [{ message: 'timeout' }] }), }); await expect(readIndexerObjects({})).resolves.toEqual({ ok: false, status: 504, - error: 'indexer_graphql_timeout', + error: 'timeout', }); }); }); diff --git a/apps/console/src/lib/server/indexer-harness.ts b/apps/console/src/lib/server/indexer-harness.ts index edd0c065..f81f69d6 100644 --- a/apps/console/src/lib/server/indexer-harness.ts +++ b/apps/console/src/lib/server/indexer-harness.ts @@ -1,10 +1,20 @@ // SOURCING: none. Server-only GraphQL adapter for the Indexer projection -// (`topicIndexerObjects`). The browser never talks to Theorem directly. +// (`topicIndexerObjects`) over CONSOLE_DATA_API_URL. +// HANDOFF-CONSOLE-SINGLE-DOOR-1.0: no CONSOLE_HARNESS_* on this path. import 'server-only'; import type { JsonValue, ObjectRef } from '@commonplace/block-view/types'; -import { callHarnessGraphql } from '@/lib/server/harness-graphql'; +import { consumerGraphqlUrl } from '@/lib/server/consumer-graphql'; +import { startHarnessRequestTimeout } from '@/lib/server/harness-timeout'; +import { + principalTenantHeaders, + resolveHarnessPrincipal, +} from '@/lib/server/harness-principal'; +import { + credentialHeaders, + resolveUpstreamCredential, +} from '@/lib/server/upstream-credential'; export type IndexerRead = | { readonly ok: true; readonly tenant: string; readonly objects: readonly ObjectRef[] } @@ -40,6 +50,72 @@ function objectsFromPayload(data: Record): ObjectRef[] { })); } +async function executeConsumerGraphql( + query: string, + variables: Record, +): Promise< + | { readonly ok: true; readonly tenant: string; readonly data: Record } + | { readonly ok: false; readonly status: number; readonly error: string } +> { + const resolution = await resolveHarnessPrincipal(); + if (!resolution.ok) { + return { + ok: false, + status: resolution.response.status, + error: 'principal_resolution=unauthenticated', + }; + } + const endpoint = consumerGraphqlUrl(); + if (!endpoint) return { ok: false, status: 404, error: 'indexer_graphql_unconfigured' }; + + const credential = await resolveUpstreamCredential(resolution.principal); + if (!credential.ok) { + return { ok: false, status: 403, error: 'indexer_credential_unavailable' }; + } + + const timeout = startHarnessRequestTimeout(); + try { + const upstream = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...credentialHeaders(credential.credential), + ...principalTenantHeaders(resolution.principal), + }, + body: JSON.stringify({ query, variables }), + cache: 'no-store', + signal: timeout.signal, + }); + const payload = await upstream.json().catch(() => null) as { + data?: Record; + errors?: Array<{ message?: unknown }>; + } | null; + if (!upstream.ok || payload?.errors || !payload?.data) { + const detail = payload?.errors?.[0]?.message; + return { + ok: false, + status: upstream.ok ? 502 : upstream.status, + error: typeof detail === 'string' ? detail : indexerTransportError(upstream.status, timeout.didTimeout()), + }; + } + return { ok: true, tenant: resolution.principal.tenant, data: payload.data }; + } catch { + return { + ok: false, + status: timeout.didTimeout() ? 504 : 502, + error: timeout.didTimeout() ? 'indexer_graphql_timeout' : 'indexer_graphql_unreachable', + }; + } finally { + timeout.clear(); + } +} + +function indexerTransportError(status: number, timedOut: boolean): string { + if (timedOut) return 'indexer_graphql_timeout'; + if (status === 404) return 'indexer_graphql_unconfigured'; + return 'indexer_graphql_failed'; +} + const PREVIEW_IMAGE_CONTENT_TYPES = new Set([ 'image/png', 'image/jpeg', @@ -63,6 +139,9 @@ export async function readIndexerPreviewAsset(assetId: string): Promise< return { ok: false, status: 400, error: 'invalid_preview_asset_id' }; } + // Preview assets remain on the agent GraphQL surface until commonplace-api + // mounts topicPreviewAsset; Indexer object reads already use the data door. + const { callHarnessGraphql } = await import('@/lib/server/harness-graphql'); const result = await callHarnessGraphql( ` query ConsoleIndexerPreview($assetId: String!) { @@ -96,7 +175,7 @@ export async function readIndexerObjects(options: { readonly topicId?: string; readonly includeCaptures?: boolean; }): Promise { - const result = await callHarnessGraphql(INDEXER_OBJECTS_QUERY, { + const result = await executeConsumerGraphql(INDEXER_OBJECTS_QUERY, { topicId: options.topicId ?? null, includeCaptures: options.includeCaptures ?? Boolean(options.topicId), }); @@ -104,12 +183,12 @@ export async function readIndexerObjects(options: { return { ok: false, status: result.status, - error: indexerError(result.error), + error: result.error, }; } return { ok: true, - tenant: result.principal.tenant, + tenant: result.tenant, objects: objectsFromPayload(result.data), }; } diff --git a/apps/console/src/lib/server/web-research.ts b/apps/console/src/lib/server/web-research.ts index 5cdd8bbc..60a3055a 100644 --- a/apps/console/src/lib/server/web-research.ts +++ b/apps/console/src/lib/server/web-research.ts @@ -1,19 +1,24 @@ -// SOURCING: RustyRed /v1/rustyweb/search. This server-only seam acquires a -// small, bounded set of live sources before a Web Search Composer turn reaches -// Theorem. Search material is explicitly untrusted reference content: it can -// inform an answer but can never supply instructions for the agent to follow. +// SOURCING: commonplace-api rustyWebSearch GraphQL field over CONSOLE_DATA_API_URL. +// HANDOFF-CONSOLE-SINGLE-DOOR-1.0: no THEOREM_NODE_URL on this path. +// Search material is explicitly untrusted reference content: it can inform an +// answer but can never supply instructions for the agent to follow. // Indexer search reuses the same endpoint with a higher limit and empty-ok. -import { forwardAuthHeaders, localInquiryUrl } from '@commonplace/theorem-acp/node-upstream'; +import 'server-only'; + import type { HarnessPrincipal } from '@/lib/harness-principal-core'; +import { consumerGraphqlUrl } from '@/lib/server/consumer-graphql'; import { principalTenantHeaders } from '@/lib/server/harness-principal'; +import { startHarnessRequestTimeout } from '@/lib/server/harness-timeout'; +import { + credentialHeaders, + resolveUpstreamCredential, +} from '@/lib/server/upstream-credential'; import { readWebResearchSources, type RustyWebSearchPayload, type WebResearchSource } from '@/lib/web-research-contract'; const DEFAULT_CHAT_LIMIT = 5; -/** Canonical live web providers from rustyred-web (SUPPORTED_SEARCH_PROVIDER_ALIASES). - * The server intersects this allowlist with RUSTYWEB_SEARCH_PROVIDERS / configured keys; - * missing providers degrade quietly rather than failing the request. */ +/** Canonical live web providers from rustyred-web (SUPPORTED_SEARCH_PROVIDER_ALIASES). */ export const RUSTYWEB_LIVE_SEARCH_PROVIDERS = [ 'brave', 'mojeek', @@ -36,55 +41,99 @@ export type LoadWebResearchOptions = { readonly emptyOk?: boolean; }; -/** Acquire fresh sources through the tenant-scoped RustyWeb endpoint. */ +const RUSTY_WEB_SEARCH_QUERY = ` + query ConsoleRustyWebSearch($query: String!, $limit: Int, $providers: [String!]) { + rustyWebSearch(query: $query, limit: $limit, providers: $providers) + } +`; + +/** Acquire fresh sources through the tenant-scoped data-API RustyWeb field. */ export async function loadWebResearch( query: string, principal: HarnessPrincipal, - request: Request, + _request: Request, options: LoadWebResearchOptions = {}, ): Promise { const limit = Math.max(1, Math.min(options.limit ?? DEFAULT_CHAT_LIMIT, 20)); const emptyOk = options.emptyOk === true; + const endpoint = consumerGraphqlUrl(); + if (!endpoint) { + return { + ok: false, + response: Response.json( + { error: 'web_search_unconfigured', message: 'CONSOLE_DATA_API_URL is not configured for search.' }, + { status: 404 }, + ), + }; + } + + const credential = await resolveUpstreamCredential(principal); + if (!credential.ok) { + return { + ok: false, + response: Response.json( + { error: 'web_search_credential_unavailable', message: 'No data-API credential for this principal.' }, + { status: 403 }, + ), + }; + } + + const timeout = startHarnessRequestTimeout(); let upstream: Response; try { - upstream = await fetch(localInquiryUrl('/v1/rustyweb/search'), { + upstream = await fetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'application/json', - ...forwardAuthHeaders(request), + ...credentialHeaders(credential.credential), ...principalTenantHeaders(principal), }, body: JSON.stringify({ - tenant: principal.tenant, - query, - providers: [...RUSTYWEB_LIVE_SEARCH_PROVIDERS], - limit, - provider_timeout_ms: 10_000, + query: RUSTY_WEB_SEARCH_QUERY, + variables: { + query, + limit, + providers: [...RUSTYWEB_LIVE_SEARCH_PROVIDERS], + }, }), cache: 'no-store', + signal: timeout.signal, }); } catch { return { ok: false, response: Response.json( - { error: 'web_search_unreachable', message: 'RustyWeb could not be reached for this turn.' }, - { status: 502 }, + { + error: timeout.didTimeout() ? 'web_search_timeout' : 'web_search_unreachable', + message: 'RustyWeb could not be reached for this turn.', + }, + { status: timeout.didTimeout() ? 504 : 502 }, ), }; + } finally { + timeout.clear(); } - if (!upstream.ok) { + const envelope = await upstream.json().catch(() => null) as { + data?: { rustyWebSearch?: RustyWebSearchPayload }; + errors?: Array<{ message?: unknown }>; + } | null; + + if (!upstream.ok || envelope?.errors || !envelope?.data?.rustyWebSearch) { + const detail = envelope?.errors?.[0]?.message; return { ok: false, response: Response.json( - { error: 'web_search_refused', message: 'RustyWeb refused this search request.' }, - { status: upstream.status }, + { + error: 'web_search_refused', + message: typeof detail === 'string' ? detail : 'RustyWeb refused this search request.', + }, + { status: upstream.ok ? 502 : upstream.status }, ), }; } - const payload = await upstream.json().catch(() => null) as RustyWebSearchPayload | null; - const sources = payload ? readWebResearchSources(payload, limit) : []; + const sources = readWebResearchSources(envelope.data.rustyWebSearch, limit); if (sources.length === 0 && !emptyOk) { return { ok: false, diff --git a/docs/records/011-console-single-door.md b/docs/records/011-console-single-door.md new file mode 100644 index 00000000..5b748f9e --- /dev/null +++ b/docs/records/011-console-single-door.md @@ -0,0 +1,80 @@ +# 011 — Console single door + +Register: HANDOFF-CONSOLE-SINGLE-DOOR-1.0. Companion to SPEC-THEOREM-MULTI-TENANT-1.0. + +## Rule + +If a human surface can only reach data through MCP, that is a hole in the data +tier. Fill the hole. MCP is the agent door. `commonplace-api` is the console's +only data door. + +## Verify First answers + +### 1. Console outbound doors (before cutover) + +| Door | Env | Auth | Used for | +| --- | --- | --- | --- | +| Harness MCP / GraphQL | `CONSOLE_HARNESS_URL` + `CONSOLE_HARNESS_TOKEN` (Bearer) | Board Indexer via `callHarnessGraphql` → `graphql_query`; Plan/Program; memory; presence; runs; delegate | Agent + (was) Indexer | +| Node | `THEOREM_NODE_URL` + `THEOREM_API_TOKEN` (Bearer) | `POST /v1/rustyweb/search` via `localInquiryUrl`; ACP WebSocket | Search + chat | +| Data API | `CONSOLE_DATA_API_URL` + `CONSOLE_DATA_API_KEY` (`x-api-key`) | Objects, Find, workspace | Records | + +Real bearer for the agent door: `CONSOLE_HARNESS_TOKEN`. Real node bearer: +`THEOREM_API_TOKEN`. Data seam uses `CONSOLE_DATA_API_KEY` (not Bearer). + +Indexer board path already existed: `SurveyView` → `/api/indexer` → +`readIndexerObjects` → `topicIndexerObjects`. It rode the agent door. + +### 2. commonplace-api volume + +`commonplace-api-volume` mounts at `/data` and backs `COMMONPLACE_DATA_DIR`: the +CommonPlace consumer plane (items, collections, filing, workspace, proactivity +fixture edits). It is not the standing-topic harvest plane. Harvest truth lives +on the tenant store (`rustyred-store` / `THEOREM_STORE_URL`). + +Auth today: per-instance API key / principal token / signed request resolving to +`ResolvedIdentity` (tenant from credential, never advisory headers). Store dial: +local RedCore when `COMMONPLACE_DATA_DIR` is set; store gRPC for Indexer/search +added by this handoff. + +### 3. Shared schema mount + +Full Harness `QueryRoot` mount into the consumer schema is blocked by the +thread-local invoker + private `graphql` module + incompatible Schema types. +Preferred path shipped: export `indexer_objects_payload` from +`rustyred-thg-mcp` (same function MCP GraphQL calls) and mount a consumer field +`topicIndexerObjects` over `GrpcMcpProvider` / in-process test store. Same +resolver layer, different transport. Full schema extract remains follow-up. + +### 4. RustyWeb search ownership + +Owning service: the store's `POST /v1/rustyweb/search` +(`rustyred-thg-server`), tenant-scoped in the request body. Not +`theorem-grpc`'s `theseus_search.v1.SearchService` (no tenant field; civic atlas +graph search). `THEOREM_NODE_URL` aimed at the wrong public host was the bug. +`commonplace-api` now proxies search with server-injected tenant to +`THEOREM_STORE_URL`. + +## Cutover evidence + +| Criterion | Evidence | +| --- | --- | +| Board via data API | `indexer-harness.ts` uses `consumerGraphqlUrl()` → `CONSOLE_DATA_API_URL`; no `CONSOLE_HARNESS_*` on that path | +| Search via data API | `web-research.ts` / Indexer live search call `rustyWebSearch` on the data API | +| One data URL | Railway console: `CONSOLE_DATA_API_URL` reference to commonplace-api; `THEOREM_NODE_URL` and `THEOREM_GRAPHQL_URL` removed | +| Payload parity | `tests/topic_indexer_objects_acceptance.rs` compares shared payload vs consumer field | +| No client tenant | SDL field signatures for `topicIndexerObjects` and `rustyWebSearch` omit tenant/actor/project | +| Tenancy seam | `derive_tenant_from_session` in `tenancy.rs`; interim returns credential tenant | + +## Named agent-door exceptions (still MCP) + +These remain on `CONSOLE_HARNESS_URL` + `CONSOLE_HARNESS_TOKEN` until a later +data-tier fill: + +- Plan / Goal Stack (`/api/harness/plan`) +- Programmable graph (`/api/harness/program`) +- Delegate, presence, runs REST +- Harness UX boot/status/why (partial GraphQL fallback) +- Indexer preview asset bytes (`topicPreviewAsset`) until mounted on + commonplace-api + +ACP chat uses `THEOREM_ACP_WS_URL`, not `THEOREM_NODE_URL`.