diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 3892fc5..4d1c32e 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -53,6 +53,29 @@ Then check the routes above against `https://omni.syberlabs.io`. address. To undo a bad release, `npx wrangler rollback` restores the prior version. +### Logs + +`wrangler.jsonc` turns on Workers Logs (`observability.enabled`) with request +logs off (`observability.logs.invocation_logs: false`). Cloudflare then keeps +no log per request, the entry that would carry its method, full URL and +request metadata, which can include the client address. From the app's own +code, Workers Logs keeps only the allowlisted event lines below, for 3 days +on the Workers Free plan and 7 on Paid. Cloudflare's docs do not say what +metadata it attaches to such a line, so check one from the deployed Worker +before counting on it carrying no URL or address. + +The lines are JSON, one per event, written only through +`src/core/services/server/telemetry.ts`; a test fails on any other console +call under `src/app/api` or `src/core/services/server`. +`telemetry/events.json` names every event and every field an event may +carry. Error detail is the error's class name, never its message. Errors go +to `console.error`, the rest to `console.log`. vinext and React can still +write their own error lines, and the runtime still records an uncaught +exception; neither is on the allowlist. + +Turning request logs on would keep full URLs and can keep client addresses. +Do it only after syberlabs.io's privacy page says the preview keeps them. + ### Automatic deploys (off until enabled) `.github/workflows/deploy-preview.yml` runs the steps above after CI passes diff --git a/src/app/api/polymarket/route.test.ts b/src/app/api/polymarket/route.test.ts index b3971c1..ffc0c6b 100644 --- a/src/app/api/polymarket/route.test.ts +++ b/src/app/api/polymarket/route.test.ts @@ -3,10 +3,10 @@ import { GET } from './route'; const BODY_MARKER = 'UPSTREAM-BODY-MUST-NOT-BE-LOGGED'; -let warn: ReturnType; +let log: ReturnType; beforeEach(() => { - warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + log = vi.spyOn(console, 'log').mockImplementation(() => {}); }); afterEach(() => { @@ -14,9 +14,11 @@ afterEach(() => { vi.unstubAllGlobals(); }); +/** The whole failure body, so a change to what the route answers shows up here. */ async function failureOf(response: Response) { expect(response.status).toBe(502); const body = await response.json(); + expect(Object.keys(body).sort()).toEqual(['error', 'success']); expect(body.success).toBe(false); return body.error as string; } @@ -37,21 +39,23 @@ describe('GET /api/polymarket', () => { expect(body.markets).toHaveLength(1); expect(body.markets[0].outcomes[0]).toEqual({ id: 'mkt-1_0', name: 'Yes', probability: 0.65 }); expect(body.markets[0].volume).toBe(12345); + expect(log).not.toHaveBeenCalled(); }); it('reports an upstream 500 as a failure, not an empty success', async () => { vi.stubGlobal('fetch', vi.fn(async () => new Response(BODY_MARKER, { status: 500 }))); expect(await failureOf(await GET())).toBe('upstream_status'); - expect(warn).toHaveBeenCalledWith('[Polymarket API] upstream returned', 500); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"polymarket","reason":"status","status":500}'); }); - it('reports an unexpected shape as a failure and logs keys, never the body', async () => { + it('reports an unexpected shape as a failure and logs neither the body nor its keys', async () => { vi.stubGlobal('fetch', vi.fn(async () => Response.json({ surprise: BODY_MARKER, nested: { secret: BODY_MARKER } }))); expect(await failureOf(await GET())).toBe('upstream_shape'); - expect(warn).toHaveBeenCalledTimes(1); - const logged = JSON.stringify(warn.mock.calls); + expect(log).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"polymarket","reason":"shape","status":200}'); + const logged = JSON.stringify(log.mock.calls); expect(logged).not.toContain(BODY_MARKER); - expect(logged).toContain('surprise'); + expect(logged).not.toContain('surprise'); }); it('keeps the body out of the log when upstream returns non-JSON', async () => { @@ -59,7 +63,8 @@ describe('GET /api/polymarket', () => { status: 200, headers: { 'content-type': 'text/html' } }))); expect(await failureOf(await GET())).toBe('upstream_unreachable'); - expect(JSON.stringify(warn.mock.calls)).not.toContain(BODY_MARKER); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"polymarket","reason":"network","kind":"SyntaxError"}'); + expect(JSON.stringify(log.mock.calls)).not.toContain(BODY_MARKER); }); it('bounds the upstream fetch and reports a timeout as a failure', async () => { @@ -71,5 +76,6 @@ describe('GET /api/polymarket', () => { })); expect(await failureOf(await GET())).toBe('upstream_timeout'); expect(timeoutSpy).toHaveBeenCalledWith(8000); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"polymarket","reason":"timeout","kind":"TimeoutError"}'); }); }); diff --git a/src/app/api/polymarket/route.ts b/src/app/api/polymarket/route.ts index 348c079..8fe6694 100644 --- a/src/app/api/polymarket/route.ts +++ b/src/app/api/polymarket/route.ts @@ -3,6 +3,7 @@ // ============================================ import { NextResponse } from 'next/server'; +import { errorKind, logEvent } from '@/core/services/server/telemetry'; const UPSTREAM_URL = 'https://gamma-api.polymarket.com/markets?limit=50&closed=false&order=volumeNum&ascending=false'; const UPSTREAM_DEADLINE_MS = 8000; @@ -62,8 +63,8 @@ function asString(value: unknown, fallback: string): string { * GET /api/polymarket * * Upstream failure is reported as 502 `{ success: false, error }` so the - * settings probe and the block see the outage. Logs carry the upstream - * status and top-level keys, never the body. + * settings probe and the block see the outage. Logs carry the failure, the + * upstream status and the error class, never the body or its keys. */ export async function GET() { try { @@ -77,7 +78,7 @@ export async function GET() { }); if (!response.ok) { - console.warn('[Polymarket API] upstream returned', response.status); + logEvent('flyspace.source.failed', { provider: 'polymarket', reason: 'status', status: response.status }); return failure('upstream_status'); } @@ -85,10 +86,7 @@ export async function GET() { const marketsArray = marketsFromRaw(rawData); if (marketsArray === null) { - console.warn('[Polymarket API] unexpected upstream shape', { - status: response.status, - keys: isRecord(rawData) ? Object.keys(rawData) : typeof rawData - }); + logEvent('flyspace.source.failed', { provider: 'polymarket', reason: 'shape', status: response.status }); return failure('upstream_shape'); } @@ -148,7 +146,11 @@ export async function GET() { const name = error instanceof Error ? error.name : 'Unknown'; const timedOut = name === 'TimeoutError' || name === 'AbortError'; // Only the error name: a JSON SyntaxError message quotes the body. - console.warn('[Polymarket API] fetch failed:', name); + logEvent('flyspace.source.failed', { + provider: 'polymarket', + reason: timedOut ? 'timeout' : 'network', + kind: errorKind(error) + }); return failure(timedOut ? 'upstream_timeout' : 'upstream_unreachable'); } } diff --git a/src/app/api/public/route.test.ts b/src/app/api/public/route.test.ts index 0327ef1..e02edee 100644 --- a/src/app/api/public/route.test.ts +++ b/src/app/api/public/route.test.ts @@ -4,10 +4,10 @@ import { GET } from './route'; const BODY_MARKER = 'UPSTREAM-BODY-MUST-NOT-BE-LOGGED'; -let warn: ReturnType; +let log: ReturnType; beforeEach(() => { - warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + log = vi.spyOn(console, 'log').mockImplementation(() => {}); }); afterEach(() => { @@ -62,7 +62,7 @@ describe('GET /api/public', () => { expect(await failureOf(await GET(request('provider=wikipedia')))).toBe('upstream_timeout'); expect(timeoutSpy).toHaveBeenCalledWith(8000); - expect(JSON.stringify(warn.mock.calls)).toContain('upstream_timeout'); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"wikipedia","reason":"timeout","kind":"TimeoutError"}'); }); it('does not follow a redirect and reports it as 502 upstream_redirect', async () => { @@ -74,8 +74,9 @@ describe('GET /api/public', () => { expect(await failureOf(await GET(request('provider=frankfurter')))).toBe('upstream_redirect'); expect(fetchMock).toHaveBeenCalledTimes(1); - const logged = JSON.stringify(warn.mock.calls); - expect(logged).toContain('evil.example'); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"frankfurter","reason":"redirect","status":302}'); + const logged = JSON.stringify(log.mock.calls); + expect(logged).not.toContain('evil.example'); expect(logged).not.toContain('SECRET-IN-LOCATION'); }); @@ -86,6 +87,7 @@ describe('GET /api/public', () => { }))); expect(await failureOf(await GET(request('provider=crossref')))).toBe('upstream_too_large'); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"crossref","reason":"too_large"}'); }); it('reports an upstream non-2xx as 502 upstream_status and logs the status, never the body', async () => { @@ -94,9 +96,8 @@ describe('GET /api/public', () => { }))); expect(await failureOf(await GET(request('provider=github')))).toBe('upstream_status'); - const logged = JSON.stringify(warn.mock.calls); - expect(logged).toContain('403'); - expect(logged).not.toContain(BODY_MARKER); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"github","reason":"status","status":403}'); + expect(JSON.stringify(log.mock.calls)).not.toContain(BODY_MARKER); }); it('reports non-JSON upstream as 502 upstream_shape and keeps the body out of the log', async () => { @@ -105,7 +106,8 @@ describe('GET /api/public', () => { }))); expect(await failureOf(await GET(request('provider=openlibrary')))).toBe('upstream_shape'); - expect(JSON.stringify(warn.mock.calls)).not.toContain(BODY_MARKER); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"openlibrary","reason":"shape","kind":"SyntaxError"}'); + expect(JSON.stringify(log.mock.calls)).not.toContain(BODY_MARKER); }); it('reports a network failure as 502 upstream_unreachable without echoing the error text', async () => { @@ -116,6 +118,7 @@ describe('GET /api/public', () => { const body = await response.json(); expect(body.error.code).toBe('upstream_unreachable'); expect(JSON.stringify(body)).not.toContain('secret-in-url'); - expect(JSON.stringify(warn.mock.calls)).not.toContain('secret-in-url'); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"openmeteo","reason":"network","kind":"Error"}'); + expect(JSON.stringify(log.mock.calls)).not.toContain('secret-in-url'); }); }); diff --git a/src/core/services/server/public.providers.test.ts b/src/core/services/server/public.providers.test.ts index 56d8160..02b8ae1 100644 --- a/src/core/services/server/public.providers.test.ts +++ b/src/core/services/server/public.providers.test.ts @@ -77,10 +77,12 @@ describe('fetchPublicProvider', () => { }); it('an upstream throw becomes a 502 without echoing the error text', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); + const log = vi.spyOn(console, 'log').mockImplementation(() => {}); vi.stubGlobal('fetch', vi.fn(async () => { throw new Error('secret-in-url'); })); const { status, body } = await fetchPublicProvider('wikipedia', {}); expect(status).toBe(502); expect(JSON.stringify(body)).not.toContain('secret-in-url'); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"wikipedia","reason":"network","kind":"Error"}'); + expect(JSON.stringify(log.mock.calls)).not.toContain('secret-in-url'); }); }); diff --git a/src/core/services/server/public.providers.ts b/src/core/services/server/public.providers.ts index d6dcc76..8960d9c 100644 --- a/src/core/services/server/public.providers.ts +++ b/src/core/services/server/public.providers.ts @@ -11,6 +11,7 @@ import 'server-only'; import { getApiProvider } from '@/core/schemas/api.schema'; import { readBoundedJson, RequestBodyTooLarge } from './boundedJson'; +import { errorKind, logEvent } from './telemetry'; export const PUBLIC_PROXY_IDS = [ 'usgs', @@ -109,21 +110,24 @@ interface PublicFailure { type Outcome = { status: number; body: unknown }; -function failure(id: string, code: PublicFailureCode, message: string, detail: Record = {}): Outcome { - // Status, code and keys only. The upstream body never reaches the log. - console.warn('[public proxy] upstream failure', { provider: id, code, ...detail }); +/** A failure code, in the words telemetry/events.json allows for `reason`. */ +const REASONS: Record = { + upstream_timeout: 'timeout', + upstream_redirect: 'redirect', + upstream_status: 'status', + upstream_too_large: 'too_large', + upstream_shape: 'shape', + upstream_unreachable: 'network' +}; + +function failure(id: string, code: PublicFailureCode, message: string, detail: { status?: number; kind?: string } = {}): Outcome { + // Provider, reason, status and error class only. Neither the upstream + // body nor where a redirect pointed reaches the log. + logEvent('flyspace.source.failed', { provider: id, reason: REASONS[code], ...detail }); const body: PublicFailure = { error: { code, message: `${id}: ${message}` } }; return { status: 502, body }; } -function redirectHost(res: Response): string { - try { - return new URL(res.headers.get('location') || '').host || '(none)'; - } catch { - return '(unparseable)'; - } -} - export async function fetchPublicProvider( id: string, params: Params, @@ -152,15 +156,14 @@ export async function fetchPublicProvider( } catch (error) { const name = error instanceof Error ? error.name : 'Unknown'; if (name === 'TimeoutError' || name === 'AbortError') { - return failure(id, 'upstream_timeout', `no response within ${PUBLIC_UPSTREAM_DEADLINE_MS / 1000} s`); + return failure(id, 'upstream_timeout', `no response within ${PUBLIC_UPSTREAM_DEADLINE_MS / 1000} s`, { kind: errorKind(error) }); } - return failure(id, 'upstream_unreachable', 'request failed', { name }); + return failure(id, 'upstream_unreachable', 'request failed', { kind: errorKind(error) }); } if (res.type === 'opaqueredirect' || (res.status >= 300 && res.status < 400)) { return failure(id, 'upstream_redirect', 'upstream redirected; redirects are not followed', { - status: res.status, - location: redirectHost(res) + status: res.status }); } if (!res.ok) { @@ -176,9 +179,9 @@ export async function fetchPublicProvider( } const name = error instanceof Error ? error.name : 'Unknown'; if (name === 'TimeoutError' || name === 'AbortError') { - return failure(id, 'upstream_timeout', `body not complete within ${PUBLIC_UPSTREAM_DEADLINE_MS / 1000} s`); + return failure(id, 'upstream_timeout', `body not complete within ${PUBLIC_UPSTREAM_DEADLINE_MS / 1000} s`, { kind: errorKind(error) }); } // SyntaxError (and only its name): the message would quote the body. - return failure(id, 'upstream_shape', 'non-JSON response', { name }); + return failure(id, 'upstream_shape', 'non-JSON response', { kind: errorKind(error) }); } } diff --git a/src/core/services/server/telemetry.test.ts b/src/core/services/server/telemetry.test.ts new file mode 100644 index 0000000..a62bd55 --- /dev/null +++ b/src/core/services/server/telemetry.test.ts @@ -0,0 +1,216 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi, type MockInstance } from 'vitest'; +import allowlist from '../../../../telemetry/events.json'; +import { errorKind, logEvent, type TelemetryEvent, type TelemetryFields } from './telemetry'; +import { PUBLIC_PROXY_IDS } from './public.providers'; + +const MARKER = 'MESSAGE-TEXT-MUST-NOT-BE-LOGGED'; + +let log: MockInstance; +let error: MockInstance; + +beforeEach(() => { + log = vi.spyOn(console, 'log').mockImplementation(() => {}); + error = vi.spyOn(console, 'error').mockImplementation(() => {}); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +/** Everything written to either sink, as one string. */ +function everything(): string { + return JSON.stringify([...log.mock.calls, ...error.mock.calls]); +} + +describe('logEvent', () => { + it('writes one JSON line: the event, then its allowlisted fields in list order', () => { + logEvent('flyspace.source.failed', { status: 503, reason: 'status', provider: 'polymarket' }); + expect(log).toHaveBeenCalledTimes(1); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed","provider":"polymarket","reason":"status","status":503}'); + expect(error).not.toHaveBeenCalled(); + }); + + it('sends error events to console.error and every other event to console.log', () => { + for (const [event, spec] of Object.entries(allowlist.events)) { + log.mockClear(); + error.mockClear(); + logEvent(event as TelemetryEvent); + const line = JSON.stringify({ event }); + if (spec.level === 'error') { + expect(error).toHaveBeenCalledWith(line); + expect(log).not.toHaveBeenCalled(); + } else { + expect(log).toHaveBeenCalledWith(line); + expect(error).not.toHaveBeenCalled(); + } + } + }); + + it('drops a field the event does not list, even one another event allows', () => { + logEvent('flyspace.registry_intent.failed', { + reason: 'timeout', + provider: 'polymarket', + status: 500, + kind: 'Error', + query: 'runs', + message: MARKER, + url: `https://example.com/?q=${MARKER}`, + stack: MARKER + } as TelemetryFields); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.registry_intent.failed","reason":"timeout"}'); + }); + + it('turns a value off the list into "other" where the list has it, and drops it where it does not', () => { + logEvent('flyspace.source.failed', { provider: 'fred', reason: MARKER, status: 99, kind: MARKER }); + logEvent('flyspace.llm.failed', { llm_provider: MARKER, status: 600, kind: 'RangeError' }); + logEvent('flyspace.ledger.query_failed', { query: MARKER }); + expect(log.mock.calls).toEqual([ + ['{"event":"flyspace.source.failed","provider":"other","reason":"other","kind":"other"}'] + ]); + expect(error.mock.calls).toEqual([ + ['{"event":"flyspace.llm.failed","kind":"other"}'], + ['{"event":"flyspace.ledger.query_failed"}'] + ]); + }); + + it('keeps a status only when it is an HTTP status code', () => { + for (const status of [99, 600, 502.5, Number.NaN, Number.POSITIVE_INFINITY, '502']) { + logEvent('flyspace.source.failed', { provider: 'usgs', status: status as number }); + } + logEvent('flyspace.source.failed', { provider: 'usgs', status: 100 }); + logEvent('flyspace.source.failed', { provider: 'usgs', status: 599 }); + expect(log.mock.calls.map(call => JSON.parse(call[0] as string).status)).toEqual([ + undefined, undefined, undefined, undefined, undefined, undefined, 100, 599 + ]); + }); + + it('writes nothing for an event the allowlist does not name', () => { + for (const event of ['flyspace.made.up', 'constructor', '__proto__', 'toString', '']) { + logEvent(event as TelemetryEvent, { reason: 'timeout' }); + } + expect(log).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); + + it('never throws, whatever it is handed or wherever the line goes', () => { + const hostile = new Proxy({}, { get() { throw new Error(MARKER); } }); + const getter = { get provider(): string { throw new Error(MARKER); } }; + expect(() => logEvent('flyspace.source.failed', hostile as TelemetryFields)).not.toThrow(); + expect(() => logEvent('flyspace.source.failed', getter)).not.toThrow(); + expect(() => logEvent('flyspace.source.failed', null as unknown as TelemetryFields)).not.toThrow(); + expect(log).toHaveBeenCalledWith('{"event":"flyspace.source.failed"}'); + + error.mockImplementation(() => { throw new Error('the sink is down'); }); + expect(() => logEvent('flyspace.llm.failed', { kind: 'Error' })).not.toThrow(); + expect(everything()).not.toContain(MARKER); + }); + + it('never writes message text, whichever field it is put in', () => { + const carriers: unknown[] = [ + MARKER, + new Error(MARKER), + { toString: () => MARKER, toJSON: () => MARKER }, + new String(MARKER), + [MARKER] + ]; + for (const event of Object.keys(allowlist.events) as TelemetryEvent[]) { + for (const carrier of carriers) { + const fields = Object.fromEntries(Object.keys(allowlist.fields).map(name => [name, carrier])); + logEvent(event, fields as TelemetryFields); + logEvent(event, { kind: errorKind(carrier) }); + } + } + expect(log.mock.calls.length + error.mock.calls.length).toBeGreaterThan(0); + expect(everything()).not.toContain(MARKER); + }); +}); + +describe('errorKind', () => { + it('names the class and never the message', () => { + expect(errorKind(new Error(`connect ECONNREFUSED postgres://omni:${MARKER}@db:5432`))).toBe('Error'); + expect(errorKind(new TypeError(MARKER))).toBe('TypeError'); + expect(errorKind(new SyntaxError(MARKER))).toBe('SyntaxError'); + expect(errorKind(new DOMException(MARKER, 'TimeoutError'))).toBe('TimeoutError'); + expect(errorKind(new DOMException(MARKER, 'AbortError'))).toBe('AbortError'); + }); + + it('is "other" for anything off the list', () => { + class CustomError extends Error { + name = 'CustomError'; + } + const throwing = { get name(): string { throw new Error(MARKER); } }; + for (const value of [new CustomError(MARKER), new RangeError(MARKER), { name: MARKER }, MARKER, undefined, null, 42, throwing]) { + expect(errorKind(value)).toBe('other'); + } + }); +}); + +describe('telemetry/events.json', () => { + it('is version 1 of the syberlabs_events allowlist', () => { + expect(allowlist.version).toBe(1); + expect(allowlist.dataset).toBe('syberlabs_events'); + }); + + it('gives every event a plain name, an owner, a one-line decision, a level and defined fields', () => { + for (const [event, spec] of Object.entries(allowlist.events)) { + expect(event).toMatch(/^flyspace\.[a-z_]+\.[a-z_]+$/); + expect(spec.owner).toBe('Mateo'); + expect(spec.decision.trim()).not.toBe(''); + expect(spec.decision).not.toMatch(/\n/); + expect(['error', 'warn']).toContain(spec.level); + expect(spec.fields.length).toBe(new Set(spec.fields).size); + for (const field of spec.fields) expect(Object.keys(allowlist.fields)).toContain(field); + } + }); + + it('defines every field as a closed list of strings or an integer range', () => { + for (const spec of Object.values(allowlist.fields) as unknown[]) { + if (Array.isArray(spec)) { + expect(spec.length).toBeGreaterThan(0); + expect(spec.every(value => typeof value === 'string' && value !== '')).toBe(true); + expect(new Set(spec).size).toBe(spec.length); + } else { + const [min, max] = (spec as { integer: number[] }).integer; + expect(Number.isInteger(min) && Number.isInteger(max) && min <= max).toBe(true); + } + } + }); + + it('names every source the server relays', () => { + expect(allowlist.fields.provider).toEqual(expect.arrayContaining(['polymarket', ...PUBLIC_PROXY_IDS])); + }); +}); + +// ============================================ +// Nothing on the server writes to the console except through logEvent, so +// nothing outside the allowlist can be written. +// ============================================ + +const ROOT = path.resolve(__dirname, '../../../..'); +const SERVER_DIRS = ['src/app/api', 'src/core/services/server']; +const WRITER = 'src/core/services/server/telemetry.ts'; +const CONSOLE_CALL = /\bconsole\.(?:log|info|warn|error|debug|trace)\(/; + +function serverFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir)) { + const full = path.join(dir, entry); + if (statSync(full).isDirectory()) out.push(...serverFiles(full)); + else if (/\.(ts|tsx)$/.test(entry) && !/\.test\.(ts|tsx)$/.test(entry)) out.push(full); + } + return out; +} + +describe('server console output', () => { + it('goes through logEvent and nowhere else', () => { + const files = SERVER_DIRS.flatMap(dir => serverFiles(path.join(ROOT, dir))) + .map(file => path.relative(ROOT, file).split(path.sep).join('/')); + expect(files).toContain(WRITER); + const offenders = files + .filter(file => file !== WRITER) + .filter(file => CONSOLE_CALL.test(readFileSync(path.join(ROOT, file), 'utf8'))); + expect(offenders).toEqual([]); + }); +}); diff --git a/src/core/services/server/telemetry.ts b/src/core/services/server/telemetry.ts new file mode 100644 index 0000000..049eabb --- /dev/null +++ b/src/core/services/server/telemetry.ts @@ -0,0 +1,86 @@ +// ============================================ +// SERVER TELEMETRY +// +// One JSON line per server event: {"event":"flyspace..", ...}. +// telemetry/events.json is the allowlist. An event it does not name is not +// written. A field it does not list for that event is never read. A value +// outside the field's list becomes "other" where the list has it and is +// dropped where it does not. +// +// Error detail is the error's class name from a closed list (`kind`), never +// its message, stack, URL or payload: those can quote an upstream body, a +// prompt or a connection string, and a log line is kept somewhere the +// ledger's scrubbing does not reach. +// +// Errors go to console.error and everything else to console.log. Nothing here +// throws: a log line must never cost a request its answer. +// ============================================ + +import 'server-only'; +import allowlist from '../../../../telemetry/events.json'; + +export type TelemetryEvent = keyof typeof allowlist.events; +export type TelemetryField = keyof typeof allowlist.fields; +export type TelemetryFields = Partial>; + +/** A closed list of strings, or an inclusive integer range. */ +type FieldSpec = readonly string[] | { readonly integer: readonly number[] }; + +interface EventSpec { + readonly level: string; + readonly fields: readonly string[]; +} + +const EVENTS: Readonly> = allowlist.events; +const FIELDS: Readonly> = allowlist.fields; +const KINDS: readonly string[] = allowlist.fields.kind; + +/** The value as the allowlist permits it, or undefined to leave the field out. */ +function admit(spec: FieldSpec | undefined, value: unknown): string | number | undefined { + if (spec === undefined || value === undefined || value === null) return undefined; + if ('integer' in spec) { + const [min, max] = spec.integer; + return typeof value === 'number' && Number.isInteger(value) && value >= min && value <= max + ? value + : undefined; + } + if (typeof value === 'string' && spec.includes(value)) return value; + return spec.includes('other') ? 'other' : undefined; +} + +/** + * The error's class name when the allowlist has it, otherwise "other". Only + * `name` is read; the message is where an upstream body or a key would be. + */ +export function errorKind(error: unknown): string { + try { + const name = typeof error === 'object' && error !== null + ? (error as { name?: unknown }).name + : undefined; + return typeof name === 'string' && KINDS.includes(name) ? name : 'other'; + } catch { + return 'other'; + } +} + +/** + * Write one allowlisted event. Only the fields events.json lists for it are + * looked up on `fields`, so anything else a caller passes is never read. + */ +export function logEvent(event: TelemetryEvent, fields?: TelemetryFields): void { + try { + if (!Object.hasOwn(EVENTS, event)) return; + const spec = EVENTS[event]; + const given: Record = fields ?? {}; + const line: Record = { event }; + for (const name of spec.fields) { + const value = admit(FIELDS[name], given[name]); + if (value !== undefined) line[name] = value; + } + const text = JSON.stringify(line); + if (spec.level === 'error') console.error(text); + else console.log(text); + } catch { + // Telemetry is the cheaper loss. + } +} diff --git a/telemetry/events.json b/telemetry/events.json new file mode 100644 index 0000000..2bf71ab --- /dev/null +++ b/telemetry/events.json @@ -0,0 +1,68 @@ +{ + "version": 1, + "dataset": "syberlabs_events", + "fields": { + "provider": ["polymarket", "usgs", "wikipedia", "openlibrary", "github", "crossref", "openmeteo", "frankfurter", "other"], + "llm_provider": ["local", "anthropic", "google"], + "reason": ["timeout", "status", "redirect", "too_large", "shape", "network", "config", "other"], + "status": { "integer": [100, 599] }, + "kind": ["AbortError", "TimeoutError", "TypeError", "SyntaxError", "ProviderResponseError", "Error", "other"], + "query": ["runs", "lineage"] + }, + "events": { + "flyspace.source.failed": { + "level": "warn", + "fields": ["provider", "reason", "status", "kind"], + "owner": "Mateo", + "decision": "Count failures of the keyless sources the server relays, by provider and reason, so a dead source is seen before a canvas block goes empty." + }, + "flyspace.registry_intent.failed": { + "level": "warn", + "fields": ["reason"], + "owner": "Mateo", + "decision": "Record that JEV search routing failed and why; search already falls back to keyword results, so this is a warning and never carries the query." + }, + "flyspace.llm.failed": { + "level": "error", + "fields": ["llm_provider", "status", "kind"], + "owner": "Mateo", + "decision": "Record a failed provider call by provider, HTTP status and error class; the scrubbed error text stays in the ledger row, never in the log." + }, + "flyspace.ledger.query_failed": { + "level": "error", + "fields": ["query", "kind"], + "owner": "Mateo", + "decision": "Record a failed ledger read, runs or lineage, without the pg error, which can carry the connection string." + }, + "flyspace.ledger.open_failed": { + "level": "error", + "fields": ["kind"], + "owner": "Mateo", + "decision": "Record that a run row could not open and the ledger paused for 30 s; only the error class is kept, since its text can carry the connection string." + }, + "flyspace.ledger.close_failed": { + "level": "error", + "fields": ["kind"], + "owner": "Mateo", + "decision": "Record that a run row stayed 'running' because its closing write failed, so the stuck row has a cause in the log." + }, + "flyspace.ledger.reconcile_failed": { + "level": "error", + "fields": ["kind"], + "owner": "Mateo", + "decision": "Record that marking stale hosted runs 'uncertain' failed, so rows left 'running' have a cause in the log." + }, + "flyspace.auth.unavailable": { + "level": "error", + "fields": ["reason"], + "owner": "Mateo", + "decision": "Record that the identity provider timed out or is misconfigured, the one authentication failure that is ours to fix rather than the caller's." + }, + "flyspace.db.idle_client_failed": { + "level": "error", + "fields": ["kind"], + "owner": "Mateo", + "decision": "Record an idle Postgres client error by class only, so a dropped connection is visible without its text and never takes the process down." + } + } +} diff --git a/wrangler.jsonc b/wrangler.jsonc index 6b48f0e..4a51340 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -15,6 +15,12 @@ "OMNI_PUBLIC_DEMO": "1", "NEXT_PUBLIC_OMNI_PUBLIC_DEMO": "1" }, + "observability": { + "enabled": true, + "logs": { + "invocation_logs": false + } + }, "assets": { "directory": "dist/client", "not_found_handling": "none",