From f3ff38d029d6b3abc34a683b2c6216e9a9cb84c0 Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 18:39:18 +0200 Subject: [PATCH 1/4] fix(events): log why uploads are rejected or fail A rejected upload now carries a closed reason and stage in its 400 body and log line, with a privacy-safe batch description: size, event-name counts, the client's appVersion, buildChannel and contractRevision when they match their formats, a user-agent family, and for a property rejection the key. The body is read before the token check so token rejections keep that context. An internal failure logs the ingestion step, error class and name, and a closed storage error kind, never the message. Maintenance, identity-link and deletion failures log the same error diagnostics. Validation strictness is unchanged. Desktop's uploader ignores the body of an unsuccessful response, so the extra 400 fields are compatible. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 + workers/events/src/index.test.ts | 84 ++++++- workers/events/src/index.ts | 370 ++++++++++++++++++++++++++----- 3 files changed, 407 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 605af49..5b53935 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,16 @@ diagnostics and identity observations). Event export runs from the five-minute cron, up to ten batches of 100 per run, not after each upload, so an upload costs one D1 round trip. Each upload logs its status, event count and stage durations (`read_ms`, `rate_limit_ms`, `d1_ms`, `total_ms`) without identifiers. +A rejected upload's response and log carry a closed `reason` and `stage` +(`headers`, `body`, `batch`, `event`, `privacy` or `size`), listed in +`REJECTION_STAGES` in `workers/events/src/index.ts`. The log adds the batch +size, event-name counts, the client's `appVersion`, `buildChannel` and +`contractRevision` when they match their formats, a user-agent family, and +for a property rejection the property's key. It never logs property values, +identifiers, tokens or IP addresses. A failed upload logs `failed_step` +(`body`, `token`, `validation`, `rate-limit` or `storage`), the error's class +and name, and a closed `error_kind` for storage errors, never the error's +message. Desktop's uploader ignores the body of an unsuccessful response. The identity-link token is service-to-service authority. Rotate it if it is exposed, and never embed it in website or desktop bundles: diff --git a/workers/events/src/index.test.ts b/workers/events/src/index.test.ts index bb993c7..c7c7697 100644 --- a/workers/events/src/index.test.ts +++ b/workers/events/src/index.test.ts @@ -278,12 +278,13 @@ describe("event gateway routes", () => { properties: event.properties, })), ); - for (const [key, value] of [ - ["path", "PRIVATE document"], - ["error", "PRIVATE document"], - ["text", "PRIVATE document"], - ["documentId", "/Users/someone/PRIVATE.tex"], - ["revision", "https://example.com/PRIVATE"], + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + for (const [key, value, reason] of [ + ["path", "PRIVATE document", "property-content-key"], + ["error", "PRIVATE document", "property-content-key"], + ["text", "PRIVATE document", "property-content-key"], + ["documentId", "/Users/someone/PRIVATE.tex", "property-private-value"], + ["revision", "https://example.com/PRIVATE", "property-private-value"], ]) { const rejected = await submit([ { @@ -292,6 +293,19 @@ describe("event gateway routes", () => { }, ]); expect(rejected.status).toBe(400); + expect(await rejected.json()).toMatchObject({ reason, stage: "privacy" }); + const entry = String(log.mock.lastCall?.[0]); + expect(JSON.parse(entry)).toMatchObject({ + reason, + stage: "privacy", + rejected_event_index: 0, + rejected_event_name: "feature.viewed", + rejected_property: key, + app_version: "0.6.25", + build_channel: "stable", + contract_revision: "6", + }); + expect(entry).not.toContain("PRIVATE"); } expect(database.sqlite.prepare("SELECT count(*) AS n FROM analytics_events").get()!.n).toBe(3); }); @@ -313,8 +327,9 @@ describe("event gateway routes", () => { expect(database.batch).not.toHaveBeenCalled(); }); - it("requires an installation-owned token", async () => { + it("requires an installation-owned token and logs why without identifiers or values", async () => { const database = createDatabase(); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); const response = await worker.fetch!( incoming( new Request("https://events.scientfactory.com/v1/events", { @@ -328,7 +343,62 @@ describe("event gateway routes", () => { ); expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: "Invalid installation token", + reason: "installation-token-missing", + stage: "headers", + }); expect(database.batch).not.toHaveBeenCalled(); + const entry = String(log.mock.lastCall?.[0]); + expect(JSON.parse(entry)).toMatchObject({ + status: 400, + error_class: "validation", + reason: "installation-token-missing", + stage: "headers", + schema_version: 1, + source: "desktop", + batch_size: 1, + event_names: { "provider.turn.sent": 1 }, + app_version: "0.0.32", + build_channel: "development", + }); + for (const secret of ["installation:", "session:", "8e0ee7d5", "codex", "gpt-5.6"]) { + expect(entry).not.toContain(secret); + } + }); + + it("names the failed storage step without logging the error message", async () => { + const database = createDatabase(); + database.batch.mockRejectedValueOnce(new Error("D1_ERROR: Network connection lost.")); + const failure = vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const response = await worker.fetch!( + incoming( + new Request("https://events.scientfactory.com/v1/events", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Scient-Installation-Token": INSTALLATION_TOKEN, + }, + body: JSON.stringify(validPayload()), + }), + ), + gatewayEnv(database.database), + { waitUntil: vi.fn() } as unknown as ExecutionContext, + ); + + expect(response.status).toBe(500); + expect(await response.json()).toEqual({ + error: "Analytics ingestion failed", + }); + const entry = String(failure.mock.lastCall?.[0]); + expect(JSON.parse(entry)).toEqual({ + message: "Analytics ingestion failed", + failed_step: "storage", + error_class: "internal", + error_name: "Error", + error_kind: "network-lost", + }); }); it("acknowledges deletion when the installation has never uploaded data", async () => { diff --git a/workers/events/src/index.ts b/workers/events/src/index.ts index 926bdd1..7e0d3d3 100644 --- a/workers/events/src/index.ts +++ b/workers/events/src/index.ts @@ -10,8 +10,10 @@ import { } from "../../../functions/_lib/analyticsPeople"; import { ANALYTICS_CONTRACT_REVISION, + EVENT_NAME_PATTERN, eventContractViolation, PRIVACY_LEVELS, + PROPERTY_KEY_PATTERN, type PrivacyLevel, } from "./eventContract"; import { posthogEventUuid, posthogRequest, readBoundedJson, TransportFailure } from "./transport"; @@ -104,9 +106,72 @@ interface PendingDeletionRow { readonly posthog_submitted_at: string | null; } -class RequestValidationError extends Error {} +// Closed rejection reasons. They are logged and returned with every 400 so a +// rejection is diagnosable without property values, identifiers or IP addresses. +const REJECTION_STAGES = { + "installation-token-missing": "headers", + "installation-token-invalid": "headers", + "body-too-large": "size", + "body-invalid-json": "body", + "payload-unsupported": "batch", + "batch-empty": "batch", + "batch-too-large": "size", + "batch-mixed-installations": "batch", + "event-not-object": "event", + "event-id-invalid": "event", + "event-name-invalid": "event", + "privacy-level-invalid": "event", + "consent-level-invalid": "event", + "occurred-at-invalid": "event", + "occurred-at-future": "event", + "occurred-at-too-old": "event", + "distinct-id-invalid": "event", + "installation-identity-required": "event", + "session-id-invalid": "event", + "properties-invalid": "event", + "property-key-invalid": "event", + "property-value-invalid": "event", + "contract-violation": "event", + "consent-below-privacy": "privacy", + "property-content-key": "privacy", + "property-private-value": "privacy", + "properties-too-many": "size", + "properties-too-large": "size", + "request-invalid": "request", +} as const; +type RejectionReason = keyof typeof REJECTION_STAGES; + +class RequestValidationError extends Error { + eventIndex?: number; + constructor( + message: string, + readonly reason: RejectionReason = "request-invalid", + ) { + super(message); + } +} class InstallationAuthenticationError extends Error {} +/** Maps the generated contract guard's messages onto the closed reasons. */ +const CONTRACT_VIOLATION_REASONS: ReadonlyArray = [ + ["Invalid event name", "event-name-invalid"], + ["Invalid privacy level", "privacy-level-invalid"], + ["Invalid consent level", "consent-level-invalid"], + ["Consent level", "consent-below-privacy"], + ["Too many event properties", "properties-too-many"], + ["Invalid event property name", "property-key-invalid"], + ["Invalid event property", "property-value-invalid"], + ["Private event property", "property-content-key"], + ["Private value in event property", "property-private-value"], + ["Event properties are too large", "properties-too-large"], +]; +function contractViolationReason(message: string): RejectionReason { + return ( + CONTRACT_VIOLATION_REASONS.find(([prefix]) => message.startsWith(prefix))?.[1] ?? + "contract-violation" + ); +} + function jsonResponse(body: unknown, status = 200, origin?: string | null): Response { const headers = new Headers({ "Cache-Control": "no-store", @@ -123,38 +188,55 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function requireString(value: unknown, field: string, pattern: RegExp): string { +function requireString( + value: unknown, + field: string, + pattern: RegExp, + reason?: RejectionReason, +): string { if (typeof value !== "string" || !pattern.test(value)) { - throw new RequestValidationError(`Invalid ${field}`); + throw new RequestValidationError(`Invalid ${field}`, reason); } return value; } function parseEvent(value: unknown, identityEnabled = false): AcceptedEvent { - if (!isRecord(value)) throw new RequestValidationError("Each event must be an object"); + if (!isRecord(value)) + throw new RequestValidationError("Each event must be an object", "event-not-object"); const properties = value.properties; - if (!isRecord(properties)) throw new RequestValidationError("Invalid event properties"); + if (!isRecord(properties)) + throw new RequestValidationError("Invalid event properties", "properties-invalid"); if (Object.keys(properties).length > 64) { - throw new RequestValidationError("Too many event properties"); + throw new RequestValidationError("Too many event properties", "properties-too-many"); } if (new TextEncoder().encode(JSON.stringify(properties)).byteLength > MAX_PROPERTIES_BYTES) { - throw new RequestValidationError("Event properties are too large"); + throw new RequestValidationError("Event properties are too large", "properties-too-large"); } - const privacyLevel = requireString(value.privacy_level, "privacy_level", /^[a-z]+$/); + const privacyLevel = requireString( + value.privacy_level, + "privacy_level", + /^[a-z]+$/, + "privacy-level-invalid", + ); if (!PRIVACY_LEVELS.includes(privacyLevel as PrivacyLevel)) { - throw new RequestValidationError("Invalid privacy_level"); + throw new RequestValidationError("Invalid privacy_level", "privacy-level-invalid"); } - const occurredAtInput = requireString(value.occurred_at, "occurred_at", /^.{10,40}$/); + const occurredAtInput = requireString( + value.occurred_at, + "occurred_at", + /^.{10,40}$/, + "occurred-at-invalid", + ); const occurredAtDate = new Date(occurredAtInput); if (Number.isNaN(occurredAtDate.valueOf())) { - throw new RequestValidationError("Invalid occurred_at"); + throw new RequestValidationError("Invalid occurred_at", "occurred-at-invalid"); } const now = Date.now(); if (occurredAtDate.valueOf() > now + 24 * 60 * 60 * 1000) { - throw new RequestValidationError("occurred_at is too far in the future"); + throw new RequestValidationError("occurred_at is too far in the future", "occurred-at-future"); } const earliestAllowed = identityEnabled ? diagnosticObservation(String(value.name), privacyLevel, properties) @@ -164,30 +246,54 @@ function parseEvent(value: unknown, identityEnabled = false): AcceptedEvent { (privacyLevel === "diagnostic" ? DIAGNOSTIC_EVENT_RETENTION_DAYS : RAW_EVENT_RETENTION_DAYS) * 86400000; if (occurredAtDate.valueOf() < earliestAllowed) { - throw new RequestValidationError("occurred_at is too old"); + throw new RequestValidationError("occurred_at is too old", "occurred-at-too-old"); } - const distinctId = requireString(value.distinct_id, "distinct_id", IDENTIFIER_PATTERN); + const distinctId = requireString( + value.distinct_id, + "distinct_id", + IDENTIFIER_PATTERN, + "distinct-id-invalid", + ); if (!INSTALLATION_ID_PATTERN.test(distinctId)) { - throw new RequestValidationError("Desktop events require an installation identity"); + throw new RequestValidationError( + "Desktop events require an installation identity", + "installation-identity-required", + ); } - const sessionId = requireString(value.session_id, "session_id", SESSION_ID_PATTERN); - const consentLevel = requireString(value.consent_level, "consent_level", /^[a-z]+$/); + const sessionId = requireString( + value.session_id, + "session_id", + SESSION_ID_PATTERN, + "session-id-invalid", + ); + const consentLevel = requireString( + value.consent_level, + "consent_level", + /^[a-z]+$/, + "consent-level-invalid", + ); if (!PRIVACY_LEVELS.includes(consentLevel as PrivacyLevel)) { - throw new RequestValidationError("Invalid consent_level"); + throw new RequestValidationError("Invalid consent_level", "consent-level-invalid"); } - const name = requireString(value.name, "event name", /^[a-z][a-z0-9_.-]{0,79}$/); + const name = requireString( + value.name, + "event name", + /^[a-z][a-z0-9_.-]{0,79}$/, + "event-name-invalid", + ); const contractViolation = eventContractViolation({ name, privacyLevel: privacyLevel as PrivacyLevel, consentLevel: consentLevel as PrivacyLevel, properties, }); - if (contractViolation) throw new RequestValidationError(contractViolation); + if (contractViolation) + throw new RequestValidationError(contractViolation, contractViolationReason(contractViolation)); return { - id: requireString(value.id, "event id", IDENTIFIER_PATTERN), + id: requireString(value.id, "event id", IDENTIFIER_PATTERN, "event-id-invalid"), name, distinctId, identityType: "desktop_installation", @@ -204,27 +310,45 @@ export function validateIngestionPayload( identityEnabled = false, ): ReadonlyArray { if (!isRecord(value) || value.schema_version !== 1 || value.source !== "desktop") { - throw new RequestValidationError("Unsupported event payload"); + throw new RequestValidationError("Unsupported event payload", "payload-unsupported"); } if (!Array.isArray(value.events) || value.events.length < 1) { - throw new RequestValidationError("At least one event is required"); + throw new RequestValidationError("At least one event is required", "batch-empty"); } if (value.events.length > MAX_BATCH_SIZE) { - throw new RequestValidationError(`At most ${MAX_BATCH_SIZE} events are accepted`); + throw new RequestValidationError( + `At most ${MAX_BATCH_SIZE} events are accepted`, + "batch-too-large", + ); } - const events = value.events.map((event) => parseEvent(event, identityEnabled)); + const events = value.events.map((event, index) => { + try { + return parseEvent(event, identityEnabled); + } catch (error) { + if (error instanceof RequestValidationError) error.eventIndex = index; + throw error; + } + }); const installationId = events[0]?.distinctId; if (events.some((event) => event.distinctId !== installationId)) { - throw new RequestValidationError("A batch must contain one installation identity"); + throw new RequestValidationError( + "A batch must contain one installation identity", + "batch-mixed-installations", + ); } return events; } function requireInstallationToken(request: Request): string { + const token = request.headers.get(INSTALLATION_TOKEN_HEADER); + if (token === null) { + throw new RequestValidationError("Invalid installation token", "installation-token-missing"); + } return requireString( - request.headers.get(INSTALLATION_TOKEN_HEADER), + token, "installation token", INSTALLATION_TOKEN_PATTERN, + "installation-token-invalid", ); } @@ -237,14 +361,121 @@ async function readJsonBody(request: Request): Promise { try { return await readBoundedJson(request, MAX_REQUEST_BYTES); } catch (error) { - throw new RequestValidationError( - error instanceof TransportFailure && error.kind === "body-too-large" - ? "Request body is too large" - : "Request body must be valid JSON", - ); + throw error instanceof TransportFailure && error.kind === "body-too-large" + ? new RequestValidationError("Request body is too large", "body-too-large") + : new RequestValidationError("Request body must be valid JSON", "body-invalid-json"); } } +const APP_VERSION_PATTERN = + /^(?:\d{1,4}\.\d{1,4}\.\d{1,4}(?:-[a-z]{1,16}(?:\.\d{1,14}){0,3})?|unknown)$/; +const BUILD_CHANNEL_PATTERN = /^(?:stable|beta|nightly|development|unknown)$/; +const CONTRACT_REVISION_PATTERN = /^\d{1,4}$/; +const MAX_LOGGED_EVENT_NAMES = 20; + +/** + * A privacy-safe description of a submitted batch for rejection and failure logs: + * its size, event-name counts and the client's release fields when they match + * their closed formats. Never property values, identifiers, tokens or IP addresses. + */ +function batchDiagnostics(value: unknown, request: Request): Record { + const userAgent = request.headers.get("User-Agent") ?? ""; + const diagnostics: Record = { + user_agent_family: !userAgent + ? "none" + : /^node\b/i.test(userAgent) + ? "node" + : /\bElectron\//.test(userAgent) + ? "electron" + : /^Mozilla\//.test(userAgent) + ? "browser" + : /^curl\//i.test(userAgent) + ? "curl" + : "other", + }; + if (!isRecord(value)) return diagnostics; + if (typeof value.schema_version === "number") diagnostics.schema_version = value.schema_version; + if (typeof value.source === "string" && /^[a-z]{1,16}$/.test(value.source)) + diagnostics.source = value.source; + if (!Array.isArray(value.events)) return diagnostics; + diagnostics.batch_size = value.events.length; + const names: Record = {}; + for (const event of value.events.slice(0, MAX_BATCH_SIZE)) { + const name = isRecord(event) && typeof event.name === "string" ? event.name : ""; + const key = !EVENT_NAME_PATTERN.test(name) + ? "(invalid)" + : name in names || Object.keys(names).length < MAX_LOGGED_EVENT_NAMES + ? name + : "(other)"; + names[key] = (names[key] ?? 0) + 1; + const properties = isRecord(event) && isRecord(event.properties) ? event.properties : {}; + for (const [field, logged, pattern] of [ + ["appVersion", "app_version", APP_VERSION_PATTERN], + ["buildChannel", "build_channel", BUILD_CHANNEL_PATTERN], + ["contractRevision", "contract_revision", CONTRACT_REVISION_PATTERN], + ] as const) { + const fieldValue = properties[field]; + if (!(logged in diagnostics) && typeof fieldValue === "string" && pattern.test(fieldValue)) + diagnostics[logged] = fieldValue; + } + } + diagnostics.event_names = names; + return diagnostics; +} + +/** The offending event's index and name and, for a property rejection, its key, never its value. */ +function rejectionDiagnostics( + error: RequestValidationError, + body: unknown, +): Record { + const diagnostics: Record = {}; + if (error.eventIndex === undefined) return diagnostics; + diagnostics.rejected_event_index = error.eventIndex; + const event = isRecord(body) && Array.isArray(body.events) ? body.events[error.eventIndex] : null; + if (isRecord(event) && typeof event.name === "string" && EVENT_NAME_PATTERN.test(event.name)) + diagnostics.rejected_event_name = event.name; + const property = /'([^']*)'$/.exec(error.message)?.[1]; + if ( + property && + ["property-value-invalid", "property-content-key", "property-private-value"].includes( + error.reason, + ) && + PROPERTY_KEY_PATTERN.test(property) + ) + diagnostics.rejected_property = property; + return diagnostics; +} + +/** Error class, runtime name and a closed storage kind; never the message, which may carry SQL or data. */ +function errorDiagnostics(error: unknown): { + readonly error_class: string; + readonly error_name: string; + readonly error_kind?: string; +} { + const name = error instanceof Error ? error.name : typeof error; + const message = error instanceof Error ? error.message : ""; + const kind = /network connection lost/i.test(message) + ? "network-lost" + : /timed? ?out/i.test(message) + ? "timeout" + : /overloaded/i.test(message) + ? "overloaded" + : /SQLITE_BUSY|database is locked/i.test(message) + ? "busy" + : /constraint/i.test(message) + ? "constraint" + : /no such (?:table|column)/i.test(message) + ? "schema" + : /^D1_/.test(message) + ? "d1-other" + : null; + return { + error_class: error instanceof TransportFailure ? error.kind : "internal", + error_name: /^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(name) ? name : "unknown", + ...(kind ? { error_kind: kind } : {}), + }; +} + async function persistEvents( database: D1Database, events: ReadonlyArray, @@ -987,7 +1218,7 @@ async function handleIdentityLink( console.error( JSON.stringify({ message: "Identity linking failed", - error_class: error instanceof TransportFailure ? error.kind : "internal", + ...errorDiagnostics(error), }), ); return jsonResponse({ error: "Identity linking failed" }, 500); @@ -1136,7 +1367,12 @@ async function handleInstallationDeletion(request: Request, env: AnalyticsEnv): if (error instanceof RequestValidationError) { return jsonResponse({ error: error.message }, 400); } - console.error(JSON.stringify({ message: "Analytics deletion request failed" })); + console.error( + JSON.stringify({ + message: "Analytics deletion request failed", + ...errorDiagnostics(error), + }), + ); return jsonResponse({ error: "Analytics deletion request failed" }, 500); } } @@ -1164,13 +1400,21 @@ async function handleIngestion(request: Request, env: AnalyticsEnv): Promise { + let body: unknown; + // The ingestion step in progress, so an internal failure names where it happened. + let step: "body" | "token" | "validation" | "rate-limit" | "storage" = "body"; + const respond = ( + response: Response, + errorClass?: string, + diagnostics: Record = {}, + ) => { console.log( JSON.stringify({ message: "Analytics ingestion", status: response.status, events: eventCount, ...(errorClass ? { error_class: errorClass } : {}), + ...diagnostics, ...Object.fromEntries(Object.entries(stages).map(([name, ms]) => [`${name}_ms`, ms])), total_ms: Date.now() - started, }), @@ -1179,28 +1423,34 @@ async function handleIngestion(request: Request, env: AnalyticsEnv): Promise Date: Mon, 16 Nov 2026 18:54:49 +0200 Subject: [PATCH 2/4] test(events): match the desktop contract corpus Desktop's turn and delivery fields (retryAttempts, failure details, runtime-change details, oldestEventAgeBucket) were missing from the gateway's copy of the contract-v11 corpus. Regenerated with generateConformance.ts; --check passes against desktop. Co-Authored-By: Claude Opus 5.5 --- workers/events/fixtures/contract-v11.json | 184 ++++++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/workers/events/fixtures/contract-v11.json b/workers/events/fixtures/contract-v11.json index 3b0f9b9..d676a78 100644 --- a/workers/events/fixtures/contract-v11.json +++ b/workers/events/fixtures/contract-v11.json @@ -95,6 +95,7 @@ "queuedCountBucket": "unknown", "droppedCountBucket": "unknown", "retryCountBucket": "unknown", + "oldestEventAgeBucket": "unknown", "deliveryClass": "unknown", "appVersion": "0.6.8", "buildChannel": "stable", @@ -110,6 +111,7 @@ "queuedCountBucket": "4-10", "droppedCountBucket": "2-3", "retryCountBucket": "1", + "oldestEventAgeBucket": "unknown", "deliveryClass": "network", "appVersion": "0.6.8", "buildChannel": "stable", @@ -361,6 +363,10 @@ "crossProvider": true, "completionWake": "settled_only", "durationSeconds": 90, + "failureClass": "permission", + "failureCode": "retry_stalled", + "failurePhase": "provider", + "httpStatus": 400, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -566,6 +572,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -594,6 +601,9 @@ "properties": { "provider": "antigravity", "modelKey": "gpt-5.6-sol", + "retryAttempts": 2, + "failureCode": "retry_stalled", + "httpStatus": 400, "failureClass": "permission", "durationBucket": "5-15s", "appVersion": "0.6.8", @@ -861,6 +871,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -878,6 +889,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -895,6 +911,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -912,6 +929,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -929,6 +951,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -946,6 +969,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -963,6 +991,7 @@ "stage": "unknown", "failureClass": "unknown", "durationBucket": "unknown", + "trigger": "user", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -980,6 +1009,38 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.runtime.first_pick:fallbacks", + "name": "provider.runtime.first_pick", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "other", + "state": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.runtime.first_pick:representative", + "name": "provider.runtime.first_pick", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "provider": "antigravity", + "state": "ready", + "percentBucket": "50-74", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -1924,6 +1985,66 @@ "contractRevision": "11" } }, + { + "case": "provider.turn.failed:failure-cause-hostile", + "name": "provider.turn.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "scient", + "modelKey": "gpt-5.6-sol", + "failureClass": "provider_error", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "mcp.delegated_task.finished:delegated-failure-hostile", + "name": "mcp.delegated_task.finished", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "status": "failed", + "callerProvider": "claudeAgent", + "targetProvider": "scient", + "failurePhase": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "provider.turn.failed:failure-code-unknown-identifier", + "name": "provider.turn.failed", + "privacyLevel": "essential", + "consentLevel": "essential", + "properties": { + "provider": "scient", + "modelKey": "gpt-5.6-sol", + "failureClass": "provider_error", + "durationBucket": "unknown", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "mcp.delegated_task.finished:delegated-failure-unknown-identifier", + "name": "mcp.delegated_task.finished", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "status": "failed", + "callerProvider": "claudeAgent", + "targetProvider": "codex", + "failurePhase": "provider", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, { "case": "provider.commands.observed:command-hostile", "name": "provider.commands.observed", @@ -2243,6 +2364,45 @@ "contractRevision": "11" } }, + { + "case": "panel.viewed:file-csv", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "file-preview", + "fileType": "csv", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "panel.viewed:file-none", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "file-preview", + "fileType": "none", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, + { + "case": "panel.viewed:file-private", + "name": "panel.viewed", + "privacyLevel": "product", + "consentLevel": "product", + "properties": { + "category": "file-preview", + "fileType": "other", + "appVersion": "0.6.8", + "buildChannel": "stable", + "contractRevision": "11" + } + }, { "case": "provider.turn.sent:scient", "name": "provider.turn.sent", @@ -2272,6 +2432,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2341,6 +2502,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2375,6 +2541,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2444,6 +2611,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2478,6 +2650,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2547,6 +2720,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2581,6 +2759,7 @@ "durationBucket": "5-15s", "usedTools": true, "hasAttachment": true, + "retryAttempts": 2, "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" @@ -2650,6 +2829,11 @@ "stage": "downloading", "failureClass": "permission", "durationBucket": "5-15s", + "trigger": "agent", + "attempt": 2, + "downloadSource": "fallback", + "compressed": true, + "resumedBucket": "10-50mb", "appVersion": "0.6.8", "buildChannel": "stable", "contractRevision": "11" From 79080b79bb03f5f290075a61b7bdd152f06bb77c Mon Sep 17 00:00:00 2001 From: Yaacov Date: Mon, 16 Nov 2026 19:05:41 +0200 Subject: [PATCH 3/4] fix(events): never log an event name that could carry an address or id Review (Codex): a tokenless request could get any name of a valid shape, such as one with an IP address, into the rejection log. Only names of lowercase words joined by dots, as Scient's code writes them, are logged; anything else is counted as "(other)". Co-Authored-By: Claude Opus 5.5 --- workers/events/src/index.test.ts | 14 ++++++++++---- workers/events/src/index.ts | 9 ++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/workers/events/src/index.test.ts b/workers/events/src/index.test.ts index c7c7697..5356e69 100644 --- a/workers/events/src/index.test.ts +++ b/workers/events/src/index.test.ts @@ -330,12 +330,18 @@ describe("event gateway routes", () => { it("requires an installation-owned token and logs why without identifiers or values", async () => { const database = createDatabase(); const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const payload = validPayload(); + // A name of a valid shape that carries an address is counted, never logged. + const named = { + ...payload, + events: [...payload.events, { ...payload.events[0], name: "patient.192.0.2.10" }], + }; const response = await worker.fetch!( incoming( new Request("https://events.scientfactory.com/v1/events", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify(validPayload()), + body: JSON.stringify(named), }), ), gatewayEnv(database.database), @@ -357,12 +363,12 @@ describe("event gateway routes", () => { stage: "headers", schema_version: 1, source: "desktop", - batch_size: 1, - event_names: { "provider.turn.sent": 1 }, + batch_size: 2, + event_names: { "provider.turn.sent": 1, "(other)": 1 }, app_version: "0.0.32", build_channel: "development", }); - for (const secret of ["installation:", "session:", "8e0ee7d5", "codex", "gpt-5.6"]) { + for (const secret of ["installation:", "session:", "8e0ee7d5", "codex", "gpt-5.6", "192.0.2"]) { expect(entry).not.toContain(secret); } }); diff --git a/workers/events/src/index.ts b/workers/events/src/index.ts index 7e0d3d3..5f7a4d4 100644 --- a/workers/events/src/index.ts +++ b/workers/events/src/index.ts @@ -372,6 +372,12 @@ const APP_VERSION_PATTERN = const BUILD_CHANNEL_PATTERN = /^(?:stable|beta|nightly|development|unknown)$/; const CONTRACT_REVISION_PATTERN = /^\d{1,4}$/; const MAX_LOGGED_EVENT_NAMES = 20; +/** + * Event names come from Scient's code: lowercase words joined by dots. An unverified client can + * send any name the shape allows, so a name with digits or other characters (an address, an id) + * is counted as "(other)" and never logged. + */ +const LOGGED_EVENT_NAME_PATTERN = /^[a-z]+(?:[._][a-z]+){1,5}$/; /** * A privacy-safe description of a submitted batch for rejection and failure logs: @@ -404,7 +410,8 @@ function batchDiagnostics(value: unknown, request: Request): Record Date: Mon, 16 Nov 2026 19:10:35 +0200 Subject: [PATCH 4/4] fix(events): log no client-chosen names when a batch is rejected Re-check (Codex): a name pattern cannot tell an event name from one that carries a person's name (patient.john.smith). Nothing is authenticated when a batch is rejected, so its log keeps only counts, the rejected event's position, the client's release fields and the reason; no event names or property keys. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +++--- workers/events/src/index.test.ts | 16 +++++++---- workers/events/src/index.ts | 46 +++++--------------------------- 3 files changed, 22 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 5b53935..11d8ed5 100644 --- a/README.md +++ b/README.md @@ -219,10 +219,10 @@ durations (`read_ms`, `rate_limit_ms`, `d1_ms`, `total_ms`) without identifiers. A rejected upload's response and log carry a closed `reason` and `stage` (`headers`, `body`, `batch`, `event`, `privacy` or `size`), listed in `REJECTION_STAGES` in `workers/events/src/index.ts`. The log adds the batch -size, event-name counts, the client's `appVersion`, `buildChannel` and -`contractRevision` when they match their formats, a user-agent family, and -for a property rejection the property's key. It never logs property values, -identifiers, tokens or IP addresses. A failed upload logs `failed_step` +size, the rejected event's position, the client's `appVersion`, `buildChannel` +and `contractRevision` when they match their formats, and a user-agent family. +Nothing is authenticated when a batch is rejected, so it never logs event names, +property keys or values, identifiers, tokens or IP addresses. A failed upload logs `failed_step` (`body`, `token`, `validation`, `rate-limit` or `storage`), the error's class and name, and a closed `error_kind` for storage errors, never the error's message. Desktop's uploader ignores the body of an unsuccessful response. diff --git a/workers/events/src/index.test.ts b/workers/events/src/index.test.ts index 5356e69..1277491 100644 --- a/workers/events/src/index.test.ts +++ b/workers/events/src/index.test.ts @@ -299,13 +299,12 @@ describe("event gateway routes", () => { reason, stage: "privacy", rejected_event_index: 0, - rejected_event_name: "feature.viewed", - rejected_property: key, app_version: "0.6.25", build_channel: "stable", contract_revision: "6", }); expect(entry).not.toContain("PRIVATE"); + expect(entry).not.toContain("feature.viewed"); } expect(database.sqlite.prepare("SELECT count(*) AS n FROM analytics_events").get()!.n).toBe(3); }); @@ -331,7 +330,7 @@ describe("event gateway routes", () => { const database = createDatabase(); const log = vi.spyOn(console, "log").mockImplementation(() => undefined); const payload = validPayload(); - // A name of a valid shape that carries an address is counted, never logged. + // Nothing is authenticated yet, so no event name is logged, even one of a valid shape. const named = { ...payload, events: [...payload.events, { ...payload.events[0], name: "patient.192.0.2.10" }], @@ -364,11 +363,18 @@ describe("event gateway routes", () => { schema_version: 1, source: "desktop", batch_size: 2, - event_names: { "provider.turn.sent": 1, "(other)": 1 }, app_version: "0.0.32", build_channel: "development", }); - for (const secret of ["installation:", "session:", "8e0ee7d5", "codex", "gpt-5.6", "192.0.2"]) { + for (const secret of [ + "installation:", + "session:", + "8e0ee7d5", + "codex", + "gpt-5.6", + "192.0.2", + "provider.turn.sent", + ]) { expect(entry).not.toContain(secret); } }); diff --git a/workers/events/src/index.ts b/workers/events/src/index.ts index 5f7a4d4..2916b5d 100644 --- a/workers/events/src/index.ts +++ b/workers/events/src/index.ts @@ -10,10 +10,8 @@ import { } from "../../../functions/_lib/analyticsPeople"; import { ANALYTICS_CONTRACT_REVISION, - EVENT_NAME_PATTERN, eventContractViolation, PRIVACY_LEVELS, - PROPERTY_KEY_PATTERN, type PrivacyLevel, } from "./eventContract"; import { posthogEventUuid, posthogRequest, readBoundedJson, TransportFailure } from "./transport"; @@ -371,18 +369,11 @@ const APP_VERSION_PATTERN = /^(?:\d{1,4}\.\d{1,4}\.\d{1,4}(?:-[a-z]{1,16}(?:\.\d{1,14}){0,3})?|unknown)$/; const BUILD_CHANNEL_PATTERN = /^(?:stable|beta|nightly|development|unknown)$/; const CONTRACT_REVISION_PATTERN = /^\d{1,4}$/; -const MAX_LOGGED_EVENT_NAMES = 20; -/** - * Event names come from Scient's code: lowercase words joined by dots. An unverified client can - * send any name the shape allows, so a name with digits or other characters (an address, an id) - * is counted as "(other)" and never logged. - */ -const LOGGED_EVENT_NAME_PATTERN = /^[a-z]+(?:[._][a-z]+){1,5}$/; /** * A privacy-safe description of a submitted batch for rejection and failure logs: - * its size, event-name counts and the client's release fields when they match - * their closed formats. Never property values, identifiers, tokens or IP addresses. + * its size and the client's release fields when they match their closed formats. + * Never event names, property keys or values, identifiers, tokens or IP addresses. */ function batchDiagnostics(value: unknown, request: Request): Record { const userAgent = request.headers.get("User-Agent") ?? ""; @@ -405,16 +396,9 @@ function batchDiagnostics(value: unknown, request: Request): Record = {}; + // Nothing is authenticated when a batch is rejected, so no name or key the client chose is + // logged: an event name of a valid shape can still carry a name or an address. for (const event of value.events.slice(0, MAX_BATCH_SIZE)) { - const name = isRecord(event) && typeof event.name === "string" ? event.name : ""; - const key = !EVENT_NAME_PATTERN.test(name) - ? "(invalid)" - : LOGGED_EVENT_NAME_PATTERN.test(name) && - (name in names || Object.keys(names).length < MAX_LOGGED_EVENT_NAMES) - ? name - : "(other)"; - names[key] = (names[key] ?? 0) + 1; const properties = isRecord(event) && isRecord(event.properties) ? event.properties : {}; for (const [field, logged, pattern] of [ ["appVersion", "app_version", APP_VERSION_PATTERN], @@ -426,30 +410,14 @@ function batchDiagnostics(value: unknown, request: Request): Record { +/** The offending event's position in the batch, never its name or contents. */ +function rejectionDiagnostics(error: RequestValidationError): Record { const diagnostics: Record = {}; if (error.eventIndex === undefined) return diagnostics; diagnostics.rejected_event_index = error.eventIndex; - const event = isRecord(body) && Array.isArray(body.events) ? body.events[error.eventIndex] : null; - if (isRecord(event) && typeof event.name === "string" && EVENT_NAME_PATTERN.test(event.name)) - diagnostics.rejected_event_name = event.name; - const property = /'([^']*)'$/.exec(error.message)?.[1]; - if ( - property && - ["property-value-invalid", "property-content-key", "property-private-value"].includes( - error.reason, - ) && - PROPERTY_KEY_PATTERN.test(property) - ) - diagnostics.rejected_property = property; return diagnostics; } @@ -1482,7 +1450,7 @@ async function handleIngestion(request: Request, env: AnalyticsEnv): Promise