diff --git a/.env.example b/.env.example index 0c84547..94cbfc9 100644 --- a/.env.example +++ b/.env.example @@ -21,3 +21,18 @@ DATABASE_STARTUP_ATTEMPTS=5 DATABASE_STARTUP_RETRY_MS=1000 WORKER_DRAIN_TIMEOUT_MS=30000 WORKER_HEALTH_HOST=127.0.0.1 + +# Blizzard achievement-fingerprint sweeps (worker service only). The +# credentials and the per-sweep request cap are required; the remaining +# settings fall back to the shared budget, threshold, and cadence decisions. +BLIZZARD_CLIENT_ID=replace-with-the-battle-net-client-id +BLIZZARD_CLIENT_SECRET=replace-with-the-battle-net-client-secret +BLIZZARD_SWEEP_REQUEST_CAP=300 +BLIZZARD_HOURLY_REQUEST_BUDGET=28800 +FINGERPRINT_MINIMUM_COMMON=200 +FINGERPRINT_MINIMUM_IDENTICAL_PERCENT=20 +FINGERPRINT_SWEEP_CADENCE_HOURS=168 + +# Optional internal alert sink for budget and admission pressure. Its path and +# query string are part of the secret, so the whole URL is used as given. +# MAINTAINER_ALERT_WEBHOOK_URL=https://hooks.example.test/services/T000/B000/token diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..00833ed --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,25 @@ +# SlashWho + +SlashWho publishes World of Warcraft character-relationship information derived from public upstream data. + +## Language + +**Privacy-hidden ownership**: +The Raider.IO state indicating that a character's ownership link is intentionally not public. It is SlashWho's sole privacy signal for inferred relationships. +_Avoid_: Hidden alt, upstream opt-out + +**Fingerprint-derived link**: +A relationship between characters inferred from Blizzard achievement-completion data, rather than declared by Raider.IO. +_Avoid_: Verified link, confirmed alt + +**Alt list**: +The public list of characters linked to a root character. It intentionally does not distinguish Raider.IO-declared relationships from fingerprint-derived links. +_Avoid_: Verified-alt list, inferred-alt list + +**Partial snapshot**: +An immutable historical result known not to contain every discoverable relationship. It is public as partial while its limitation reason remains internal. +_Avoid_: Failed snapshot, incomplete refresh + +**Ephemeral fingerprint**: +Achievement-completion data held only while a single discovery sweep is running. It is discarded before snapshot publication and never becomes a stored signature. +_Avoid_: Fingerprint cache, stored signature diff --git a/apps/web/src/app/api/v1/api-contract.test.ts b/apps/web/src/app/api/v1/api-contract.test.ts index a8dbef0..988fc0e 100644 --- a/apps/web/src/app/api/v1/api-contract.test.ts +++ b/apps/web/src/app/api/v1/api-contract.test.ts @@ -129,7 +129,12 @@ const searches: SearchService = { return snapshotResult; }, async cleanupExpired() { - return { rateLimits: 0, negativeCache: 0, suppressions: 0 }; + return { + rateLimits: 0, + negativeCache: 0, + suppressions: 0, + fingerprintRequests: 0 + }; } }; diff --git a/apps/web/src/app/privacy/page.test.tsx b/apps/web/src/app/privacy/page.test.tsx new file mode 100644 index 0000000..ba4cc2f --- /dev/null +++ b/apps/web/src/app/privacy/page.test.tsx @@ -0,0 +1,21 @@ +// @vitest-environment jsdom + +import "@testing-library/jest-dom/vitest"; +import { render, screen } from "@testing-library/react"; +import { expect, it } from "vitest"; + +import PrivacyPage from "./page"; + +it("states the fingerprint privacy boundary without publishing a discovery method", () => { + // Break caught: public documentation could promise privacy while leaving it + // unclear that privacy-hidden ownership is excluded from inferred links. + render(); + + expect( + screen.getByText(/privacy-hidden Raider\.IO ownership is excluded/i) + ).toBeInTheDocument(); + expect( + screen.getByText(/public alt lists do not disclose the discovery method/i) + ).toBeInTheDocument(); + expect(screen.queryByText(/opt-out/i)).not.toBeInTheDocument(); +}); diff --git a/apps/web/src/app/privacy/page.tsx b/apps/web/src/app/privacy/page.tsx index 31bfc41..2fb6658 100644 --- a/apps/web/src/app/privacy/page.tsx +++ b/apps/web/src/app/privacy/page.tsx @@ -26,6 +26,13 @@ export default function PrivacyPage() { responses, and internal validation guesses are never stored or shown.

+

Fingerprint-derived links

+

+ Privacy-hidden Raider.IO ownership is excluded from fingerprint-derived + links. Public alt lists do not disclose the discovery method for any + character relationship. +

+

Removal requests

Removal requests are manually verified. Submit a request using the{" "} diff --git a/apps/web/src/server/container.test.ts b/apps/web/src/server/container.test.ts index b4a65a3..c30f0a5 100644 --- a/apps/web/src/server/container.test.ts +++ b/apps/web/src/server/container.test.ts @@ -21,7 +21,11 @@ it("migrates and initializes the durable queue before serving searches", async ( async enqueue() { return "54f14e37-7df7-43db-91d5-21e797d1d145"; }, + async enqueueFingerprintAdmission() { + return "54f14e37-7df7-43db-91d5-21e797d1d145"; + }, async work() {}, + async workFingerprintAdmissions() {}, async scheduleMaintenanceCleanup() {}, async stop() {}, isReady() { diff --git a/apps/worker/package.json b/apps/worker/package.json index b219461..3b5401e 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -9,6 +9,7 @@ }, "dependencies": { "@slashwho/application": "workspace:*", + "@slashwho/blizzard": "workspace:*", "@slashwho/database": "workspace:*", "@slashwho/domain": "workspace:*", "@slashwho/raiderio": "workspace:*", diff --git a/apps/worker/src/config.test.ts b/apps/worker/src/config.test.ts index 8cb6449..a753005 100644 --- a/apps/worker/src/config.test.ts +++ b/apps/worker/src/config.test.ts @@ -2,18 +2,88 @@ import { expect, it } from "vitest"; import { loadWorkerConfig } from "./config"; +const environment = { + DATABASE_URL: "postgresql://slashwho:test@db/slashwho", + BLIZZARD_CLIENT_ID: "worker-client-id", + BLIZZARD_CLIENT_SECRET: "worker-client-secret", + BLIZZARD_SWEEP_REQUEST_CAP: "300" +}; + +it("rejects missing Blizzard credentials and invalid sweep bounds", () => { + // Break caught: the worker could start a sweep without its private Blizzard + // credentials or reserve an impossible number of upstream requests. + expect(() => + loadWorkerConfig({ DATABASE_URL: environment.DATABASE_URL }) + ).toThrow("blizzard_client_id_required"); + expect(() => + loadWorkerConfig({ ...environment, BLIZZARD_SWEEP_REQUEST_CAP: "0" }) + ).toThrow("invalid_blizzard_sweep_request_cap"); + expect(() => + loadWorkerConfig({ + ...environment, + FINGERPRINT_MINIMUM_IDENTICAL_PERCENT: "101" + }) + ).toThrow("invalid_fingerprint_minimum_identical_percent"); + expect(() => + loadWorkerConfig({ + ...environment, + BLIZZARD_SWEEP_REQUEST_CAP: "301", + BLIZZARD_HOURLY_REQUEST_BUDGET: "300" + }) + ).toThrow("invalid_blizzard_sweep_request_cap"); +}); + +it("loads private Blizzard sweep defaults only for the worker", () => { + // Break caught: an omitted operational limit could silently become unbounded + // or make the planned seven-day sweep cadence depend on another process. + expect(loadWorkerConfig(environment)).toMatchObject({ + blizzardClientId: environment.BLIZZARD_CLIENT_ID, + blizzardClientSecret: environment.BLIZZARD_CLIENT_SECRET, + blizzardSweepRequestCap: 300, + blizzardHourlyRequestBudget: 28_800, + fingerprintMinimumCommon: 200, + fingerprintMinimumIdenticalPercent: 20, + fingerprintSweepCadenceHours: 168 + }); +}); + +it("accepts a local Blizzard endpoint only when explicitly configured", () => { + // Break caught: e2e could not direct its fake credentials and sweep requests + // to its deterministic local fixture. + expect( + loadWorkerConfig({ + ...environment, + BLIZZARD_BASE_URL: "http://127.0.0.1:43101" + }).blizzardBaseUrl + ).toBe("http://127.0.0.1:43101"); +}); + +it("preserves a maintainer webhook path and query string", () => { + // Break caught: URL validation could reduce a provider webhook to its origin, + // posting alerts to the provider homepage instead of the secret endpoint. + const webhookUrl = + "https://hooks.example.test/services/T000/B000/token?wait=true"; + + expect( + loadWorkerConfig({ + ...environment, + MAINTAINER_ALERT_WEBHOOK_URL: webhookUrl + }).maintainerAlertWebhookUrl + ).toBe(webhookUrl); +}); + it("accepts only explicit loopback or container health hosts", () => { // Break caught: a deploy could silently bind to an unusable or arbitrary // interface instead of the intended local/container health boundary. expect( loadWorkerConfig({ - DATABASE_URL: "postgresql://slashwho:test@db/slashwho", + ...environment, WORKER_HEALTH_HOST: "0.0.0.0" }).healthHost ).toBe("0.0.0.0"); expect(() => loadWorkerConfig({ - DATABASE_URL: "postgresql://slashwho:test@db/slashwho", + ...environment, WORKER_HEALTH_HOST: "public.example" }) ).toThrow("invalid_worker_health_host"); diff --git a/apps/worker/src/config.ts b/apps/worker/src/config.ts index 56606a1..3fdfe72 100644 --- a/apps/worker/src/config.ts +++ b/apps/worker/src/config.ts @@ -9,6 +9,15 @@ export type WorkerConfig = { negativeCacheTtlMs: number; raiderIoBaseUrl: string; raiderIoTimeoutMs: number; + blizzardClientId: string; + blizzardClientSecret: string; + blizzardBaseUrl?: string; + blizzardSweepRequestCap: number; + blizzardHourlyRequestBudget: number; + fingerprintMinimumCommon: number; + fingerprintMinimumIdenticalPercent: number; + fingerprintSweepCadenceHours: number; + maintainerAlertWebhookUrl?: string; }; function positiveInteger( @@ -21,6 +30,41 @@ function positiveInteger( return parsed; } +function integerInRange( + value: string | undefined, + fallback: number, + minimum: number, + maximum: number, + code: string +): number { + const parsed = value === undefined ? fallback : Number(value); + if (!Number.isInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new Error(code); + } + return parsed; +} + +function requiredString(value: string | undefined, code: string): string { + if (!value?.trim()) throw new Error(code); + return value; +} + +function optionalHttpUrl( + value: string | undefined, + code: string +): string | undefined { + if (value === undefined) return undefined; + try { + const normalized = value.trim(); + const url = new URL(normalized); + if (url.protocol !== "http:" && url.protocol !== "https:") + throw new Error(); + return normalized; + } catch { + throw new Error(code); + } +} + export function loadWorkerConfig( environment: NodeJS.ProcessEnv = process.env ): WorkerConfig { @@ -29,6 +73,27 @@ export function loadWorkerConfig( if (healthHost !== "127.0.0.1" && healthHost !== "0.0.0.0") { throw new Error("invalid_worker_health_host"); } + const blizzardClientId = requiredString( + environment.BLIZZARD_CLIENT_ID, + "blizzard_client_id_required" + ); + const blizzardClientSecret = requiredString( + environment.BLIZZARD_CLIENT_SECRET, + "blizzard_client_secret_required" + ); + const blizzardSweepRequestCap = positiveInteger( + environment.BLIZZARD_SWEEP_REQUEST_CAP, + 0, + "invalid_blizzard_sweep_request_cap" + ); + const blizzardHourlyRequestBudget = positiveInteger( + environment.BLIZZARD_HOURLY_REQUEST_BUDGET, + 28_800, + "invalid_blizzard_hourly_request_budget" + ); + if (blizzardSweepRequestCap > blizzardHourlyRequestBudget) { + throw new Error("invalid_blizzard_sweep_request_cap"); + } return { databaseUrl: environment.DATABASE_URL, @@ -65,6 +130,35 @@ export function loadWorkerConfig( environment.RAIDER_IO_TIMEOUT_MS, 10_000, "invalid_raiderio_timeout" + ), + blizzardClientId, + blizzardClientSecret, + blizzardBaseUrl: optionalHttpUrl( + environment.BLIZZARD_BASE_URL, + "invalid_blizzard_base_url" + ), + blizzardSweepRequestCap, + blizzardHourlyRequestBudget, + fingerprintMinimumCommon: positiveInteger( + environment.FINGERPRINT_MINIMUM_COMMON, + 200, + "invalid_fingerprint_minimum_common" + ), + fingerprintMinimumIdenticalPercent: integerInRange( + environment.FINGERPRINT_MINIMUM_IDENTICAL_PERCENT, + 20, + 1, + 100, + "invalid_fingerprint_minimum_identical_percent" + ), + fingerprintSweepCadenceHours: positiveInteger( + environment.FINGERPRINT_SWEEP_CADENCE_HOURS, + 168, + "invalid_fingerprint_sweep_cadence_hours" + ), + maintainerAlertWebhookUrl: optionalHttpUrl( + environment.MAINTAINER_ALERT_WEBHOOK_URL, + "invalid_maintainer_alert_webhook_url" ) }; } diff --git a/apps/worker/src/logger.test.ts b/apps/worker/src/logger.test.ts index 3075a91..083dc02 100644 --- a/apps/worker/src/logger.test.ts +++ b/apps/worker/src/logger.test.ts @@ -67,4 +67,42 @@ describe("worker logger", () => { expect(captured).toContain("[Circular]"); expect(captured).not.toContain(marker); }); + + it("redacts every ephemeral fingerprint and credential marker", async () => { + // Break caught: diagnostic objects could serialize achievement material, + // access tokens, or comparison scores outside the handler allowlist. + const marker = "UNIQUE_FINGERPRINT_MARKER_414f8b"; + const output = new PassThrough(); + let captured = ""; + output.on("data", (chunk) => { + captured += chunk.toString(); + }); + const logger = createWorkerLogger(output); + + logger.info( + { + achievementId: marker, + achievementIds: marker, + achievementTimestamp: marker, + completionTimestamp: marker, + accessToken: marker, + refreshToken: marker, + fingerprint: marker, + fingerprintScore: marker, + matchScore: marker, + identicalPercent: marker, + nested: { + achievements: marker, + timestamps: marker, + token: marker, + score: marker + } + }, + "fingerprint_event" + ); + await new Promise((resolve) => setImmediate(resolve)); + + expect(captured).toContain("fingerprint_event"); + expect(captured).not.toContain(marker); + }); }); diff --git a/apps/worker/src/logger.ts b/apps/worker/src/logger.ts index 3f4a96e..2ecf187 100644 --- a/apps/worker/src/logger.ts +++ b/apps/worker/src/logger.ts @@ -34,7 +34,21 @@ const sensitiveKeys = new Set([ "validationname", "rawurl", "rawpayload", - "rawupstreampayload" + "rawupstreampayload", + "achievementid", + "achievementids", + "achievements", + "achievementtimestamp", + "completiontimestamp", + "timestamps", + "accesstoken", + "refreshtoken", + "token", + "fingerprint", + "fingerprintscore", + "matchscore", + "identicalpercent", + "score" ]); function sanitize(value: unknown, visited = new WeakSet()): unknown { diff --git a/apps/worker/src/main.test.ts b/apps/worker/src/main.test.ts index 15e8397..8e7ec73 100644 --- a/apps/worker/src/main.test.ts +++ b/apps/worker/src/main.test.ts @@ -15,7 +15,14 @@ const config: WorkerConfig = { discoveryRequestCap: 12, negativeCacheTtlMs: 300_000, raiderIoBaseUrl: "https://raider.io", - raiderIoTimeoutMs: 1_000 + raiderIoTimeoutMs: 1_000, + blizzardClientId: "worker-client-id", + blizzardClientSecret: "worker-client-secret", + blizzardSweepRequestCap: 300, + blizzardHourlyRequestBudget: 28_800, + fingerprintMinimumCommon: 200, + fingerprintMinimumIdenticalPercent: 20, + fingerprintSweepCadenceHours: 168 }; describe("worker main", () => { diff --git a/apps/worker/src/runtime.test.ts b/apps/worker/src/runtime.test.ts index 73f833d..7c930a1 100644 --- a/apps/worker/src/runtime.test.ts +++ b/apps/worker/src/runtime.test.ts @@ -13,7 +13,11 @@ import type { RaiderIoGateway } from "@slashwho/domain"; import { describe, expect, it, vi } from "vitest"; import type { WorkerConfig } from "./config"; -import { createWorkerRuntime } from "./runtime"; +import { + createFingerprintAlertNotifier, + createFingerprintIntegration, + createWorkerRuntime +} from "./runtime"; const config: WorkerConfig = { databaseUrl: "postgres://worker:secret@database/slashwho", @@ -25,7 +29,14 @@ const config: WorkerConfig = { discoveryRequestCap: 12, negativeCacheTtlMs: 300_000, raiderIoBaseUrl: "https://raider.io", - raiderIoTimeoutMs: 5_000 + raiderIoTimeoutMs: 5_000, + blizzardClientId: "worker-client-id", + blizzardClientSecret: "worker-client-secret", + blizzardSweepRequestCap: 300, + blizzardHourlyRequestBudget: 28_800, + fingerprintMinimumCommon: 200, + fingerprintMinimumIdenticalPercent: 20, + fingerprintSweepCadenceHours: 168 }; function runtimeFakes() { @@ -39,9 +50,15 @@ function runtimeFakes() { ) => Promise) | undefined; let maintenanceHandler: (() => Promise) | undefined; + let admissionHandler: ((runId: string) => Promise) | undefined; const pendingDispatches: DiscoverCharacterJob[] = []; const recoveredDispatches: string[] = []; const enqueued: DiscoverCharacterJob[] = []; + const fingerprintAdmissions: string[] = []; + const waitingFingerprintRuns: string[] = []; + const admittedFingerprintRuns = new Set(); + const admittedUndispatchedFingerprintRuns: string[] = []; + const dispatchedFingerprintRuns: string[] = []; const queue: DiscoveryQueue = { async start() { queueReady = true; @@ -50,12 +67,19 @@ function runtimeFakes() { enqueued.push(payload); return payload.runId; }, + async enqueueFingerprintAdmission(runId) { + fingerprintAdmissions.push(runId); + return runId; + }, async work(handler) { workHandler = handler; }, async scheduleMaintenanceCleanup(handler) { maintenanceHandler = handler; }, + async workFingerprintAdmissions(handler) { + admissionHandler = handler; + }, async stop() { queueReady = false; }, @@ -78,7 +102,8 @@ function runtimeFakes() { const cleanup = { rateLimits: vi.fn(async () => 2), negativeCache: vi.fn(async () => 3), - suppressions: vi.fn(async () => 4) + suppressions: vi.fn(async () => 4), + fingerprintRequests: vi.fn(async () => 5) }; const repositories = { searchReservations: { @@ -91,7 +116,26 @@ function runtimeFakes() { }, rateLimits: { cleanupExpired: cleanup.rateLimits }, negativeCache: { cleanupExpired: cleanup.negativeCache }, - suppressions: { cleanupExpired: cleanup.suppressions } + suppressions: { cleanupExpired: cleanup.suppressions }, + fingerprintSweeps: { + async admitWaiting(runId: string) { + return admittedFingerprintRuns.has(runId) + ? { kind: "admitted" as const } + : { kind: "waiting" as const, retryAt: new Date() }; + }, + async listWaiting(limit: number, offset = 0) { + return waitingFingerprintRuns.slice(offset, offset + limit); + }, + async listAdmittedUndispatched() { + return [...admittedUndispatchedFingerprintRuns]; + }, + async markDispatched(runId: string) { + dispatchedFingerprintRuns.push(runId); + const index = admittedUndispatchedFingerprintRuns.indexOf(runId); + if (index >= 0) admittedUndispatchedFingerprintRuns.splice(index, 1); + }, + cleanupExpired: cleanup.fingerprintRequests + } } as unknown as Repositories; const sleeps: number[] = []; @@ -113,9 +157,15 @@ function runtimeFakes() { handler, migrations, cleanup, + repositories, pendingDispatches, recoveredDispatches, enqueued, + fingerprintAdmissions, + waitingFingerprintRuns, + admittedFingerprintRuns, + admittedUndispatchedFingerprintRuns, + dispatchedFingerprintRuns, queue, get connectionAttempts() { return connectionAttempts; @@ -129,11 +179,103 @@ function runtimeFakes() { get maintenanceHandler() { return maintenanceHandler; }, + get admissionHandler() { + return admissionHandler; + }, sleeps }; } describe("worker runtime", () => { + it("composes the worker-only Blizzard gateway and fingerprint limits", () => { + // Break caught: worker configuration could be loaded but never reach the + // fingerprint handler, leaving the private sweep feature dormant. + const integration = createFingerprintIntegration(config); + + expect(integration.blizzardGateway).toMatchObject({ + getGuildRoster: expect.any(Function), + getAchievementFingerprint: expect.any(Function) + }); + expect(integration.fingerprint).toEqual({ + requestCap: 300, + hourlyBudget: 28_800, + cadenceMs: 604_800_000, + minimumCommon: 200, + minimumIdenticalPercent: 20 + }); + }); + + it("swallows and logs a non-successful maintainer webhook response", async () => { + // Break caught: a provider outage could reject discovery work and cause the + // durable job to retry after its sweep had already changed state. + const logger = { info: vi.fn() }; + const fetch = vi.fn(async () => new Response(null, { status: 503 })); + const notifier = createFingerprintAlertNotifier( + { + ...config, + maintainerAlertWebhookUrl: + "https://hooks.example.test/services/T000/B000/token?wait=true" + }, + { logger, fetch } + ); + + await expect( + notifier.notify({ + event: "fingerprint_reservation_pressure", + details: { committedRequests: 95, hourlyBudget: 100 } + }) + ).resolves.toBeUndefined(); + expect(logger.info).toHaveBeenCalledWith({ + event: "maintainer_alert_delivery_failed", + alertEvent: "fingerprint_reservation_pressure", + failure: "http_status", + status: 503 + }); + }); + + it("times out and swallows a stalled maintainer webhook request", async () => { + // Break caught: an unresponsive webhook could strand a sweep indefinitely + // even though alert delivery is only an operational side effect. + const logger = { info: vi.fn() }; + let requestSignal: AbortSignal | null | undefined; + const fetch = vi.fn( + async ( + _input: string | URL | Request, + init?: RequestInit + ): Promise => { + requestSignal = init?.signal; + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => reject(init.signal?.reason), + { once: true } + ); + }); + } + ); + const notifier = createFingerprintAlertNotifier( + { + ...config, + maintainerAlertWebhookUrl: + "https://hooks.example.test/services/T000/B000/token?wait=true" + }, + { logger, fetch, timeoutMs: 5 } + ); + + await expect( + notifier.notify({ + event: "fingerprint_admission_blocked", + details: { blockedForMs: 900_000 } + }) + ).resolves.toBeUndefined(); + expect(requestSignal?.aborted).toBe(true); + expect(logger.info).toHaveBeenCalledWith({ + event: "maintainer_alert_delivery_failed", + alertEvent: "fingerprint_admission_blocked", + failure: "network_or_timeout" + }); + }); + it("retries database startup before becoming ready and registering work", async () => { // Break caught: an independently-started worker could exit before PostgreSQL is ready. const fakes = runtimeFakes(); @@ -172,6 +314,46 @@ describe("worker runtime", () => { await runtime.stop(); }); + it("passes an injected fingerprint integration to the discovery handler", async () => { + // Break caught: Task 6 composition could construct Blizzard dependencies + // that the runtime silently drops before handler orchestration. + const fakes = runtimeFakes(); + const blizzardGateway = {} as NonNullable< + DiscoveryJobHandlerOptions["blizzardGateway"] + >; + let handlerOptions: DiscoveryJobHandlerOptions | undefined; + Object.assign(fakes.dependencies, { + createFingerprintIntegration: () => ({ + blizzardGateway, + fingerprint: { + requestCap: 300, + hourlyBudget: 28_800, + cadenceMs: 604_800_000, + minimumCommon: 200, + minimumIdenticalPercent: 20 + } + }), + createHandler(options: DiscoveryJobHandlerOptions) { + handlerOptions = options; + return fakes.handler; + } + }); + + const runtime = await createWorkerRuntime(config, fakes.dependencies); + + expect(handlerOptions).toMatchObject({ + blizzardGateway, + fingerprint: { + requestCap: 300, + hourlyBudget: 28_800, + cadenceMs: 604_800_000, + minimumCommon: 200, + minimumIdenticalPercent: 20 + } + }); + await runtime.stop(); + }); + it("routes only run ids to the handler", async () => { // Break caught: private character lookup values could be forwarded into logs or handlers. const fakes = runtimeFakes(); @@ -207,6 +389,7 @@ describe("worker runtime", () => { expect(fakes.cleanup.rateLimits).toHaveBeenCalledOnce(); expect(fakes.cleanup.negativeCache).toHaveBeenCalledOnce(); expect(fakes.cleanup.suppressions).toHaveBeenCalledOnce(); + expect(fakes.cleanup.fingerprintRequests).toHaveBeenCalledOnce(); await runtime.stop(); }); @@ -227,6 +410,116 @@ describe("worker runtime", () => { await runtime.stop(); }); + it("registers the private admission worker and re-enqueues only admitted discovery runs", async () => { + // Break caught: waiting fingerprint sweeps could consume discovery delivery attempts before budget admission. + const fakes = runtimeFakes(); + const waitingRunId = "00000000-0000-4000-8000-000000000012"; + const key = { region: "eu" as const, realm: "silvermoon", name: "waiting" }; + fakes.waitingFingerprintRuns.push(waitingRunId); + fakes.admittedFingerprintRuns.add(waitingRunId); + const existingRun = { + id: waitingRunId, + rootKey: key, + rootCharacterId: null, + queueJobId: null, + status: "queued" as const, + callerClass: "anonymous" as const, + attempt: 0, + nextRetryAt: null, + errorCode: null, + createdAt: new Date(), + startedAt: null, + completedAt: null, + snapshotId: null + }; + fakes.repositories.runs = { + async find(runId: string) { + return runId === waitingRunId ? existingRun : null; + } + } as Repositories["runs"]; + + const runtime = await createWorkerRuntime(config, fakes.dependencies); + + expect(fakes.fingerprintAdmissions).toEqual([waitingRunId]); + expect(fakes.admissionHandler).toBeTypeOf("function"); + await fakes.admissionHandler?.(waitingRunId); + expect(fakes.enqueued).toEqual([{ runId: waitingRunId, key }]); + expect(fakes.handler.execute).not.toHaveBeenCalled(); + await runtime.stop(); + }); + + it("keeps a budget-blocked fingerprint run out of discovery work", async () => { + // Break caught: a waiting admission could be redispatched into a discovery worker before capacity exists. + const fakes = runtimeFakes(); + const waitingRunId = "00000000-0000-4000-8000-000000000013"; + fakes.waitingFingerprintRuns.push(waitingRunId); + + const runtime = await createWorkerRuntime(config, fakes.dependencies); + + await expect(fakes.admissionHandler?.(waitingRunId)).rejects.toMatchObject({ + retryable: true + }); + expect(fakes.enqueued).toEqual([]); + expect(fakes.handler.execute).not.toHaveBeenCalled(); + await runtime.stop(); + }); + + it("recovers an admitted fingerprint run that was not durably dispatched", async () => { + // Break caught: a process failure between admission and enqueue could strand a reserved sweep forever. + const fakes = runtimeFakes(); + const runId = "00000000-0000-4000-8000-000000000014"; + const key = { + region: "eu" as const, + realm: "silvermoon", + name: "admitted" + }; + fakes.admittedUndispatchedFingerprintRuns.push(runId); + fakes.repositories.runs = { + async find(id: string) { + return id === runId + ? { + id: runId, + rootKey: key, + rootCharacterId: null, + queueJobId: null, + status: "queued" as const, + callerClass: "anonymous" as const, + attempt: 0, + nextRetryAt: null, + errorCode: null, + createdAt: new Date(), + startedAt: null, + completedAt: null, + snapshotId: null + } + : null; + } + } as Repositories["runs"]; + + const runtime = await createWorkerRuntime(config, fakes.dependencies); + + expect(fakes.enqueued).toEqual([{ runId, key }]); + expect(fakes.dispatchedFingerprintRuns).toEqual([runId]); + await runtime.stop(); + }); + + it("recovers every waiting fingerprint admission before readiness", async () => { + // Break caught: a fixed recovery batch could strand the 101st durable admission after a restart. + const fakes = runtimeFakes(); + fakes.waitingFingerprintRuns.push( + ...Array.from( + { length: 101 }, + (_unused, index) => + `00000000-0000-4000-8000-${String(index + 100).padStart(12, "0")}` + ) + ); + + const runtime = await createWorkerRuntime(config, fakes.dependencies); + + expect(fakes.fingerprintAdmissions).toEqual(fakes.waitingFingerprintRuns); + await runtime.stop(); + }); + it("drops readiness before gracefully draining and closing PostgreSQL", async () => { // Break caught: shutdown could close storage under an in-flight job. const fakes = runtimeFakes(); diff --git a/apps/worker/src/runtime.ts b/apps/worker/src/runtime.ts index d9eafe5..ec55945 100644 --- a/apps/worker/src/runtime.ts +++ b/apps/worker/src/runtime.ts @@ -4,8 +4,10 @@ import { recoverPendingSearches, type DiscoveryJobHandler, type DiscoveryJobHandlerOptions, - type DiscoveryLogger + type DiscoveryLogger, + type FingerprintAlertNotifier } from "@slashwho/application"; +import { createBlizzardClient } from "@slashwho/blizzard"; import { createDiscoveryQueue, createPostgresRepositories, @@ -32,6 +34,13 @@ export type WorkerRuntimeDependencies = { createRepositories: (pool: RuntimePool) => Repositories; createQueue: (connectionString: string) => DiscoveryQueue; createGateway: (config: WorkerConfig) => RaiderIoGateway; + createFingerprintIntegration?: ( + config: WorkerConfig + ) => Pick; + createFingerprintAlertNotifier?: ( + config: WorkerConfig, + logger?: DiscoveryLogger + ) => FingerprintAlertNotifier; createHandler: (options: DiscoveryJobHandlerOptions) => DiscoveryJobHandler; sleep: (milliseconds: number) => Promise; }; @@ -41,6 +50,65 @@ export type WorkerRuntime = { stop(): Promise; }; +export function createFingerprintIntegration( + config: WorkerConfig +): Pick { + return { + blizzardGateway: createBlizzardClient({ + fetch: globalThis.fetch, + clientId: config.blizzardClientId, + clientSecret: config.blizzardClientSecret, + baseUrl: config.blizzardBaseUrl + }), + fingerprint: { + requestCap: config.blizzardSweepRequestCap, + hourlyBudget: config.blizzardHourlyRequestBudget, + cadenceMs: config.fingerprintSweepCadenceHours * 60 * 60 * 1_000, + minimumCommon: config.fingerprintMinimumCommon, + minimumIdenticalPercent: config.fingerprintMinimumIdenticalPercent + } + }; +} + +export function createFingerprintAlertNotifier( + config: WorkerConfig, + options: { + fetch?: typeof globalThis.fetch; + logger?: DiscoveryLogger; + timeoutMs?: number; + } = {} +): FingerprintAlertNotifier { + const fetch = options.fetch ?? globalThis.fetch; + const timeoutMs = options.timeoutMs ?? 5_000; + return { + async notify(alert) { + if (!config.maintainerAlertWebhookUrl) return; + try { + const response = await fetch(config.maintainerAlertWebhookUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(alert), + signal: AbortSignal.timeout(timeoutMs) + }); + if (!response.ok) { + options.logger?.info({ + event: "maintainer_alert_delivery_failed", + alertEvent: alert.event, + failure: "http_status", + status: response.status + }); + } + } catch { + options.logger?.info({ + event: "maintainer_alert_delivery_failed", + alertEvent: alert.event, + failure: "network_or_timeout" + }); + } + } + }; +} + const defaultDependencies: WorkerRuntimeDependencies = { createPool: (connectionString) => new Pool({ connectionString }), runMigrations: (pool) => runMigrations(pool as Pool), @@ -52,11 +120,24 @@ const defaultDependencies: WorkerRuntimeDependencies = { baseUrl: config.raiderIoBaseUrl, timeoutMs: config.raiderIoTimeoutMs }), + createFingerprintIntegration, + createFingerprintAlertNotifier: (config, logger) => + createFingerprintAlertNotifier(config, { logger }), createHandler: createDiscoveryJobHandler, sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)) }; +function fingerprintAdmissionRetry(retryAt: Date): Error & { + retryable: true; + retryAfterMs: number; +} { + return Object.assign(new Error("fingerprint_admission_waiting"), { + retryable: true as const, + retryAfterMs: Math.max(1_000, retryAt.getTime() - Date.now()) + }); +} + export async function createWorkerRuntime( config: WorkerConfig, dependencies: WorkerRuntimeDependencies = defaultDependencies, @@ -85,15 +166,66 @@ export async function createWorkerRuntime( const initializedQueue = dependencies.createQueue(config.databaseUrl); queue = initializedQueue; const gateway = dependencies.createGateway(config); + const fingerprintIntegration = + dependencies.createFingerprintIntegration?.(config); + const fingerprintAlertNotifier = + dependencies.createFingerprintAlertNotifier?.(config, logger); const handler = dependencies.createHandler({ repositories, gateway, + ...fingerprintIntegration, + ...(fingerprintAlertNotifier ? { fingerprintAlertNotifier } : {}), + enqueueFingerprintAdmission: (runId) => + initializedQueue.enqueueFingerprintAdmission(runId), requestCap: config.discoveryRequestCap, negativeCacheTtlMs: config.negativeCacheTtlMs, ...(logger ? { logger } : {}) }); await initializedQueue.start(); await recoverPendingSearches(repositories, initializedQueue); + const dispatchAdmittedFingerprintRun = async (runId: string) => { + const run = await repositories.runs.find(runId); + if (!run) return; + await initializedQueue.enqueue({ runId, key: run.rootKey }); + await repositories.fingerprintSweeps.markDispatched(runId, new Date()); + }; + for (let offset = 0; ;) { + const waitingFingerprintRuns = + await repositories.fingerprintSweeps.listWaiting(100, offset); + for (const runId of waitingFingerprintRuns) { + await initializedQueue.enqueueFingerprintAdmission(runId); + } + if (waitingFingerprintRuns.length < 100) break; + offset += waitingFingerprintRuns.length; + } + for (;;) { + const admittedFingerprintRuns = + await repositories.fingerprintSweeps.listAdmittedUndispatched(100); + if (admittedFingerprintRuns.length === 0) break; + for (const runId of admittedFingerprintRuns) { + await dispatchAdmittedFingerprintRun(runId); + } + } + await initializedQueue.workFingerprintAdmissions(async (runId) => { + const admission = await repositories.fingerprintSweeps.admitWaiting( + runId, + new Date() + ); + if (admission.kind === "waiting") { + const blockedForMs = admission.blockedSince + ? Math.max(0, Date.now() - admission.blockedSince.getTime()) + : 0; + if (blockedForMs >= 15 * 60_000) { + logger?.info({ + event: "fingerprint_admission_blocked", + blockedForMs + }); + } + throw fingerprintAdmissionRetry(admission.retryAt); + } + if (admission.kind !== "admitted") return; + await dispatchAdmittedFingerprintRun(runId); + }); await initializedQueue.scheduleMaintenanceCleanup(async () => { await cleanupExpired(repositories); await recoverPendingSearches(repositories, initializedQueue); diff --git a/docs/deployment/railway.md b/docs/deployment/railway.md index 60c1fef..297d6e2 100644 --- a/docs/deployment/railway.md +++ b/docs/deployment/railway.md @@ -41,7 +41,13 @@ PUBLIC_READS_PER_MINUTE=300 FRESHNESS_HOURS=24 ``` -Worker variables. `DISCOVERY_REQUEST_CAP` and `NEGATIVE_CACHE_TTL_MS` are read only by the worker, so set them on the worker service alone; `NEGATIVE_CACHE_TTL_MS` defaults to 300000 milliseconds (5 minutes) when unset: +Worker variables. `DISCOVERY_REQUEST_CAP`, `NEGATIVE_CACHE_TTL_MS`, and the +Blizzard fingerprint settings are read only by the worker, so set them on the +worker service alone. `BLIZZARD_CLIENT_ID` and `BLIZZARD_CLIENT_SECRET` must +be Railway secret variables. `NEGATIVE_CACHE_TTL_MS` defaults to 300000 +milliseconds (5 minutes) when unset; the fingerprint budget defaults shown +below are the application defaults and can be omitted after the required +credentials and sweep cap are configured: ```text DATABASE_URL=${{Postgres.DATABASE_URL}} @@ -53,8 +59,23 @@ DATABASE_STARTUP_ATTEMPTS=5 DATABASE_STARTUP_RETRY_MS=1000 WORKER_DRAIN_TIMEOUT_MS=30000 WORKER_HEALTH_HOST=0.0.0.0 +BLIZZARD_CLIENT_ID= +BLIZZARD_CLIENT_SECRET= +BLIZZARD_SWEEP_REQUEST_CAP=300 +BLIZZARD_HOURLY_REQUEST_BUDGET=28800 +FINGERPRINT_MINIMUM_COMMON=200 +FINGERPRINT_MINIMUM_IDENTICAL_PERCENT=20 +FINGERPRINT_SWEEP_CADENCE_HOURS=168 ``` +`MAINTAINER_ALERT_WEBHOOK_URL` is optional and worker-only. Set it as a secret +variable to receive the internal budget and admission-pressure alerts; leave it +unset to keep those alerts in the logs alone. Its path and query string carry +the shared secret for most providers, so configure the complete URL — it is +used exactly as given. Delivery is best effort: a rejected or unresponsive +webhook is logged as `maintainer_alert_delivery_failed` and never fails the +sweep that raised it. + Railway currently documents `X-Real-IP` as the single remote-client header supplied by its public proxy. SlashWho intentionally accepts only that header for anonymous rate-limit identity and fails closed when it is absent or invalid; it does not trust an arbitrary forwarded chain or a runtime-selectable header name. Verify this exact contract against Railway's public-networking documentation before first launch and after any proxy change. ## Health, readiness, and restarts diff --git a/docs/superpowers/plans/2026-08-10-achievement-fingerprint-discovery-implementation.md b/docs/superpowers/plans/2026-08-10-achievement-fingerprint-discovery-implementation.md new file mode 100644 index 0000000..940d5a2 --- /dev/null +++ b/docs/superpowers/plans/2026-08-10-achievement-fingerprint-discovery-implementation.md @@ -0,0 +1,651 @@ +# Achievement-Fingerprint Discovery Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add automatic, privacy-preserving Blizzard achievement-fingerprint discovery to SlashWho's existing durable search workflow. + +**Architecture:** A `discover-character` run still owns all public state and is the only snapshot writer. A private FIFO admission queue coordinates eligible runs against a PostgreSQL-backed Blizzard request-budget ledger, then re-dispatches the existing discovery run to perform one in-memory Blizzard guild-roster sweep and atomically merge matches into its normal snapshot. + +**Tech Stack:** TypeScript, pnpm workspaces, Zod, Vitest, PostgreSQL, Drizzle migrations, pg-boss, Node `fetch`, Railway worker configuration. + +## Global Constraints + +- Use `BLIZZARD_CLIENT_ID` and `BLIZZARD_CLIENT_SECRET` only in the private worker service; never serialize or log them. +- Persist no achievement IDs, completion timestamps, fingerprint signatures, match scores, raw Blizzard bodies, access tokens, or candidate cursor/list. +- Raider.IO privacy-hidden ownership is the sole privacy signal. It excludes fingerprint-derived linkage; there is no SlashWho opt-out. +- Match only within one region; do not support CN fingerprints or cross-region comparisons. +- Accept only at least 200 common achievements with at least 20% identical completion timestamps. +- The initial shared limit is 28,800 Blizzard requests per rolling hour. Reserve a sweep's full configured cap before it begins; wait FIFO rather than reject. +- A root may publish at most one fingerprint sweep every seven days. Only successful snapshot publication advances that time. +- A cap-bounded sweep publishes a partial snapshot with internal `fingerprint_sweep_capped`; transport failures, 429s, 5xxs, schema failures, aborts, and shutdowns publish nothing and retain the previous snapshot. +- Public API payloads, pages, and snapshot history remain provenance-free; fingerprint and budget details are internal only. +- Follow `docs/contributing.md`: short-lived `feat/` branches, conventional commits, PR to `main`, squash merge. + +--- + +## File structure + +| Path | Responsibility | +| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `packages/blizzard/src/client.ts` | OAuth token acquisition, regional Profile API requests, response validation, safe failure conversion. | +| `packages/blizzard/src/fingerprint.ts` | Ephemeral achievement extraction and pure threshold comparison. | +| `packages/blizzard/src/types.ts` | Blizzard gateway and roster/fingerprint value types; no persistent representations. | +| `packages/domain/src/fingerprint-discovery.ts` | Cap-aware root-guild sweep over a `BlizzardGateway`, producing candidate character observations and partial/failure outcomes. | +| `packages/database/src/schema.ts` and `drizzle/0002_fingerprint_sweeps.sql` | Internal source enum extension, per-root sweep state, FIFO admission rows, and rolling reservation ledger. | +| `packages/database/drizzle/0004_simple_venom.sql` | Individual timestamped fingerprint request events for rolling-hour admission accounting. | +| `packages/database/src/repositories.ts` / `postgres-repositories.ts` | Transactional sweep eligibility, FIFO admission, budget reservation/use/release, and snapshot completion bookkeeping. | +| `packages/database/src/queue.ts` | Private `fingerprint-admission` pg-boss queue and dispatch contract. | +| `packages/application/src/discovery-job-handler.ts` | Coordinates Raider.IO discovery, deferred admission, fingerprint sweep, merged atomic snapshot, and safe retry/abort behaviour. | +| `apps/worker/src/config.ts` / `runtime.ts` | Validated Blizzard and sweep settings; creates the Blizzard client and registers admission workers/maintenance. | +| Existing unit, integration, and runtime tests | Demonstrate privacy, budget, snapshot, retry, and public-contract invariants. | + +### Dependency seam + +`@slashwho/blizzard` depends on `@slashwho/domain` for the existing canonical character key. To avoid a reverse workspace dependency, the domain module owns the small `FingerprintGateway` interface it needs. The application layer supplies an adapter around `BlizzardGateway`; domain tests use a fake. The domain module never imports `@slashwho/blizzard`. + +## Task 1: Create the Blizzard boundary and pure matcher + +**Files:** + +- Create: `packages/blizzard/src/types.ts` +- Create: `packages/blizzard/src/fingerprint.ts` +- Create: `packages/blizzard/src/fingerprint.test.ts` +- Create: `packages/blizzard/src/client.ts` +- Create: `packages/blizzard/src/client.test.ts` +- Create: `packages/blizzard/src/index.ts` +- Create: `packages/blizzard/package.json` + +**Interfaces:** + +- Consumes: `CharacterKey` from `@slashwho/domain` and an injected `fetch` implementation. +- Produces: + +```ts +export type AchievementFingerprint = ReadonlyMap; + +export type BlizzardRosterCharacter = Readonly<{ + key: CharacterKey; + displayName: string; + className: string; + level: number; +}>; + +export interface BlizzardGateway { + getGuildRoster( + root: CharacterKey, + signal?: AbortSignal + ): Promise; + getAchievementFingerprint( + key: CharacterKey, + signal?: AbortSignal + ): Promise; +} + +export function compareFingerprints( + root: AchievementFingerprint, + candidate: AchievementFingerprint, + policy: { minimumCommon: number; minimumIdenticalPercent: number } +): { common: number; identical: number; isMatch: boolean }; +``` + +- [ ] **Step 1: Write failing matcher tests** + +```ts +it("requires both the common-achievement floor and identical-timestamp floor", () => { + expect(compareFingerprints(root, tooSmall, policy).isMatch).toBe(false); + expect(compareFingerprints(root, belowPercent, policy).isMatch).toBe(false); + expect(compareFingerprints(root, exactBoundary, policy)).toMatchObject({ + common: 200, + identical: 40, + isMatch: true + }); +}); +``` + +- [ ] **Step 2: Run the matcher test to verify it fails** + +Run: `pnpm --filter @slashwho/blizzard test -- fingerprint.test.ts` + +Expected: FAIL because the workspace and matcher do not exist. + +- [ ] **Step 3: Implement the smallest pure matcher and ephemeral types** + +Extract only numeric achievement ID/timestamp pairs from a validated response. Compare maps without mutation, return counts only to the caller, and do not add serialization or storage helpers. + +- [ ] **Step 4: Write failing HTTP-boundary tests** + +```ts +it("passes the abort signal and never includes an upstream body in its error", async () => { + const gateway = createBlizzardClient({ + fetch, + clientId: "id", + clientSecret: "secret" + }); + await expect( + gateway.getAchievementFingerprint(key, controller.signal) + ).rejects.toMatchObject({ + kind: "transient", + retryAfterMs: 60_000 + }); + expect(fetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ signal: controller.signal }) + ); +}); +``` + +- [ ] **Step 5: Implement the Blizzard client** + +Implement cached-in-process OAuth token acquisition, regional API URL construction, roster normalization to `BlizzardRosterCharacter`, achievement extraction, `Retry-After` parsing, and typed `not_found`, `transient`, and `schema_drift` failures. Keep token and raw payload values local to `client.ts`. + +- [ ] **Step 6: Run package tests and static checks** + +Run: `pnpm --filter @slashwho/blizzard test && pnpm --filter @slashwho/blizzard typecheck` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add packages/blizzard +git commit -m "feat(blizzard): add ephemeral fingerprint client" +``` + +## Task 2: Build the cap-aware domain sweep + +**Files:** + +- Create: `packages/domain/src/fingerprint-discovery.ts` +- Create: `packages/domain/src/fingerprint-discovery.test.ts` +- Modify: `packages/domain/src/index.ts` +- Modify: `packages/domain/package.json` + +**Interfaces:** + +- Consumes: `CharacterKey`, `DiscoveredCharacter`, and `toRaiderIoUrl` from existing domain modules. The caller supplies a domain-owned adapter; the domain package does not import `@slashwho/blizzard`. +- Produces: + +```ts +export type FingerprintCandidate = Readonly<{ + key: CharacterKey; + displayName: string; + className: string; + level: number; +}>; + +export interface FingerprintGateway { + getGuildRoster( + root: CharacterKey, + signal?: AbortSignal + ): Promise; + getAchievementFingerprint( + key: CharacterKey, + signal?: AbortSignal + ): Promise>; +} + +export type FingerprintSweepOutcome = + | { + kind: "matched"; + characters: readonly DiscoveredCharacter[]; + requestsUsed: number; + } + | { + kind: "capped"; + characters: readonly DiscoveredCharacter[]; + requestsUsed: number; + } + | { + kind: "failure"; + code: "upstream_unavailable" | "upstream_schema_changed"; + retryable: boolean; + retryAfterMs?: number; + }; + +export function discoverFingerprintMatches( + root: CharacterKey, + gateway: FingerprintGateway, + options: { + requestCap: number; + minimumCommon: number; + minimumIdenticalPercent: number; + isSuppressed(key: CharacterKey): Promise; + isPrivacyHidden(key: CharacterKey): Promise; + signal?: AbortSignal; + } +): Promise; +``` + +- [ ] **Step 1: Write failing domain tests** + +```ts +it("fetches the root once, skips suppressed/privacy-hidden candidates, and stops at its cap", async () => { + await expect( + discoverFingerprintMatches(root, gateway, options) + ).resolves.toMatchObject({ + kind: "capped", + requestsUsed: 3, + characters: [expect.objectContaining({ source: "fingerprint" })] + }); +}); +``` + +- [ ] **Step 2: Run the domain test to verify it fails** + +Run: `pnpm --filter @slashwho/domain test -- fingerprint-discovery.test.ts` + +Expected: FAIL because the domain sweep is not exported. + +- [ ] **Step 3: Implement the in-memory sweep** + +Count every roster and achievement request against `requestCap`; request the root fingerprint once; evaluate roster candidates deterministically; check suppression/privacy before retaining a result; convert accepted matches to ordinary `DiscoveredCharacter` rows with source `fingerprint`; discard each candidate fingerprint after comparison. Return `capped` only after a measured cap stop. + +- [ ] **Step 4: Add failure and abort tests** + +```ts +it("returns a retryable failure for a 429 and throws the abort reason without a partial result", async () => { + gateway.getAchievementFingerprint = async () => { + throw rateLimited; + }; + await expect( + discoverFingerprintMatches(root, gateway, options) + ).resolves.toMatchObject({ kind: "failure", retryable: true }); + await expect( + discoverFingerprintMatches(root, gateway, { + ...options, + signal: aborted.signal + }) + ).rejects.toBe(aborted.signal.reason); +}); +``` + +- [ ] **Step 5: Run focused tests and commit** + +Run: `pnpm --filter @slashwho/domain test -- fingerprint-discovery.test.ts` + +Expected: PASS. + +```bash +git add packages/domain +git commit -m "feat(domain): add cap-aware fingerprint sweep" +``` + +## Task 3: Add durable sweep state and rolling budget admission + +**Files:** + +- Modify: `packages/database/src/schema.ts` +- Create: `packages/database/drizzle/0002_fingerprint_sweeps.sql` +- Create: `packages/database/drizzle/meta/0002_snapshot.json` +- Modify: `packages/database/drizzle/meta/_journal.json` +- Modify: `packages/database/src/repositories.ts` +- Modify: `packages/database/src/postgres-repositories.ts` +- Modify: `packages/database/src/postgres-repositories.test.ts` +- Modify: `packages/database/src/public-api.typecheck.ts` + +**Interfaces:** + +- Consumes: canonical root keys and discovery-run IDs. +- Produces: + +```ts +export type FingerprintAdmission = + | { kind: "not_due" } + | { kind: "waiting"; retryAt: Date } + | { kind: "admitted"; reservationId: string; requestCap: number }; + +export interface FingerprintSweepRepository { + requestAdmission(input: { + runId: string; + key: CharacterKey; + requestCap: number; + hourlyBudget: number; + cadenceCutoff: Date; + at: Date; + }): Promise; + recordRequest(reservationId: string, count: number, at: Date): Promise; + finish( + reservationId: string, + input: { published: boolean; at: Date; limitationCode: string | null } + ): Promise; + release(reservationId: string, at: Date): Promise; + listWaiting(limit: number): Promise; +} +``` + +- [ ] **Step 1: Write failing PostgreSQL integration tests** + +```ts +it("admits only the FIFO head when two caps would exceed the rolling budget", async () => { + await repository.requestAdmission(first); + await expect(repository.requestAdmission(second)).resolves.toMatchObject({ + kind: "waiting" + }); + await repository.finish(firstReservation, { + published: true, + at, + limitationCode: null + }); + await expect(repository.requestAdmission(second)).resolves.toMatchObject({ + kind: "admitted" + }); +}); +``` + +- [ ] **Step 2: Run the integration test to verify it fails** + +Run: `pnpm test:integration -- postgres-repositories.test.ts` + +Expected: FAIL because no fingerprint tables or repository exist. + +- [ ] **Step 3: Add the migration and schema types** + +Create internal tables for per-root sweep state, FIFO admission rows, and reservation accounting. Add `fingerprint` to `discovery_source`. A reservation records cap, used count, admitted time, expiry time, and terminal release/completion metadata; it stores no upstream or matching data. + +- [ ] **Step 4: Implement transactional repository methods** + +Under one global PostgreSQL advisory lock, select the oldest waiting eligible row, calculate active commitment as used plus unreleased reservation capacity, and admit only when the full cap fits. On finish/release, retain used count until the reservation's one-hour expiry, release unused count immediately, and set the seven-day timestamp only when the snapshot was published. + +- [ ] **Step 5: Add atomicity and cadence tests** + +```ts +it("does not advance cadence or retain unused capacity after an aborted sweep", async () => { + const admitted = await repository.requestAdmission(input); + await repository.recordRequest(admitted.reservationId, 3, at); + await repository.release(admitted.reservationId, at); + await expect( + repository.requestAdmission({ ...input, at: plusOneMinute }) + ).resolves.toMatchObject({ kind: "admitted" }); +}); +``` + +- [ ] **Step 6: Run migration and integration verification** + +Run: `pnpm test:integration -- postgres-repositories.test.ts && pnpm --filter @slashwho/database typecheck` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add packages/database +git commit -m "feat(database): reserve fingerprint sweep budget" +``` + +## Task 4: Add private FIFO admission dispatch + +**Files:** + +- Modify: `packages/database/src/queue.ts` +- Modify: `packages/database/src/queue.test.ts` +- Modify: `packages/database/src/index.ts` +- Modify: `apps/worker/src/runtime.ts` +- Modify: `apps/worker/src/runtime.test.ts` + +**Interfaces:** + +- Consumes: waiting discovery-run IDs from `FingerprintSweepRepository`. +- Produces: + +```ts +export const fingerprintAdmissionQueueName = "fingerprint-admission"; + +export interface DiscoveryQueue { + // Existing members remain unchanged. + enqueueFingerprintAdmission(runId: string): Promise; + workFingerprintAdmissions( + handler: (runId: string) => Promise + ): Promise; +} +``` + +- [ ] **Step 1: Write failing queue/runtime tests** + +```ts +it("registers the private admission worker and re-enqueues only admitted discovery runs", async () => { + await admissionHandler(waitingRunId); + expect(fakes.enqueued).toEqual([{ runId: waitingRunId, key }]); + expect(fakes.handler.execute).not.toHaveBeenCalled(); +}); +``` + +- [ ] **Step 2: Run focused tests to verify they fail** + +Run: `pnpm --filter @slashwho/database test -- queue.test.ts && pnpm --filter @slashwho/worker test -- runtime.test.ts` + +Expected: FAIL because the private queue is not registered. + +- [ ] **Step 3: Implement the private pg-boss queue** + +Create and start `fingerprint-admission` with a singleton key per run. Its handler asks the repository to admit FIFO work, then enqueues an admitted run back onto `discover-character`. Waiting runs remain durable in the admission table/queue and do not consume a discovery worker execution or delivery retry. + +- [ ] **Step 4: Wire shutdown and recovery** + +Make runtime startup recover waiting admission rows before readiness, and make `stop()` cease new admission work before its existing graceful drain. Do not add a public queue or API route. + +- [ ] **Step 5: Run focused tests and commit** + +Run: `pnpm --filter @slashwho/database test -- queue.test.ts && pnpm --filter @slashwho/worker test -- runtime.test.ts` + +Expected: PASS. + +```bash +git add packages/database apps/worker +git commit -m "feat(worker): dispatch fingerprint admissions" +``` + +## Task 5: Orchestrate merged snapshots in the discovery handler + +**Files:** + +- Modify: `packages/application/src/discovery-job-handler.ts` +- Modify: `packages/application/src/discovery-job-handler.test.ts` +- Modify: `packages/application/src/index.ts` +- Modify: `apps/worker/src/runtime.ts` + +**Interfaces:** + +- Consumes: existing `discoverCharacter`, `discoverFingerprintMatches`, `FingerprintSweepRepository`, `BlizzardGateway`, and `DiscoveryWorkContext`. `packages/application/src/blizzard-fingerprint-adapter.ts` adapts `BlizzardGateway` to the domain-owned `FingerprintGateway`; it does not duplicate matching or upstream logic. +- Produces an extended handler option: + +```ts +export type DiscoveryJobHandlerOptions = { + repositories: Repositories; + gateway: RaiderIoGateway; + blizzardGateway: BlizzardGateway; + fingerprint: { + requestCap: number; + hourlyBudget: number; + cadenceMs: number; + minimumCommon: number; + minimumIdenticalPercent: number; + }; + // Existing retry, clock, logger, and cache options remain. +}; +``` + +- [ ] **Step 1: Write failing handler tests for deferred admission** + +```ts +it("defers an eligible run to private FIFO admission without consuming a delivery retry", async () => { + repositories.fingerprintSweeps.requestAdmission = async () => ({ + kind: "waiting", + retryAt + }); + await handler.execute(run.id, delivery()); + expect(repositories.runs.find(run.id)).resolves.toMatchObject({ + status: "queued" + }); + expect(gateway.getCharacter).toHaveBeenCalled(); + expect(blizzardGateway.getGuildRoster).not.toHaveBeenCalled(); +}); +``` + +- [ ] **Step 2: Run the handler test to verify it fails** + +Run: `pnpm --filter @slashwho/application test -- discovery-job-handler.test.ts` + +Expected: FAIL because the handler has no fingerprint admission branch. + +- [ ] **Step 3: Implement admission, sweeping, and merge** + +Run existing Raider.IO discovery first. If the normal result cannot produce a trustworthy snapshot, preserve its existing behaviour and never start a fingerprint sweep. For a trustworthy result, request admission. On `waiting`, persist only the internal admission state and return without snapshot publication. On `admitted`, invoke the domain sweep, record each consumed Blizzard request, merge deduplicated fingerprint observations with Raider.IO observations, then call the existing atomic snapshot repository once. + +- [ ] **Step 4: Write failing failure/partial/abort tests** + +```ts +it("publishes only a cap-bounded partial result and releases an aborted reservation", async () => { + fingerprintSweep.mockResolvedValue({ + kind: "capped", + characters: [match], + requestsUsed: 300 + }); + await handler.execute(run.id, delivery()); + expect(snapshot.limitationCode).toBe("fingerprint_sweep_capped"); + + controller.abort(abortReason); + await expect( + handler.execute(nextRun.id, { ...delivery(), signal: controller.signal }) + ).rejects.toBe(abortReason); + expect(repositories.fingerprintSweeps.release).toHaveBeenCalled(); +}); +``` + +- [ ] **Step 5: Extend allowlisted logs without sensitive data** + +Add only queue wait, reservation/use counts, duration, and final limitation class. Extend `apps/worker/src/logger.test.ts` with achievement IDs, timestamps, tokens, and scores as redaction markers, and prove none reaches output. + +- [ ] **Step 6: Run focused tests and commit** + +Run: `pnpm --filter @slashwho/application test -- discovery-job-handler.test.ts && pnpm --filter @slashwho/worker test -- logger.test.ts` + +Expected: PASS. + +```bash +git add packages/application apps/worker +git commit -m "feat(application): merge fingerprint discovery snapshots" +``` + +## Task 6: Validate worker configuration and public non-disclosure + +**Files:** + +- Modify: `apps/worker/src/config.ts` +- Modify: `apps/worker/src/config.test.ts` +- Modify: `apps/worker/src/runtime.ts` +- Modify: `packages/contracts/src/contracts.test.ts` +- Modify: `packages/application/src/serializers.test.ts` +- Modify: `apps/web/src/app/privacy/page.tsx` (or create it if absent) +- Modify: privacy-page test colocated with the route/component + +**Interfaces:** + +- Consumes: worker environment values `BLIZZARD_CLIENT_ID`, `BLIZZARD_CLIENT_SECRET`, `BLIZZARD_SWEEP_REQUEST_CAP`, `BLIZZARD_HOURLY_REQUEST_BUDGET`, `FINGERPRINT_MINIMUM_COMMON`, `FINGERPRINT_MINIMUM_IDENTICAL_PERCENT`, and `FINGERPRINT_SWEEP_CADENCE_HOURS`. +- Produces a `WorkerConfig` whose fingerprint fields are positive validated numbers and whose cadence defaults to 168 hours. + +- [ ] **Step 1: Write failing configuration and serializer tests** + +```ts +it("rejects missing Blizzard credentials and invalid sweep bounds", () => { + expect(() => loadWorkerConfig({ DATABASE_URL: url })).toThrow( + "blizzard_client_id_required" + ); + expect(() => + loadWorkerConfig({ ...env, BLIZZARD_SWEEP_REQUEST_CAP: "0" }) + ).toThrow("invalid_blizzard_sweep_request_cap"); +}); + +it("never exposes fingerprint source, score, queue, or reservation fields", () => { + expect(serializeCharacterResource(snapshot)).not.toHaveProperty( + "discoverySource" + ); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pnpm --filter @slashwho/worker test -- config.test.ts && pnpm --filter @slashwho/application test -- serializers.test.ts` + +Expected: FAIL because the fingerprint configuration and privacy wording are absent. + +- [ ] **Step 3: Implement validated configuration and runtime construction** + +Load credentials only in worker configuration; pass them directly to `createBlizzardClient`; do not expose them to web configuration. Register defaults of 28,800/hour, 20%, 200 common achievements, and 168 hours. Keep all public schemas and serializers unchanged except for tests proving no internal field leaks. + +- [ ] **Step 4: Document the privacy boundary** + +Add concise `/privacy` copy stating that privacy-hidden Raider.IO ownership is excluded from fingerprint-derived links and that public lists do not disclose discovery method. Do not add a public opt-out flow. + +- [ ] **Step 5: Run focused tests and commit** + +Run: `pnpm --filter @slashwho/worker test -- config.test.ts && pnpm --filter @slashwho/application test -- serializers.test.ts && pnpm --filter @slashwho/web test -- privacy` + +Expected: PASS. + +```bash +git add apps/worker packages/application packages/contracts apps/web +git commit -m "feat(worker): configure private Blizzard sweeps" +``` + +## Task 7: Run end-to-end verification and staging smoke test + +**Files:** + +- Modify: deployment/environment documentation if Railway variable setup is not already recorded. +- Modify: `README.md` only if it names discovery sources or privacy behaviour contradicted by this feature. + +**Interfaces:** + +- Consumes: all prior tasks and operator-managed Railway worker secrets. +- Produces: a verified branch and a manually recorded staging smoke-test result without secret or raw upstream data. + +- [ ] **Step 1: Run the complete local gate** + +Run: + +```bash +pnpm format:check +pnpm lint +pnpm typecheck +pnpm test +pnpm build +``` + +Expected: every command exits 0. If Docker is unavailable, start Docker Desktop hidden, verify `docker version`, then rerun the integration suite; do not alter tests to skip it. + +- [ ] **Step 2: Review public-contract and retention evidence** + +Run: + +```bash +rg -n "fingerprint|achievement|blizzard" apps/web packages/contracts packages/application/src/serializers.ts +git diff main...HEAD --check +``` + +Expected: only the approved `/privacy` wording and internal implementation references appear; contracts and serializers contain no score, source, queue, credential, ID, timestamp, or raw-payload field. + +- [ ] **Step 3: Stage and run the bounded staging smoke test** + +Deploy to Railway `test` with only the already-provisioned worker credentials. Submit one known eligible public root, confirm the worker reserves its cap, completes within it, and the public page/API shows a normal undifferentiated list. Record only root key, run outcome, request count, duration, snapshot state, and limitation class. + +- [ ] **Step 4: Commit documentation evidence and open the feature PR** + +```bash +git add README.md docs apps packages +git commit -m "docs: record fingerprint sweep validation" +git push -u origin feat/achievement-fingerprint-discovery +gh pr create --base main --title "feat: add achievement fingerprint discovery" +``` + +## Plan self-review + +- Spec coverage: Tasks 1–2 implement ephemeral Blizzard matching and the threshold; Tasks 3–4 implement seven-day eligibility, FIFO admission, and rolling budget; Task 5 implements merged atomic snapshots, cap handling, retries, and shutdown release; Task 6 implements config, privacy copy, and public non-disclosure; Task 7 verifies all acceptance criteria in CI and staging. +- Placeholder scan: no deferred implementation steps, unnamed types, or generic testing directions remain; every task names its files, interfaces, commands, and expected result. +- Type consistency: `BlizzardGateway`, `FingerprintSweepOutcome`, `FingerprintSweepRepository`, `FingerprintAdmission`, and the extended `DiscoveryJobHandlerOptions` are introduced before later tasks consume them. + +## Execution handoff + +Plan complete and saved to `docs/superpowers/plans/2026-08-10-achievement-fingerprint-discovery-implementation.md`. + +1. **Subagent-Driven (recommended)** — dispatch a fresh subagent per task and review between tasks. +2. **Inline Execution** — execute the tasks in this session with checkpoints. diff --git a/docs/superpowers/specs/2026-08-10-achievement-fingerprint-discovery-design.md b/docs/superpowers/specs/2026-08-10-achievement-fingerprint-discovery-design.md new file mode 100644 index 0000000..00481fe --- /dev/null +++ b/docs/superpowers/specs/2026-08-10-achievement-fingerprint-discovery-design.md @@ -0,0 +1,241 @@ +# Achievement-Fingerprint Discovery Design + +**Date:** 2026-08-10 + +**Status:** Approved for implementation planning + +## Summary + +SlashWho will extend its existing durable `discover-character` job with Blizzard +achievement-fingerprint discovery. The job keeps Raider.IO discovery as its first +phase, then—when a root character is eligible—uses the root's current Blizzard +guild roster as the initial candidate source. It compares each candidate's +achievement completion timestamps with the root's in memory and folds accepted +matches into the same immutable snapshot as Raider.IO relationships. + +This is one deployable feature. Public search, character, history, and snapshot +API shapes do not change. Fingerprint-derived and Raider.IO-declared characters +appear in one undifferentiated alt list. The implementation persists neither +achievement fingerprints nor scores, timestamps, achievement IDs, raw Blizzard +responses, access tokens, or credentials. + +The design records the decisions in the completed +[achievement-fingerprint map](https://github.com/Erilla/SlashWho/issues/4). + +## Goals + +- Automatically discover same-account characters from Blizzard achievement + completion data during eligible searches. +- Begin candidate enumeration with the root character's current Blizzard guild + roster. +- Preserve the existing immutable-snapshot and durable-worker contracts. +- Bound Blizzard use with a FIFO, shared rolling budget of 28,800 requests per + hour and a configured per-sweep request cap. +- Reuse the existing public character and search API without exposing discovery + provenance, confidence, queue state, or budget information. +- Exclude privacy-hidden ownership from fingerprint-derived linkage, using + Raider.IO as the sole privacy signal. +- Keep all achievement material ephemeral to one in-memory sweep. + +## Non-goals + +- Candidate sources beyond the root's current guild roster, including guild + history, raid-log collection, and a global guild index. +- Cross-region matching or China-region matching; Blizzard's relevant Profile + API does not support the latter, and fingerprints are not comparable across + regions. +- Persisting achievement IDs, timestamps, signatures, match scores, raw bodies, + access tokens, or user-supplied Blizzard credentials. +- A public match score, source badge, queue indicator, or operational budget + display. +- Stable Blizzard character IDs. Rename/transfer continuity remains a deferred + standalone effort ([#27](https://github.com/Erilla/SlashWho/issues/27)). +- Tracing `Ictinus` to `Mistakinus`; that is a separate investigation + ([#24](https://github.com/Erilla/SlashWho/issues/24)). + +## Terminology + +This design uses the project terms recorded in `CONTEXT.md`: + +- **privacy-hidden ownership** is Raider.IO's intentionally non-public ownership + state and the only privacy signal used for inferred links; +- **fingerprint-derived link** is a relationship inferred from Blizzard + achievement completion data; +- **alt list** is the public, provenance-free relationship list; +- **partial snapshot** is an immutable result known not to contain every + discoverable relationship; and +- **ephemeral fingerprint** is achievement data held only for one sweep and + discarded before publication. + +## Architecture + +`discover-character` remains the only durable job and only snapshot writer. Its +phases become: + +1. Run the existing Raider.IO traversal and produce its transient relationship + observations. +2. Determine whether the root is eligible for a fingerprint sweep. A root is + eligible when no successfully published fingerprint sweep has occurred in + the previous seven days. +3. When eligible, enter the shared Blizzard admission queue. The job waits in + FIFO order for capacity; it does not fail a user search or occupy a worker + execution while waiting. +4. At admission, reserve the full configured sweep cap against the shared, + rolling 28,800-request/hour budget. No Blizzard request begins before this + reservation succeeds. +5. Fetch the root guild roster and root achievement data, then fetch and compare + candidates until the cap is reached or the roster is exhausted. +6. Merge accepted fingerprint-derived characters with the Raider.IO observations + and atomically publish one snapshot. + +The job has no separate public API, snapshot type, or completion state for the +fingerprint phase. A completed cap-bounded sweep is a successful partial +snapshot with the internal limitation `fingerprint_sweep_capped`. A successful +sweep with no guild or an empty roster is a measured result with no +fingerprint-derived additions. + +## Blizzard integration and matching + +The worker obtains a client-credentials access token using the operator-managed +`BLIZZARD_CLIENT_ID` and `BLIZZARD_CLIENT_SECRET`. It uses Blizzard's regional +Profile API to resolve the root's current guild and roster, then fetches +achievement completion data for the root and each roster candidate. + +An ephemeral fingerprint maps achievement IDs to their completion timestamps. +For a same-region candidate, the matcher counts common achievement IDs and the +subset whose timestamps are identical. It accepts a candidate only when both +conditions hold: + +- at least 200 achievement IDs are common; and +- at least 20% of those common IDs have identical completion timestamps. + +The accepted candidate contributes only its ordinary character fields and an +internal `fingerprint` discovery source to the combined snapshot membership. +Scores and the comparison inputs are discarded immediately after each candidate +is evaluated. No fingerprint may cross a job boundary or survive publication. + +Before a fingerprint-derived character is admitted to snapshot membership, the +worker applies the existing Raider.IO privacy-hidden ownership check. A +privacy-hidden candidate is excluded from fingerprint discovery. Raider.IO +relationships continue to follow Raider.IO visibility as they do today. + +## Freshness, queueing, and budget accounting + +Raider.IO retains its existing 24-hour refresh model. Fingerprint discovery is +decoupled: a root may successfully run at most one sweep every seven days. + +When a root is due, the first search that creates or refreshes its discovery run +causes the fingerprint phase to be queued. Later searches for that root reuse +the same active run. While an eligible sweep waits or runs, the existing current +snapshot remains visible. Public responses do not state that a fingerprint sweep +is due, queued, admitted, or running. + +The system persists only operational state required to enforce this policy: + +- per-root successful fingerprint-sweep time and the active/queued run + reference; +- an internal terminal reason for the sweep; and +- a Blizzard-budget reservation ledger containing run identity, reserved count, + accounting window, and release/expiry state. + +The ledger is updated transactionally when capacity is reserved. Used requests +remain charged to the rolling window. Unused reserved capacity is released when +the sweep finishes or aborts. A retryable failure releases only unused capacity; +it never erases the usage already consumed. Only successful snapshot publication +advances the seven-day eligibility window. + +All values are validated worker configuration: client credentials, sweep cap, +hourly budget (initially 28,800), identical-timestamp percentage (initially +20), common-achievement floor (initially 200), and sweep cadence (initially +seven days). + +## Snapshot and failure semantics + +A fingerprint sweep is atomic. Its roster, candidate list, fingerprints, +comparison results, and progress cursor exist only in process memory. + +- Reaching the configured request cap publishes the allowed partial snapshot and + ends the sweep. A later eligible run starts again from the root; it does not + resume a cursor or reuse a candidate list. +- A transport failure, 429, 5xx, malformed response, schema drift, process + abort, or deployment shutdown before publication discards all in-memory sweep + state and leaves the prior snapshot current. +- Retryable failures use the existing bounded exponential-backoff path. A retry + restarts the whole atomic job from the root. +- Graceful shutdown stops beginning new Blizzard requests, abandons an unfinished + sweep before the existing drain deadline, and relies on a later retry instead + of extending deployment draining. +- A no-guild or empty-roster response is a successful measured sweep. Unsupported + region coverage is an explicitly recorded internal limitation rather than an + unmeasured failure. + +Snapshot membership stores no score or confidence. It may retain the existing +internal discovery-source field for diagnostics, extended with `fingerprint`; +the shared serializers continue to omit this field from every public response. + +## Observability and privacy + +Structured logs and internal metrics record only operational information: + +- FIFO queue depth and admission wait time; +- per-caller admission; +- per-sweep cap reservation versus actual request use; +- rolling shared-budget commitment; +- retry and failure accounting; and +- sweep duration and final internal limitation class. + +The worker alerts a maintainer when admission has been blocked for 15 minutes, +reserved capacity exceeds 90%, or Blizzard returns a 429 response. No log, +metric, public response, or alert includes credentials, tokens, raw response +bodies, achievement IDs, timestamps, scores, or per-character comparison data. + +The `/privacy` page documents that privacy-hidden Raider.IO ownership excludes +fingerprint-derived links. There is no separate SlashWho opt-out mechanism. + +## Testing strategy + +Unit tests cover: + +- fingerprint extraction and comparison, including both threshold boundaries; +- privacy-hidden exclusion; +- root-guild-roster candidate ordering and no-guild/empty-roster outcomes; +- request-cap and rolling-budget accounting; +- seven-day eligibility and active-run reuse; and +- every classification of success, partial result, failure, abort, and retry. + +PostgreSQL integration tests prove that concurrent sweeps cannot over-reserve +the shared rolling budget, waiting sweeps are admitted FIFO, duplicate searches +reuse one root run, only successful publication advances cadence, cap-bounded +runs publish an allowed partial snapshot, and aborted/retried runs neither +persist fingerprint material nor replace the prior snapshot. + +Worker integration tests use sanitized Blizzard fixtures for token acquisition, +no guild, empty roster, successful matches, 429 responses, transport failures, +and schema drift. Existing API and browser tests continue to prove that public +payloads and character pages reveal neither provenance nor queue/budget state. + +Staging acceptance uses the operator-managed credentials to complete a small, +known public eligible sweep within its configured cap and shared-budget +reservation. It must find known eligible matches, create no sensitive retained +fingerprint material, and leave the public API shape unchanged. + +## Acceptance criteria + +The feature is ready for staging validation when: + +1. An eligible search automatically queues one fingerprint sweep for its root; + later searches reuse it. +2. The worker uses the current root guild roster, only compares same-region + candidates, and accepts only the approved 20%/200 threshold. +3. Privacy-hidden ownership never produces a fingerprint-derived public link. +4. The worker cannot begin a sweep without first reserving its whole configured + cap within the shared rolling budget. +5. A cap-bounded sweep publishes one partial snapshot with an internal + `fingerprint_sweep_capped` reason; all other interruption paths preserve the + previous snapshot. +6. A successfully published sweep suppresses another sweep for that root for + seven days, while daily Raider.IO refresh behavior remains intact. +7. Database, worker, API, and browser tests prove that no raw or compact + fingerprint material, score, or public provenance is retained or exposed. +8. Internal alerts fire for the agreed queue-blocked, 90%-reservation, and 429 + conditions. diff --git a/packages/application/package.json b/packages/application/package.json index 27f77cc..b7f7839 100644 --- a/packages/application/package.json +++ b/packages/application/package.json @@ -8,6 +8,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@slashwho/blizzard": "workspace:*", "@slashwho/contracts": "workspace:*", "@slashwho/database": "workspace:*", "@slashwho/domain": "workspace:*", diff --git a/packages/application/src/blizzard-fingerprint-adapter.ts b/packages/application/src/blizzard-fingerprint-adapter.ts new file mode 100644 index 0000000..6b60aec --- /dev/null +++ b/packages/application/src/blizzard-fingerprint-adapter.ts @@ -0,0 +1,49 @@ +import type { BlizzardGateway } from "@slashwho/blizzard"; +import type { FingerprintGateway } from "@slashwho/domain"; + +export function createBlizzardFingerprintAdapter( + gateway: BlizzardGateway, + options: { + requestCap: number; + recordRequest: () => Promise; + onRateLimited?: () => Promise | void; + } +): FingerprintGateway { + let requestsUsed = 0; + + async function recordProfileRequest(): Promise { + if (requestsUsed >= options.requestCap) { + throw Object.assign(new Error("fingerprint_cap_reached"), { + kind: "fingerprint_cap_reached" + }); + } + await options.recordRequest(); + requestsUsed += 1; + } + + async function request(operation: () => Promise): Promise { + try { + return await operation(); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "kind" in error && + (error as { kind?: unknown }).kind === "transient" && + (error as { status?: unknown }).status === 429 + ) { + await options.onRateLimited?.(); + } + throw error; + } + } + + return { + getGuildRoster: (root, signal) => + request(() => gateway.getGuildRoster(root, signal, recordProfileRequest)), + getAchievementFingerprint: (key, signal) => + request(() => + gateway.getAchievementFingerprint(key, signal, recordProfileRequest) + ) + }; +} diff --git a/packages/application/src/discovery-job-handler.test.ts b/packages/application/src/discovery-job-handler.test.ts index 94f634c..0318271 100644 --- a/packages/application/src/discovery-job-handler.test.ts +++ b/packages/application/src/discovery-job-handler.test.ts @@ -5,11 +5,13 @@ import type { } from "@slashwho/database"; import type { CharacterKey, + FingerprintCandidate, RaiderIoCharacter, RaiderIoGateway, RaiderIoProfile } from "@slashwho/domain"; -import { describe, expect, it } from "vitest"; +import type { BlizzardGateway } from "@slashwho/blizzard"; +import { describe, expect, it, vi } from "vitest"; import { createDiscoveryJobHandler } from "./discovery-job-handler"; @@ -28,6 +30,20 @@ const thirdKey: CharacterKey = { realm: "area-52", name: "third" }; +const fingerprintKey: CharacterKey = { + region: "eu", + realm: "silvermoon", + name: "fingerprint-match" +}; + +function achievementFingerprint(count = 200): ReadonlyMap { + return new Map( + Array.from({ length: count }, (_unused, index) => [ + index + 1, + 1_700_000_000 + index + ]) + ); +} function character(key: CharacterKey): RaiderIoCharacter { return { @@ -75,6 +91,30 @@ class MutableGateway implements RaiderIoGateway { } } +class MutableBlizzardGateway implements BlizzardGateway { + roster: readonly FingerprintCandidate[] = []; + fingerprints = new Map>(); + + async getGuildRoster( + _key?: CharacterKey, + _signal?: AbortSignal, + onProfileRequest?: () => Promise | void + ): Promise { + await onProfileRequest?.(); + if (this.roster.length > 0) await onProfileRequest?.(); + return this.roster; + } + + async getAchievementFingerprint( + key: CharacterKey, + _signal?: AbortSignal, + onProfileRequest?: () => Promise | void + ): Promise> { + await onProfileRequest?.(); + return this.fingerprints.get(keyId(key)) ?? new Map(); + } +} + function keyId(key: CharacterKey): string { return `${key.region}/${key.realm}/${key.name}`; } @@ -204,6 +244,9 @@ function createMemoryRepositories(): Repositories { await thisRunComplete(input.runId, id); return snapshot; }, + async createAndFinishFingerprintSweep(input) { + return this.create(input); + }, async getCurrent(key) { return ( [...snapshots.values()] @@ -263,6 +306,27 @@ function createMemoryRepositories(): Repositories { async cleanupExpired() { return 0; } + }, + fingerprintSweeps: { + async requestAdmission() { + return { kind: "not_due" }; + }, + async recordRequest() {}, + async finish() {}, + async release() {}, + async listWaiting() { + return []; + }, + async listAdmittedUndispatched() { + return []; + }, + async markDispatched() {}, + async admitWaiting() { + return { kind: "settled" }; + }, + async cleanupExpired() { + return 0; + } } }; @@ -283,6 +347,15 @@ function handlerFor( return createDiscoveryJobHandler({ repositories, gateway, + blizzardGateway: new MutableBlizzardGateway(), + fingerprint: { + requestCap: 300, + hourlyBudget: 28_800, + cadenceMs: 7 * 24 * 60 * 60 * 1_000, + minimumCommon: 200, + minimumIdenticalPercent: 20 + }, + enqueueFingerprintAdmission: async () => {}, requestCap: 12, now: () => new Date("2026-08-05T08:00:00.000Z"), random: () => 0, @@ -303,6 +376,395 @@ function delivery(attempt = 1, maxAttempts = 5) { } describe("discovery job handler", () => { + it("defers an eligible run to private FIFO admission without consuming a delivery retry", async () => { + // Break caught: budget waiting could consume a discovery retry or publish + // the Raider.IO-only intermediate result before the atomic sweep resumes. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + const retryAt = new Date("2026-08-05T08:15:00.000Z"); + const blockedSince = new Date("2026-08-05T07:44:00.000Z"); + repositories.fingerprintSweeps.requestAdmission = async () => { + const claimed = await repositories.runs.find(run.id); + if (!claimed) throw new Error("discovery_run_not_found"); + claimed.status = "queued"; + claimed.attempt -= 1; + return { kind: "waiting", retryAt, blockedSince }; + }; + const gateway = new MutableGateway(); + gateway.getCharacter = vi.fn(gateway.getCharacter.bind(gateway)); + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.getGuildRoster = vi.fn( + blizzardGateway.getGuildRoster.bind(blizzardGateway) + ); + + const enqueueFingerprintAdmission = vi.fn(async () => {}); + const alerts: unknown[] = []; + await handlerFor(repositories, gateway, { + blizzardGateway, + enqueueFingerprintAdmission, + fingerprintAlertNotifier: { + notify: async (alert) => { + alerts.push(alert); + } + } + }).execute(run.id, delivery()); + + await expect(repositories.runs.find(run.id)).resolves.toMatchObject({ + status: "queued", + attempt: 0 + }); + expect(gateway.getCharacter).toHaveBeenCalled(); + expect(blizzardGateway.getGuildRoster).not.toHaveBeenCalled(); + expect(enqueueFingerprintAdmission).toHaveBeenCalledWith(run.id); + expect(alerts).toEqual([ + { + event: "fingerprint_admission_blocked", + details: { blockedForMs: 16 * 60_000 } + } + ]); + await expect( + repositories.snapshots.getCurrent(rootKey) + ).resolves.toBeNull(); + }); + + it("accounts for an admitted sweep and publishes one deduplicated merged snapshot", async () => { + // Break caught: fingerprint observations could be published separately, + // duplicated, or consume Blizzard capacity without durable accounting. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = vi.fn(async () => ({ + kind: "admitted" as const, + reservationId: "reservation-1", + requestCap: 300 + })); + repositories.fingerprintSweeps.recordRequest = vi.fn(async () => {}); + repositories.fingerprintSweeps.finish = vi.fn(async () => {}); + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.roster = [ + { + key: secondKey, + displayName: "Second from Blizzard", + className: "Mage", + level: 80 + }, + { + key: fingerprintKey, + displayName: "Fingerprint Match", + className: "Priest", + level: 80 + } + ]; + const fingerprint = achievementFingerprint(); + blizzardGateway.fingerprints.set(keyId(rootKey), fingerprint); + blizzardGateway.fingerprints.set(keyId(secondKey), fingerprint); + blizzardGateway.fingerprints.set(keyId(fingerprintKey), fingerprint); + const publish = vi.spyOn( + repositories.snapshots, + "createAndFinishFingerprintSweep" + ); + + await handlerFor(repositories, new MutableGateway(), { + blizzardGateway + }).execute(run.id, delivery()); + + expect(publish).toHaveBeenCalledOnce(); + await expect( + repositories.snapshots.getCurrent(rootKey) + ).resolves.toMatchObject({ + state: "complete", + limitationCode: null, + characterCount: 4, + characters: expect.arrayContaining([ + expect.objectContaining({ key: fingerprintKey, source: "fingerprint" }), + expect.objectContaining({ key: secondKey, source: "claimed" }) + ]) + }); + expect(repositories.fingerprintSweeps.recordRequest).toHaveBeenCalledTimes( + 5 + ); + expect(publish).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + reservationId: "reservation-1", + limitationCode: null + }), + expect.any(Object) + ); + }); + + it("publishes a cap-bounded partial result", async () => { + // Break caught: exhausting the reserved cap could publish a complete result + // or retry and discard the permitted partial snapshot. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = async () => ({ + kind: "admitted", + reservationId: "reservation-capped", + requestCap: 2 + }); + repositories.fingerprintSweeps.recordRequest = vi.fn(async () => {}); + const publish = vi.spyOn( + repositories.snapshots, + "createAndFinishFingerprintSweep" + ); + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.roster = [ + { + key: fingerprintKey, + displayName: "Fingerprint Match", + className: "Priest", + level: 80 + } + ]; + blizzardGateway.fingerprints.set(rootKey.name, achievementFingerprint()); + + await handlerFor(repositories, new MutableGateway(), { + blizzardGateway + }).execute(run.id, delivery()); + + await expect( + repositories.snapshots.getCurrent(rootKey) + ).resolves.toMatchObject({ + state: "partial", + limitationCode: "fingerprint_sweep_capped", + characterCount: 3 + }); + expect(repositories.fingerprintSweeps.recordRequest).toHaveBeenCalledTimes( + 2 + ); + expect(publish).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + reservationId: "reservation-capped", + limitationCode: "fingerprint_sweep_capped" + }), + expect.any(Object) + ); + }); + + it("releases a failed fingerprint reservation and retries without publication", async () => { + // Break caught: a Blizzard failure could expose a half-merged snapshot or + // retain unused reserved capacity across the retry. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = async () => ({ + kind: "admitted", + reservationId: "reservation-failed", + requestCap: 300 + }); + const events: string[] = []; + repositories.fingerprintSweeps.recordRequest = vi.fn(async () => { + events.push("accounted"); + }); + repositories.fingerprintSweeps.release = vi.fn(async () => {}); + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.getGuildRoster = async ( + _key, + _signal, + onProfileRequest + ) => { + await onProfileRequest?.(); + events.push("upstream"); + throw Object.assign(new Error("private-upstream-marker"), { + kind: "transient", + retryAfterMs: 30_000 + }); + }; + + await expect( + handlerFor(repositories, new MutableGateway(), { + blizzardGateway + }).execute(run.id, delivery()) + ).rejects.toMatchObject({ retryable: true, retryAfterMs: 30_000 }); + + expect(repositories.fingerprintSweeps.recordRequest).toHaveBeenCalledOnce(); + expect(events).toEqual(["accounted", "upstream"]); + expect(repositories.fingerprintSweeps.release).toHaveBeenCalledWith( + "reservation-failed", + expect.any(Date) + ); + await expect( + repositories.snapshots.getCurrent(rootKey) + ).resolves.toBeNull(); + }); + + it("keeps release retryable when the first release write fails", async () => { + // Break caught: a transient release failure could be treated as settled and + // strand the reservation for its whole accounting window. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = async () => ({ + kind: "admitted", + reservationId: "reservation-release-retry", + requestCap: 300 + }); + repositories.fingerprintSweeps.recordRequest = async () => {}; + let releases = 0; + repositories.fingerprintSweeps.release = async () => { + releases += 1; + if (releases === 1) throw new Error("release_write_failed"); + }; + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.getGuildRoster = async () => { + throw Object.assign(new Error("transient"), { kind: "transient" }); + }; + + await expect( + handlerFor(repositories, new MutableGateway(), { + blizzardGateway + }).execute(run.id, delivery()) + ).rejects.toMatchObject({ retryable: true }); + + expect(releases).toBe(2); + await expect(repositories.runs.find(run.id)).resolves.toMatchObject({ + status: "retrying" + }); + }); + + it("releases an aborted fingerprint reservation without publishing or reconciling", async () => { + // Break caught: worker shutdown could leak a reservation or persist the + // transient Raider.IO half of an abandoned atomic sweep. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = async () => ({ + kind: "admitted", + reservationId: "reservation-aborted", + requestCap: 300 + }); + repositories.fingerprintSweeps.recordRequest = vi.fn(async () => {}); + repositories.fingerprintSweeps.release = vi.fn(async () => {}); + const controller = new AbortController(); + const abortReason = new DOMException("drain timeout", "AbortError"); + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.getGuildRoster = async ( + _key, + _signal, + onProfileRequest + ) => { + await onProfileRequest?.(); + controller.abort(abortReason); + return []; + }; + + await expect( + handlerFor(repositories, new MutableGateway(), { + blizzardGateway + }).execute(run.id, { ...delivery(), signal: controller.signal }) + ).rejects.toBe(abortReason); + + expect(repositories.fingerprintSweeps.recordRequest).toHaveBeenCalledOnce(); + expect(repositories.fingerprintSweeps.release).toHaveBeenCalledWith( + "reservation-aborted", + expect.any(Date) + ); + await expect(repositories.runs.find(run.id)).resolves.toMatchObject({ + status: "running", + snapshotId: null, + errorCode: null + }); + }); + + it("retries an aborted delivery when reservation release cannot be persisted", async () => { + // Break caught: cancellation could hide a failed release and retain a full + // reservation until expiry with no durable path to retry the cleanup. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = async () => ({ + kind: "admitted", + reservationId: "reservation-abort-release-failure", + requestCap: 300 + }); + repositories.fingerprintSweeps.recordRequest = async () => {}; + repositories.fingerprintSweeps.release = async () => { + throw new Error("release_write_failed"); + }; + const controller = new AbortController(); + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.getGuildRoster = async () => { + controller.abort(new DOMException("drain timeout", "AbortError")); + return []; + }; + + await expect( + handlerFor(repositories, new MutableGateway(), { + blizzardGateway + }).execute(run.id, { ...delivery(), signal: controller.signal }) + ).rejects.toMatchObject({ retryable: true }); + + await expect(repositories.runs.find(run.id)).resolves.toMatchObject({ + status: "retrying" + }); + }); + + it("never starts a fingerprint sweep from privacy-hidden root ownership", async () => { + // Break caught: a root whose Raider.IO ownership is intentionally hidden + // could seed inferred links despite the project's sole privacy signal. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = vi.fn(async () => ({ + kind: "admitted" as const, + reservationId: "privacy-reservation", + requestCap: 300 + })); + const gateway = new MutableGateway(); + gateway.getCharacter = async () => ({ + ...character(rootKey), + ownerId: null + }); + gateway.resolveProfileGuess = async () => null; + const blizzardGateway = new MutableBlizzardGateway(); + blizzardGateway.getGuildRoster = vi.fn( + blizzardGateway.getGuildRoster.bind(blizzardGateway) + ); + + await handlerFor(repositories, gateway, { blizzardGateway }).execute( + run.id, + delivery() + ); + + expect( + repositories.fingerprintSweeps.requestAdmission + ).not.toHaveBeenCalled(); + expect(blizzardGateway.getGuildRoster).not.toHaveBeenCalled(); + await expect( + repositories.snapshots.getCurrent(rootKey) + ).resolves.toMatchObject({ + state: "partial", + limitationCode: "privacy_hidden" + }); + }); + + it("never starts a fingerprint sweep when request capping masks hidden root ownership", async () => { + // Break caught: request_cap can take precedence over privacy_hidden while + // preserving the same privacy fact that must bar fingerprint inference. + const repositories = createMemoryRepositories(); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + repositories.fingerprintSweeps.requestAdmission = vi.fn(async () => ({ + kind: "admitted" as const, + reservationId: "capped-privacy-reservation", + requestCap: 300 + })); + const gateway = new MutableGateway(); + gateway.getCharacter = async () => ({ + ...character(rootKey), + ownerId: null, + profileGuess: "private-alias" + }); + gateway.resolveProfileGuess = async () => null; + + await handlerFor(repositories, gateway, { requestCap: 1 }).execute( + run.id, + delivery() + ); + + expect( + repositories.fingerprintSweeps.requestAdmission + ).not.toHaveBeenCalled(); + await expect( + repositories.snapshots.getCurrent(rootKey) + ).resolves.toMatchObject({ limitationCode: "request_cap" }); + }); + it("emits one allowlisted operational record per completed discovery", async () => { // Break caught: a production discovery could succeed or fail with nothing // operable in the logs, or could log private lookup values while becoming visible. @@ -331,7 +793,11 @@ describe("discovery job handler", () => { state: "complete", limitationCode: null, characterCount: 3, - durationMs: 0 + durationMs: 0, + fingerprintQueueWaitMs: null, + fingerprintReservedRequests: 0, + fingerprintUsedRequests: 0, + fingerprintDurationMs: 0 } ]); }); @@ -367,7 +833,11 @@ describe("discovery job handler", () => { state: null, limitationCode: null, characterCount: 0, - durationMs: 0 + durationMs: 0, + fingerprintQueueWaitMs: null, + fingerprintReservedRequests: 0, + fingerprintUsedRequests: 0, + fingerprintDurationMs: 0 } ]); expect(JSON.stringify(events)).not.toContain(marker); diff --git a/packages/application/src/discovery-job-handler.ts b/packages/application/src/discovery-job-handler.ts index 775c332..0ef9e55 100644 --- a/packages/application/src/discovery-job-handler.ts +++ b/packages/application/src/discovery-job-handler.ts @@ -1,13 +1,44 @@ import type { DiscoveryWorkContext, Repositories } from "@slashwho/database"; -import { discoverCharacter, type RaiderIoGateway } from "@slashwho/domain"; +import type { BlizzardGateway } from "@slashwho/blizzard"; +import { + deduplicateCharacters, + discoverCharacter, + discoverFingerprintMatches, + type DiscoveryOutcome, + type RaiderIoGateway +} from "@slashwho/domain"; + +import { createBlizzardFingerprintAdapter } from "./blizzard-fingerprint-adapter"; export type DiscoveryLogger = { info(value: Record): void; }; +/** Delivery seam for a maintainer-owned alert integration (PagerDuty, email, etc.). */ +export type FingerprintAlertNotifier = { + notify(alert: { + event: + | "fingerprint_admission_blocked" + | "fingerprint_reservation_pressure" + | "fingerprint_blizzard_rate_limited"; + details: Record; + }): Promise | void; +}; + export type DiscoveryJobHandlerOptions = { repositories: Repositories; gateway: RaiderIoGateway; + /** Optional only until worker credential composition lands in Task 6. */ + blizzardGateway?: BlizzardGateway; + /** Optional only until worker credential composition lands in Task 6. */ + fingerprint?: { + requestCap: number; + hourlyBudget: number; + cadenceMs: number; + minimumCommon: number; + minimumIdenticalPercent: number; + }; + enqueueFingerprintAdmission?: (runId: string) => Promise; requestCap: number; now?: () => Date; random?: () => number; @@ -17,6 +48,7 @@ export type DiscoveryJobHandlerOptions = { maxAttempts?: number; negativeCacheTtlMs?: number; logger?: DiscoveryLogger; + fingerprintAlertNotifier?: FingerprintAlertNotifier; monotonic?: () => number; }; @@ -37,6 +69,10 @@ type DiscoveryRunRecord = { limitationCode: string | null; characterCount: number; durationMs: number; + fingerprintQueueWaitMs: number | null; + fingerprintReservedRequests: number; + fingerprintUsedRequests: number; + fingerprintDurationMs: number; }; export type RetryableDiscoveryError = Error & { @@ -44,6 +80,10 @@ export type RetryableDiscoveryError = Error & { retryAfterMs: number; }; +type FingerprintReleaseRetryableError = Error & { + fingerprintReleaseRetryable: true; +}; + function retryableError(retryAfterMs: number): RetryableDiscoveryError { return Object.assign(new Error("discovery_retryable"), { retryable: true as const, @@ -63,6 +103,24 @@ function isRetryableDiscoveryError( ); } +function fingerprintReleaseRetryableError( + cause: unknown +): FingerprintReleaseRetryableError { + return Object.assign(new Error("fingerprint_release_failed", { cause }), { + fingerprintReleaseRetryable: true as const + }); +} + +function isFingerprintReleaseRetryableError( + error: unknown +): error is FingerprintReleaseRetryableError { + return ( + error instanceof Error && + "fingerprintReleaseRetryable" in error && + error.fingerprintReleaseRetryable === true + ); +} + export function createDiscoveryJobHandler(options: DiscoveryJobHandlerOptions) { const now = options.now ?? (() => new Date()); const random = options.random ?? Math.random; @@ -136,7 +194,11 @@ export function createDiscoveryJobHandler(options: DiscoveryJobHandlerOptions) { state: null, limitationCode: null, characterCount: 0, - durationMs: 0 + durationMs: 0, + fingerprintQueueWaitMs: null, + fingerprintReservedRequests: 0, + fingerprintUsedRequests: 0, + fingerprintDurationMs: 0 }; try { @@ -151,12 +213,16 @@ export function createDiscoveryJobHandler(options: DiscoveryJobHandlerOptions) { return; } - const outcome = await discoverCharacter(run.rootKey, options.gateway, { - requestCap: options.requestCap, - isSuppressed: (key) => - options.repositories.suppressions.isActive(key), - signal: context.signal - }); + let outcome: DiscoveryOutcome = await discoverCharacter( + run.rootKey, + options.gateway, + { + requestCap: options.requestCap, + isSuppressed: (key) => + options.repositories.suppressions.isActive(key), + signal: context.signal + } + ); context.signal.throwIfAborted(); const persistenceTime = now(); if ( @@ -169,25 +235,220 @@ export function createDiscoveryJobHandler(options: DiscoveryJobHandlerOptions) { } if (outcome.kind === "snapshot") { - context.signal.throwIfAborted(); - record.outcome = "snapshot"; - record.state = outcome.state; - record.limitationCode = - outcome.state === "partial" ? outcome.limitationCode : null; - record.characterCount = outcome.characters.length; - await options.repositories.snapshots.create( - { - runId, - rootKey: run.rootKey, - state: outcome.state, - limitationCode: - outcome.state === "partial" ? outcome.limitationCode : null, - refreshedAt: persistenceTime, - characters: [...outcome.characters] - }, - { signal: context.signal } - ); - return; + let fingerprintFailure: + Extract | undefined; + const fingerprint = options.fingerprint; + const blizzardGateway = options.blizzardGateway; + const privacyHiddenRoot = + outcome.state === "partial" && + (outcome.limitationCode === "privacy_hidden" || + outcome.privacyHiddenObserved === true); + if (fingerprint && blizzardGateway && !privacyHiddenRoot) { + const admissionTime = now(); + const admission = + await options.repositories.fingerprintSweeps.requestAdmission({ + runId, + key: run.rootKey, + requestCap: fingerprint.requestCap, + hourlyBudget: fingerprint.hourlyBudget, + cadenceCutoff: new Date( + admissionTime.getTime() - fingerprint.cadenceMs + ), + at: admissionTime + }); + + if (admission.kind === "waiting") { + if (!options.enqueueFingerprintAdmission) { + throw new Error("fingerprint_admission_queue_unavailable"); + } + await options.enqueueFingerprintAdmission(runId); + record.outcome = "fingerprint_admission_waiting"; + record.fingerprintQueueWaitMs = Math.max( + 0, + admission.retryAt.getTime() - admissionTime.getTime() + ); + const blockedForMs = admission.blockedSince + ? Math.max( + 0, + admissionTime.getTime() - admission.blockedSince.getTime() + ) + : 0; + if (blockedForMs >= 15 * 60_000) { + options.logger?.info({ + event: "fingerprint_admission_blocked", + blockedForMs + }); + await options.fingerprintAlertNotifier?.notify({ + event: "fingerprint_admission_blocked", + details: { blockedForMs } + }); + } + return; + } + + if (admission.kind === "admitted") { + const fingerprintStartedAt = monotonic(); + let reservationActive = true; + record.fingerprintReservedRequests = admission.requestCap; + if ( + admission.committedRequests !== undefined && + admission.hourlyBudget !== undefined && + admission.committedRequests > admission.hourlyBudget * 0.9 + ) { + options.logger?.info({ + event: "fingerprint_reservation_pressure", + committedRequests: admission.committedRequests, + hourlyBudget: admission.hourlyBudget + }); + await options.fingerprintAlertNotifier?.notify({ + event: "fingerprint_reservation_pressure", + details: { + committedRequests: admission.committedRequests, + hourlyBudget: admission.hourlyBudget + } + }); + } + const releaseReservation = async () => { + if (!reservationActive) return; + await options.repositories.fingerprintSweeps.release( + admission.reservationId, + now() + ); + reservationActive = false; + }; + try { + const adaptedGateway = createBlizzardFingerprintAdapter( + blizzardGateway, + { + requestCap: admission.requestCap, + recordRequest: async () => { + await options.repositories.fingerprintSweeps.recordRequest( + admission.reservationId, + 1, + now() + ); + record.fingerprintUsedRequests += 1; + }, + onRateLimited: async () => { + options.logger?.info({ + event: "fingerprint_blizzard_rate_limited" + }); + await options.fingerprintAlertNotifier?.notify({ + event: "fingerprint_blizzard_rate_limited", + details: {} + }); + } + } + ); + const sweep = await discoverFingerprintMatches( + run.rootKey, + adaptedGateway, + { + requestCap: Number.MAX_SAFE_INTEGER, + minimumCommon: fingerprint.minimumCommon, + minimumIdenticalPercent: + fingerprint.minimumIdenticalPercent, + isSuppressed: (key) => + options.repositories.suppressions.isActive(key), + isPrivacyHidden: async (key) => + (await options.gateway.getCharacter(key, context.signal)) + .ownerId === null, + signal: context.signal + } + ); + + if (sweep.kind === "failure") { + await releaseReservation(); + fingerprintFailure = sweep; + } else { + context.signal.throwIfAborted(); + const fingerprintPersistenceTime = now(); + if ( + fingerprintPersistenceTime.getTime() - + run.createdAt.getTime() >= + maxJobLifetimeMs + ) { + record.outcome = "lifetime_exceeded"; + await releaseReservation(); + await options.repositories.runs.fail( + runId, + "upstream_unavailable" + ); + return; + } + const limitationCode = + sweep.kind === "capped" + ? "fingerprint_sweep_capped" + : outcome.state === "partial" + ? outcome.limitationCode + : null; + const characters = deduplicateCharacters([ + ...outcome.characters, + ...sweep.characters + ]); + record.outcome = "snapshot"; + record.state = + limitationCode === null ? "complete" : "partial"; + record.limitationCode = limitationCode; + record.characterCount = characters.length; + await options.repositories.snapshots.createAndFinishFingerprintSweep( + { + runId, + rootKey: run.rootKey, + state: limitationCode === null ? "complete" : "partial", + limitationCode, + refreshedAt: fingerprintPersistenceTime, + characters + }, + { + reservationId: admission.reservationId, + finishedAt: now(), + limitationCode + }, + { signal: context.signal } + ); + reservationActive = false; + return; + } + } catch (error) { + try { + await releaseReservation(); + } catch (releaseError) { + throw fingerprintReleaseRetryableError(releaseError); + } + throw error; + } finally { + record.fingerprintDurationMs = Math.max( + 0, + monotonic() - fingerprintStartedAt + ); + } + } + } + + if (fingerprintFailure) { + outcome = fingerprintFailure; + } else { + context.signal.throwIfAborted(); + record.outcome = "snapshot"; + record.state = outcome.state; + record.limitationCode = + outcome.state === "partial" ? outcome.limitationCode : null; + record.characterCount = outcome.characters.length; + await options.repositories.snapshots.create( + { + runId, + rootKey: run.rootKey, + state: outcome.state, + limitationCode: + outcome.state === "partial" ? outcome.limitationCode : null, + refreshedAt: persistenceTime, + characters: [...outcome.characters] + }, + { signal: context.signal } + ); + return; + } } if (!outcome.retryable) { @@ -232,7 +493,10 @@ export function createDiscoveryJobHandler(options: DiscoveryJobHandlerOptions) { ); throw retryableError(schedule.retryAfterMs); } catch (error) { - if (context.signal.aborted) { + if ( + context.signal.aborted && + !isFingerprintReleaseRetryableError(error) + ) { record.outcome = "cancelled"; throw context.signal.reason; } diff --git a/packages/application/src/index.ts b/packages/application/src/index.ts index 8ac6c42..29db23c 100644 --- a/packages/application/src/index.ts +++ b/packages/application/src/index.ts @@ -1,8 +1,10 @@ export { createDiscoveryJobHandler } from "./discovery-job-handler"; +export { createBlizzardFingerprintAdapter } from "./blizzard-fingerprint-adapter"; export type { DiscoveryJobHandler, DiscoveryJobHandlerOptions, DiscoveryLogger, + FingerprintAlertNotifier, RetryableDiscoveryError } from "./discovery-job-handler"; export { diff --git a/packages/application/src/search-service.test.ts b/packages/application/src/search-service.test.ts index 0bdaa4c..84bce6b 100644 --- a/packages/application/src/search-service.test.ts +++ b/packages/application/src/search-service.test.ts @@ -142,6 +142,9 @@ function policyFixture( async create() { throw new Error("not used"); }, + async createAndFinishFingerprintSweep() { + throw new Error("not used"); + }, async getCurrent() { return options.current ?? null; }, @@ -184,6 +187,27 @@ function policyFixture( async cleanupExpired() { return 0; } + }, + fingerprintSweeps: { + async requestAdmission() { + return { kind: "not_due" }; + }, + async recordRequest() {}, + async finish() {}, + async release() {}, + async listWaiting() { + return []; + }, + async listAdmittedUndispatched() { + return []; + }, + async markDispatched() {}, + async admitWaiting() { + return { kind: "settled" }; + }, + async cleanupExpired() { + return 0; + } } } satisfies Repositories; diff --git a/packages/application/src/search-service.ts b/packages/application/src/search-service.ts index 77b252a..fd594ea 100644 --- a/packages/application/src/search-service.ts +++ b/packages/application/src/search-service.ts @@ -85,27 +85,28 @@ export interface SearchService { key: CharacterKey, snapshotId: string ): Promise; - cleanupExpired(now?: Date): Promise<{ - rateLimits: number; - negativeCache: number; - suppressions: number; - }>; + cleanupExpired(now?: Date): Promise; } -export async function cleanupExpired( - repositories: Repositories, - at: Date = new Date() -): Promise<{ +export type CleanupCounts = { rateLimits: number; negativeCache: number; suppressions: number; -}> { - const [rateLimits, negativeCache, suppressions] = await Promise.all([ - repositories.rateLimits.cleanupExpired(at), - repositories.negativeCache.cleanupExpired(at), - repositories.suppressions.cleanupExpired(at) - ]); - return { rateLimits, negativeCache, suppressions }; + fingerprintRequests: number; +}; + +export async function cleanupExpired( + repositories: Repositories, + at: Date = new Date() +): Promise { + const [rateLimits, negativeCache, suppressions, fingerprintRequests] = + await Promise.all([ + repositories.rateLimits.cleanupExpired(at), + repositories.negativeCache.cleanupExpired(at), + repositories.suppressions.cleanupExpired(at), + repositories.fingerprintSweeps.cleanupExpired(at) + ]); + return { rateLimits, negativeCache, suppressions, fingerprintRequests }; } export async function recoverPendingSearches( diff --git a/packages/application/src/serializers.test.ts b/packages/application/src/serializers.test.ts index 6297227..9accee7 100644 --- a/packages/application/src/serializers.test.ts +++ b/packages/application/src/serializers.test.ts @@ -77,6 +77,25 @@ describe("public serializers", () => { ); }); + it("never exposes fingerprint source, score, queue, or reservation fields", () => { + // Break caught: private sweep evidence could turn a public alt list into a + // disclosure of how its links were discovered. + const fingerprintSnapshot = { + ...snapshot, + characters: snapshot.characters.map((character, index) => ({ + ...character, + source: index === 0 ? "input" : "fingerprint" + })) + } as StoredSnapshot; + + const resource = serializeCharacterResource(fingerprintSnapshot, run); + + expect(resource).not.toHaveProperty("discoverySource"); + expect(JSON.stringify(resource)).not.toMatch( + /fingerprint|score|reservation|queue/i + ); + }); + it("returns only safe lifecycle fields for job status", () => { // Break caught: caller class, attempts, queue IDs, or root persistence fields could leak. const resource = serializeJobStatus({ diff --git a/packages/blizzard/package.json b/packages/blizzard/package.json new file mode 100644 index 0000000..8676f31 --- /dev/null +++ b/packages/blizzard/package.json @@ -0,0 +1,13 @@ +{ + "name": "@slashwho/blizzard", + "private": true, + "exports": "./src/index.ts", + "scripts": { + "build": "tsc --noEmit", + "test": "vitest run --root ../.. packages/blizzard/src", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@slashwho/domain": "workspace:*" + } +} diff --git a/packages/blizzard/src/client.test.ts b/packages/blizzard/src/client.test.ts new file mode 100644 index 0000000..69cdec2 --- /dev/null +++ b/packages/blizzard/src/client.test.ts @@ -0,0 +1,205 @@ +import type { CharacterKey } from "@slashwho/domain"; +import { describe, expect, it, vi } from "vitest"; + +import { createBlizzardClient } from "./index"; + +const key: CharacterKey = { + region: "eu", + realm: "silvermoon", + name: "sentinel" +}; + +function clientFor( + responder: (url: URL, init?: RequestInit) => Response | Promise +) { + const fetchSpy = vi.fn( + async (input: RequestInfo | URL, init?: RequestInit): Promise => + responder( + new URL( + typeof input === "string" || input instanceof URL ? input : input.url + ), + init + ) + ); + return { + fetchSpy, + gateway: createBlizzardClient({ + fetch: fetchSpy as unknown as typeof globalThis.fetch, + clientId: "id", + clientSecret: "secret" + }) + }; +} + +function tokenResponse(): Response { + return Response.json({ + access_token: "private-access-token", + expires_in: 3600 + }); +} + +describe("Blizzard gateway", () => { + it("uses an explicitly configured endpoint for local integration fixtures", async () => { + // Break caught: e2e sweeps could send test credentials to the public + // Blizzard endpoints even when the test suite provides a local fixture. + const endpoints: string[] = []; + const gateway = createBlizzardClient({ + fetch: (async (input: RequestInfo | URL) => { + const url = new URL(String(input)); + endpoints.push(url.toString()); + return url.pathname === "/token" + ? tokenResponse() + : Response.json({ achievements: [] }); + }) as typeof globalThis.fetch, + clientId: "id", + clientSecret: "secret", + baseUrl: "http://127.0.0.1:43101" + }); + + await expect(gateway.getAchievementFingerprint(key)).resolves.toEqual( + new Map() + ); + expect(endpoints).toEqual([ + "http://127.0.0.1:43101/token", + "http://127.0.0.1:43101/profile/wow/character/silvermoon/sentinel/achievements?namespace=profile-eu&locale=en_GB" + ]); + }); + + it("uses the root region profile API and normalizes the current guild roster", async () => { + // Break caught: roster requests could cross regions or leak upstream member + // shapes into discovery snapshots. + const { gateway } = clientFor((url) => { + if (url.hostname === "oauth.battle.net") return tokenResponse(); + if (url.pathname.endsWith("/character/silvermoon/sentinel")) { + return Response.json({ + guild: { name: "A Guild", realm: { slug: "silvermoon" } } + }); + } + if (url.pathname.endsWith("/guild/silvermoon/a-guild/roster")) { + return Response.json({ + members: [ + { + character: { + name: "Alt", + realm: { slug: "Silvermoon" }, + playable_class: { name: "Mage" }, + level: 80 + } + } + ] + }); + } + throw new Error(`unexpected endpoint: ${url.pathname}`); + }); + + const onProfileRequest = vi.fn(); + await expect( + gateway.getGuildRoster(key, undefined, onProfileRequest) + ).resolves.toEqual([ + { + key: { region: "eu", realm: "silvermoon", name: "alt" }, + displayName: "Alt", + className: "Mage", + level: 80 + } + ]); + expect(onProfileRequest).toHaveBeenCalledTimes(2); + }); + + it("returns an empty roster when the root has no guild", async () => { + const { gateway } = clientFor((url) => { + if (url.hostname === "oauth.battle.net") return tokenResponse(); + return Response.json({}); + }); + + await expect(gateway.getGuildRoster(key)).resolves.toEqual([]); + }); + + it("extracts only numeric achievement pairs and caches the process token", async () => { + // Break caught: malformed achievement entries could reach comparison, or a + // token request could be made per character. + const { fetchSpy, gateway } = clientFor((url) => { + if (url.hostname === "oauth.battle.net") return tokenResponse(); + return Response.json({ + achievements: [ + { id: 1, completed_timestamp: 100 }, + { id: "2", completed_timestamp: 200 }, + { id: 3, completed_timestamp: "300" } + ] + }); + }); + + await expect(gateway.getAchievementFingerprint(key)).resolves.toEqual( + new Map([[1, 100]]) + ); + await expect(gateway.getAchievementFingerprint(key)).resolves.toEqual( + new Map([[1, 100]]) + ); + expect( + fetchSpy.mock.calls.filter( + ([input]) => new URL(String(input)).hostname === "oauth.battle.net" + ) + ).toHaveLength(1); + }); + + it("passes the abort signal and never includes an upstream body in its error", async () => { + // Break caught: cancellation could be omitted, or an upstream error body + // could enter a typed failure and be logged later. + const controller = new AbortController(); + const { fetchSpy, gateway } = clientFor((url) => { + if (url.hostname === "oauth.battle.net") return tokenResponse(); + return new Response("upstream-private-body-marker", { + status: 429, + headers: { "Retry-After": "60" } + }); + }); + + const request = gateway.getAchievementFingerprint(key, controller.signal); + await expect(request).rejects.toMatchObject({ + kind: "transient", + retryAfterMs: 60_000 + }); + await expect(request).rejects.not.toThrow(/upstream-private-body-marker/); + expect( + JSON.stringify(await request.catch((error: unknown) => error)) + ).not.toContain("upstream-private-body-marker"); + expect(fetchSpy).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ signal: controller.signal }) + ); + }); + + it("classifies unexpected success payloads as schema drift", async () => { + const { gateway } = clientFor((url) => { + if (url.hostname === "oauth.battle.net") return tokenResponse(); + return Response.json({ unexpected: true }); + }); + + await expect(gateway.getAchievementFingerprint(key)).rejects.toMatchObject({ + kind: "schema_drift" + }); + }); + + it("classifies missing Blizzard resources without exposing their body", async () => { + const { gateway } = clientFor((url) => { + if (url.hostname === "oauth.battle.net") return tokenResponse(); + return new Response("missing-private-body-marker", { status: 404 }); + }); + + const request = gateway.getAchievementFingerprint(key); + await expect(request).rejects.toMatchObject({ kind: "not_found" }); + await expect(request).rejects.not.toThrow(/missing-private-body-marker/); + }); + + it("rejects regions outside the supported same-region profile boundary", async () => { + // Break caught: a forged key could send fingerprint data to the unsupported + // China API rather than keeping every request in the domain's region set. + const { fetchSpy, gateway } = clientFor(() => tokenResponse()); + const unsupportedKey = { ...key, region: "cn" } as unknown as CharacterKey; + + await expect( + gateway.getAchievementFingerprint(unsupportedKey) + ).rejects.toThrow("invalid_character_key"); + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/blizzard/src/client.ts b/packages/blizzard/src/client.ts new file mode 100644 index 0000000..b509355 --- /dev/null +++ b/packages/blizzard/src/client.ts @@ -0,0 +1,315 @@ +import { supportedRegions, type CharacterKey } from "@slashwho/domain"; + +import type { + AchievementFingerprint, + BlizzardError, + BlizzardFailure, + BlizzardGateway, + BlizzardProfileRequestObserver, + BlizzardRosterCharacter +} from "./types"; + +export type CreateBlizzardClientOptions = Readonly<{ + fetch: typeof globalThis.fetch; + clientId: string; + clientSecret: string; + /** Overrides both Blizzard hosts for deterministic local integration tests. */ + baseUrl?: string; +}>; + +type AccessToken = Readonly<{ + value: string; + expiresAt: number; +}>; + +function createBlizzardError(failure: BlizzardFailure): BlizzardError { + return Object.assign( + new Error(`blizzard_${failure.kind}`), + failure + ) as BlizzardError; +} + +function retryAfterMs(response: Response): number | undefined { + const value = response.headers.get("Retry-After")?.trim(); + if (!value) return undefined; + + if (/^\d+$/.test(value)) return Number(value) * 1_000; + + const retryAt = Date.parse(value); + return Number.isFinite(retryAt) + ? Math.max(0, retryAt - Date.now()) + : undefined; +} + +function responseFailure(response: Response): BlizzardFailure { + if (response.status === 404) return { kind: "not_found" }; + + const retryAfter = retryAfterMs(response); + return { + kind: "transient", + status: response.status, + ...(retryAfter === undefined ? {} : { retryAfterMs: retryAfter }) + }; +} + +function validCharacterKey(value: CharacterKey): CharacterKey { + const valid = + supportedRegions.includes(value.region) && + /^[a-z0-9-]+$/.test(value.realm) && + /^[\p{L}\p{M}'-]+$/u.test(value.name) && + value.realm === value.realm.toLocaleLowerCase("en-US") && + value.name === value.name.toLocaleLowerCase("en-US"); + if (!valid) throw new Error("invalid_character_key"); + return value; +} + +function valueRecord(value: unknown): Record | null { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function nonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function finiteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function normalizedRosterCharacter( + value: unknown, + region: CharacterKey["region"] +): BlizzardRosterCharacter | null { + const member = valueRecord(value); + const character = member && valueRecord(member.character); + const realm = character && valueRecord(character.realm); + const playableClass = character && valueRecord(character.playable_class); + const displayName = character && nonEmptyString(character.name); + const realmSlug = realm && nonEmptyString(realm.slug); + const className = playableClass && nonEmptyString(playableClass.name); + const level = character && finiteNumber(character.level); + if ( + !displayName || + !realmSlug || + !className || + level === null || + !Number.isInteger(level) || + level < 0 + ) { + return null; + } + + const key = { + region, + realm: realmSlug.toLocaleLowerCase("en-US"), + name: displayName.toLocaleLowerCase("en-US") + } as CharacterKey; + try { + validCharacterKey(key); + } catch { + return null; + } + + return { key, displayName, className, level }; +} + +function fingerprintFromResponse( + value: unknown +): AchievementFingerprint | null { + const response = valueRecord(value); + if (!response || !Array.isArray(response.achievements)) return null; + + const fingerprint = new Map(); + for (const achievement of response.achievements) { + const entry = valueRecord(achievement); + const id = entry && finiteNumber(entry.id); + const timestamp = entry && finiteNumber(entry.completed_timestamp); + if (id !== null && timestamp !== null) fingerprint.set(id, timestamp); + } + return fingerprint; +} + +function blizzardSlug(value: string): string { + return value.trim().toLocaleLowerCase("en-US").replace(/\s+/g, "-"); +} + +export function createBlizzardClient( + options: CreateBlizzardClientOptions +): BlizzardGateway { + let cachedToken: AccessToken | undefined; + + async function accessToken(signal?: AbortSignal): Promise { + if (cachedToken && cachedToken.expiresAt > Date.now()) { + return cachedToken.value; + } + + let response: Response; + try { + response = await options.fetch( + new URL( + "/token", + options.baseUrl ?? "https://oauth.battle.net" + ).toString(), + { + method: "POST", + headers: { + "Content-Type": "application/x-www-form-urlencoded", + Authorization: `Basic ${Buffer.from( + `${options.clientId}:${options.clientSecret}` + ).toString("base64")}` + }, + body: "grant_type=client_credentials", + signal + } + ); + } catch { + if (signal?.aborted) throw signal.reason; + throw createBlizzardError({ kind: "transient" }); + } + + signal?.throwIfAborted(); + if (!response.ok) throw createBlizzardError(responseFailure(response)); + + try { + const body = valueRecord(await response.json()); + signal?.throwIfAborted(); + const token = body && nonEmptyString(body.access_token); + const expiresIn = body && finiteNumber(body.expires_in); + if (!token || expiresIn === null || expiresIn <= 0) { + throw new Error("invalid_token_response"); + } + cachedToken = { + value: token, + expiresAt: Date.now() + Math.max(0, expiresIn * 1_000 - 60_000) + }; + return token; + } catch { + if (signal?.aborted) throw signal.reason; + throw createBlizzardError({ kind: "schema_drift" }); + } + } + + async function request( + url: URL, + normalize: (value: unknown) => T | null, + signal?: AbortSignal, + onProfileRequest?: BlizzardProfileRequestObserver + ): Promise { + const token = await accessToken(signal); + await onProfileRequest?.(); + signal?.throwIfAborted(); + let response: Response; + try { + response = await options.fetch(url.toString(), { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/json" + }, + signal + }); + } catch { + if (signal?.aborted) throw signal.reason; + throw createBlizzardError({ kind: "transient" }); + } + + signal?.throwIfAborted(); + if (!response.ok) throw createBlizzardError(responseFailure(response)); + + try { + const normalized = normalize(await response.json()); + signal?.throwIfAborted(); + if (normalized === null) throw new Error("invalid_response"); + return normalized; + } catch { + if (signal?.aborted) throw signal.reason; + throw createBlizzardError({ kind: "schema_drift" }); + } + } + + function profileUrl(key: CharacterKey): URL { + const url = new URL( + `/profile/wow/character/${encodeURIComponent(key.realm)}/${encodeURIComponent(key.name)}`, + options.baseUrl ?? `https://${key.region}.api.blizzard.com` + ); + url.searchParams.set("namespace", `profile-${key.region}`); + url.searchParams.set("locale", "en_GB"); + return url; + } + + function achievementsUrl(key: CharacterKey): URL { + const url = profileUrl(key); + url.pathname = `${url.pathname}/achievements`; + return url; + } + + function rosterUrl( + region: CharacterKey["region"], + realm: string, + guildName: string + ): URL { + const url = new URL( + `/data/wow/guild/${encodeURIComponent(blizzardSlug(realm))}/${encodeURIComponent(blizzardSlug(guildName))}/roster`, + options.baseUrl ?? `https://${region}.api.blizzard.com` + ); + url.searchParams.set("namespace", `profile-${region}`); + url.searchParams.set("locale", "en_GB"); + return url; + } + + async function getGuildRoster( + root: CharacterKey, + signal?: AbortSignal, + onProfileRequest?: BlizzardProfileRequestObserver + ): Promise { + const key = validCharacterKey(root); + const profile = await request( + profileUrl(key), + (value) => valueRecord(value), + signal, + onProfileRequest + ); + if (!("guild" in profile) || profile.guild === null) return []; + + const guild = valueRecord(profile.guild); + const name = guild && nonEmptyString(guild.name); + const realm = guild && valueRecord(guild.realm); + const realmSlug = realm && nonEmptyString(realm.slug); + if (!name || !realmSlug) + throw createBlizzardError({ kind: "schema_drift" }); + + return request( + rosterUrl(key.region, realmSlug, name), + (value) => { + const roster = valueRecord(value); + if (!roster || !Array.isArray(roster.members)) return null; + const members = roster.members.map((member) => + normalizedRosterCharacter(member, key.region) + ); + return members.every((member) => member !== null) + ? (members as BlizzardRosterCharacter[]) + : null; + }, + signal, + onProfileRequest + ); + } + + async function getAchievementFingerprint( + key: CharacterKey, + signal?: AbortSignal, + onProfileRequest?: BlizzardProfileRequestObserver + ): Promise { + const validKey = validCharacterKey(key); + return request( + achievementsUrl(validKey), + fingerprintFromResponse, + signal, + onProfileRequest + ); + } + + return { getGuildRoster, getAchievementFingerprint }; +} + +export { createBlizzardError }; diff --git a/packages/blizzard/src/fingerprint.test.ts b/packages/blizzard/src/fingerprint.test.ts new file mode 100644 index 0000000..b861b15 --- /dev/null +++ b/packages/blizzard/src/fingerprint.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; + +import { compareFingerprints } from "./fingerprint"; + +function fingerprint( + common: number, + identical: number +): ReadonlyMap { + return new Map( + Array.from({ length: common }, (_, id) => [id, id < identical ? id : -id]) + ); +} + +describe("compareFingerprints", () => { + it("requires both the common-achievement floor and identical-timestamp floor", () => { + // Break caught: accepting a candidate when either threshold is not met. + const root = new Map(Array.from({ length: 200 }, (_, id) => [id, id])); + const tooSmall = fingerprint(199, 199); + const belowPercent = fingerprint(200, 39); + const exactBoundary = fingerprint(200, 40); + const policy = { minimumCommon: 200, minimumIdenticalPercent: 20 }; + + expect(compareFingerprints(root, tooSmall, policy).isMatch).toBe(false); + expect(compareFingerprints(root, belowPercent, policy).isMatch).toBe(false); + expect(compareFingerprints(root, exactBoundary, policy)).toMatchObject({ + common: 200, + identical: 40, + isMatch: true + }); + }); + + it("does not allow caller policy to lower the mandatory match floors", () => { + // Break caught: worker configuration could turn a weak coincidence into a + // fingerprint-derived relationship by supplying lower thresholds. + const weakPolicy = { minimumCommon: 1, minimumIdenticalPercent: 0 }; + const fewerThanMandatoryCommon = fingerprint(199, 199); + const belowMandatoryIdenticalPercent = fingerprint(200, 0); + + expect( + compareFingerprints( + fewerThanMandatoryCommon, + fewerThanMandatoryCommon, + weakPolicy + ).isMatch + ).toBe(false); + expect( + compareFingerprints( + fingerprint(200, 200), + belowMandatoryIdenticalPercent, + weakPolicy + ).isMatch + ).toBe(false); + }); +}); diff --git a/packages/blizzard/src/fingerprint.ts b/packages/blizzard/src/fingerprint.ts new file mode 100644 index 0000000..f8d4c28 --- /dev/null +++ b/packages/blizzard/src/fingerprint.ts @@ -0,0 +1,34 @@ +import type { AchievementFingerprint } from "./types"; + +const mandatoryMinimumCommon = 200; +const mandatoryMinimumIdenticalPercent = 20; + +export function compareFingerprints( + root: AchievementFingerprint, + candidate: AchievementFingerprint, + policy: { minimumCommon: number; minimumIdenticalPercent: number } +): { common: number; identical: number; isMatch: boolean } { + let common = 0; + let identical = 0; + + for (const [achievementId, timestamp] of root) { + const candidateTimestamp = candidate.get(achievementId); + if (candidateTimestamp === undefined) continue; + + common += 1; + if (candidateTimestamp === timestamp) identical += 1; + } + + const identicalPercent = common === 0 ? 0 : (identical / common) * 100; + const minimumCommon = Math.max(mandatoryMinimumCommon, policy.minimumCommon); + const minimumIdenticalPercent = Math.max( + mandatoryMinimumIdenticalPercent, + policy.minimumIdenticalPercent + ); + return { + common, + identical, + isMatch: + common >= minimumCommon && identicalPercent >= minimumIdenticalPercent + }; +} diff --git a/packages/blizzard/src/index.ts b/packages/blizzard/src/index.ts new file mode 100644 index 0000000..7265487 --- /dev/null +++ b/packages/blizzard/src/index.ts @@ -0,0 +1,11 @@ +export { createBlizzardClient } from "./client"; +export type { CreateBlizzardClientOptions } from "./client"; +export { compareFingerprints } from "./fingerprint"; +export type { + AchievementFingerprint, + BlizzardError, + BlizzardFailure, + BlizzardGateway, + BlizzardProfileRequestObserver, + BlizzardRosterCharacter +} from "./types"; diff --git a/packages/blizzard/src/types.ts b/packages/blizzard/src/types.ts new file mode 100644 index 0000000..b850f66 --- /dev/null +++ b/packages/blizzard/src/types.ts @@ -0,0 +1,37 @@ +import type { CharacterKey } from "@slashwho/domain"; + +export type AchievementFingerprint = ReadonlyMap; + +export type BlizzardRosterCharacter = Readonly<{ + key: CharacterKey; + displayName: string; + className: string; + level: number; +}>; + +/** Called immediately before a request to the Blizzard profile API. */ +export type BlizzardProfileRequestObserver = () => Promise | void; + +export interface BlizzardGateway { + getGuildRoster( + root: CharacterKey, + signal?: AbortSignal, + onProfileRequest?: BlizzardProfileRequestObserver + ): Promise; + getAchievementFingerprint( + key: CharacterKey, + signal?: AbortSignal, + onProfileRequest?: BlizzardProfileRequestObserver + ): Promise; +} + +export type BlizzardFailure = + | { kind: "not_found" } + | { + kind: "transient"; + status?: number; + retryAfterMs?: number; + } + | { kind: "schema_drift" }; + +export type BlizzardError = Error & BlizzardFailure; diff --git a/packages/blizzard/tsconfig.json b/packages/blizzard/tsconfig.json new file mode 100644 index 0000000..9e25e6e --- /dev/null +++ b/packages/blizzard/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*.ts"] +} diff --git a/packages/contracts/src/contracts.test.ts b/packages/contracts/src/contracts.test.ts index 1a317c0..31a5a6c 100644 --- a/packages/contracts/src/contracts.test.ts +++ b/packages/contracts/src/contracts.test.ts @@ -45,6 +45,28 @@ it("rejects internal provenance in a public character response", () => { expect(characterResourceSchema.safeParse(value).success).toBe(false); }); +it("rejects fingerprint sweep internals in a public character response", () => { + // Break caught: adding a worker-only field to a public API response would + // disclose the source or confidence of a fingerprint-derived link. + const value = { + ...currentCharacter, + discoverySource: "fingerprint", + snapshot: { + ...currentCharacter.snapshot, + reservationId: "private-reservation-id", + characters: [ + { + ...character, + source: "fingerprint", + fingerprintScore: 100 + } + ] + } + }; + + expect(characterResourceSchema.safeParse(value).success).toBe(false); +}); + it("accepts every character value the upstream normalizer accepts", () => { // Break caught: a level the Raider.IO normalizer commits to an immutable snapshot // could be rejected by the public schema, breaking that character page forever. diff --git a/packages/database/drizzle/0002_fingerprint_sweeps.sql b/packages/database/drizzle/0002_fingerprint_sweeps.sql new file mode 100644 index 0000000..813ea38 --- /dev/null +++ b/packages/database/drizzle/0002_fingerprint_sweeps.sql @@ -0,0 +1,47 @@ +ALTER TYPE "public"."discovery_source" ADD VALUE 'fingerprint';--> statement-breakpoint +CREATE TABLE "fingerprint_sweep_admissions" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "queue_order" bigserial NOT NULL, + "discovery_run_id" uuid NOT NULL, + "region" text NOT NULL, + "realm_slug" text NOT NULL, + "normalized_name" text NOT NULL, + "request_cap" integer NOT NULL, + "hourly_budget" integer NOT NULL, + "cadence_cutoff" timestamp with time zone NOT NULL, + "status" text DEFAULT 'waiting' NOT NULL, + "requested_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "fingerprint_sweep_admissions_request_cap_check" CHECK ("fingerprint_sweep_admissions"."request_cap" > 0), + CONSTRAINT "fingerprint_sweep_admissions_hourly_budget_check" CHECK ("fingerprint_sweep_admissions"."hourly_budget" > 0) +); +--> statement-breakpoint +CREATE TABLE "fingerprint_sweep_reservations" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "admission_id" uuid NOT NULL, + "request_cap" integer NOT NULL, + "used_count" integer DEFAULT 0 NOT NULL, + "admitted_at" timestamp with time zone NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "released_at" timestamp with time zone, + "finished_at" timestamp with time zone, + "published" boolean, + "limitation_code" text, + CONSTRAINT "fingerprint_sweep_reservations_request_cap_check" CHECK ("fingerprint_sweep_reservations"."request_cap" > 0), + CONSTRAINT "fingerprint_sweep_reservations_used_count_check" CHECK ("fingerprint_sweep_reservations"."used_count" >= 0 AND "fingerprint_sweep_reservations"."used_count" <= "fingerprint_sweep_reservations"."request_cap"), + CONSTRAINT "fingerprint_sweep_reservations_expiry_check" CHECK ("fingerprint_sweep_reservations"."expires_at" > "fingerprint_sweep_reservations"."admitted_at") +); +--> statement-breakpoint +CREATE TABLE "fingerprint_sweep_states" ( + "region" text NOT NULL, + "realm_slug" text NOT NULL, + "normalized_name" text NOT NULL, + "last_published_at" timestamp with time zone, + CONSTRAINT "fingerprint_sweep_states_pkey" PRIMARY KEY("region","realm_slug","normalized_name") +); +--> statement-breakpoint +ALTER TABLE "fingerprint_sweep_admissions" ADD CONSTRAINT "fingerprint_sweep_admissions_discovery_run_id_discovery_runs_id_fk" FOREIGN KEY ("discovery_run_id") REFERENCES "public"."discovery_runs"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "fingerprint_sweep_reservations" ADD CONSTRAINT "fingerprint_sweep_reservations_admission_id_fingerprint_sweep_admissions_id_fk" FOREIGN KEY ("admission_id") REFERENCES "public"."fingerprint_sweep_admissions"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "fingerprint_sweep_admissions_waiting_idx" ON "fingerprint_sweep_admissions" USING btree ("status","requested_at","queue_order");--> statement-breakpoint +CREATE INDEX "fingerprint_sweep_admissions_root_idx" ON "fingerprint_sweep_admissions" USING btree ("region","realm_slug","normalized_name");--> statement-breakpoint +CREATE UNIQUE INDEX "fingerprint_sweep_reservations_admission_idx" ON "fingerprint_sweep_reservations" USING btree ("admission_id");--> statement-breakpoint +CREATE INDEX "fingerprint_sweep_reservations_expiry_idx" ON "fingerprint_sweep_reservations" USING btree ("expires_at"); \ No newline at end of file diff --git a/packages/database/drizzle/0003_fingerprint_admission_dispatch.sql b/packages/database/drizzle/0003_fingerprint_admission_dispatch.sql new file mode 100644 index 0000000..943276b --- /dev/null +++ b/packages/database/drizzle/0003_fingerprint_admission_dispatch.sql @@ -0,0 +1,3 @@ +ALTER TABLE "fingerprint_sweep_admissions" ADD COLUMN "dispatched_at" timestamp with time zone; +--> statement-breakpoint +CREATE INDEX "fingerprint_sweep_admissions_dispatch_idx" ON "fingerprint_sweep_admissions" USING btree ("status","dispatched_at","requested_at","queue_order"); diff --git a/packages/database/drizzle/0004_simple_venom.sql b/packages/database/drizzle/0004_simple_venom.sql new file mode 100644 index 0000000..b2ecc3b --- /dev/null +++ b/packages/database/drizzle/0004_simple_venom.sql @@ -0,0 +1,8 @@ +CREATE TABLE "fingerprint_sweep_request_events" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "reservation_id" uuid NOT NULL, + "requested_at" timestamp with time zone NOT NULL +); +--> statement-breakpoint +ALTER TABLE "fingerprint_sweep_request_events" ADD CONSTRAINT "fingerprint_sweep_request_events_reservation_id_fingerprint_sweep_reservations_id_fk" FOREIGN KEY ("reservation_id") REFERENCES "public"."fingerprint_sweep_reservations"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "fingerprint_sweep_request_events_window_idx" ON "fingerprint_sweep_request_events" USING btree ("requested_at"); \ No newline at end of file diff --git a/packages/database/drizzle/meta/0002_snapshot.json b/packages/database/drizzle/meta/0002_snapshot.json new file mode 100644 index 0000000..3a983f5 --- /dev/null +++ b/packages/database/drizzle/meta/0002_snapshot.json @@ -0,0 +1,1179 @@ +{ + "id": "f5f059bd-f78a-47d5-bfaa-9874bdffacd5", + "prevId": "3b6dfffc-fdbb-46bb-b4f1-078a749ddd35", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.characters": { + "name": "characters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "class_name": { + "name": "class_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raider_io_url": { + "name": "raider_io_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "characters_canonical_key_idx": { + "name": "characters_canonical_key_idx", + "columns": [ + { + "expression": "region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.discovery_runs": { + "name": "discovery_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "root_region": { + "name": "root_region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_realm_slug": { + "name": "root_realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_normalized_name": { + "name": "root_normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_character_id": { + "name": "root_character_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "queue_job_id": { + "name": "queue_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "discovery_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "caller_class": { + "name": "caller_class", + "type": "caller_class", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "discovery_runs_one_active_root_idx": { + "name": "discovery_runs_one_active_root_idx", + "columns": [ + { + "expression": "root_region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"discovery_runs\".\"status\" in ('queued', 'running', 'retrying')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "discovery_runs_root_character_id_characters_id_fk": { + "name": "discovery_runs_root_character_id_characters_id_fk", + "tableFrom": "discovery_runs", + "tableTo": "characters", + "columnsFrom": [ + "root_character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "discovery_runs_snapshot_id_snapshots_id_fk": { + "name": "discovery_runs_snapshot_id_snapshots_id_fk", + "tableFrom": "discovery_runs", + "tableTo": "snapshots", + "columnsFrom": [ + "snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_admissions": { + "name": "fingerprint_sweep_admissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "queue_order": { + "name": "queue_order", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_cap": { + "name": "request_cap", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hourly_budget": { + "name": "hourly_budget", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cadence_cutoff": { + "name": "cadence_cutoff", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'waiting'" + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fingerprint_sweep_admissions_waiting_idx": { + "name": "fingerprint_sweep_admissions_waiting_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queue_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_admissions_root_idx": { + "name": "fingerprint_sweep_admissions_root_idx", + "columns": [ + { + "expression": "region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fingerprint_sweep_admissions_discovery_run_id_discovery_runs_id_fk": { + "name": "fingerprint_sweep_admissions_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "fingerprint_sweep_admissions", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "fingerprint_sweep_admissions_request_cap_check": { + "name": "fingerprint_sweep_admissions_request_cap_check", + "value": "\"fingerprint_sweep_admissions\".\"request_cap\" > 0" + }, + "fingerprint_sweep_admissions_hourly_budget_check": { + "name": "fingerprint_sweep_admissions_hourly_budget_check", + "value": "\"fingerprint_sweep_admissions\".\"hourly_budget\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_reservations": { + "name": "fingerprint_sweep_reservations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "admission_id": { + "name": "admission_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_cap": { + "name": "request_cap", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "used_count": { + "name": "used_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "admitted_at": { + "name": "admitted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "limitation_code": { + "name": "limitation_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fingerprint_sweep_reservations_admission_idx": { + "name": "fingerprint_sweep_reservations_admission_idx", + "columns": [ + { + "expression": "admission_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_reservations_expiry_idx": { + "name": "fingerprint_sweep_reservations_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fingerprint_sweep_reservations_admission_id_fingerprint_sweep_admissions_id_fk": { + "name": "fingerprint_sweep_reservations_admission_id_fingerprint_sweep_admissions_id_fk", + "tableFrom": "fingerprint_sweep_reservations", + "tableTo": "fingerprint_sweep_admissions", + "columnsFrom": [ + "admission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "fingerprint_sweep_reservations_request_cap_check": { + "name": "fingerprint_sweep_reservations_request_cap_check", + "value": "\"fingerprint_sweep_reservations\".\"request_cap\" > 0" + }, + "fingerprint_sweep_reservations_used_count_check": { + "name": "fingerprint_sweep_reservations_used_count_check", + "value": "\"fingerprint_sweep_reservations\".\"used_count\" >= 0 AND \"fingerprint_sweep_reservations\".\"used_count\" <= \"fingerprint_sweep_reservations\".\"request_cap\"" + }, + "fingerprint_sweep_reservations_expiry_check": { + "name": "fingerprint_sweep_reservations_expiry_check", + "value": "\"fingerprint_sweep_reservations\".\"expires_at\" > \"fingerprint_sweep_reservations\".\"admitted_at\"" + } + }, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_states": { + "name": "fingerprint_sweep_states", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_published_at": { + "name": "last_published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "fingerprint_sweep_states_pkey": { + "name": "fingerprint_sweep_states_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negative_character_cache": { + "name": "negative_character_cache", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "negative_character_cache_expiry_idx": { + "name": "negative_character_cache_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "negative_character_cache_pkey": { + "name": "negative_character_cache_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_events": { + "name": "rate_limit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "caller_bucket_hash": { + "name": "caller_bucket_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "rate_limit_events_bucket_expiry_idx": { + "name": "rate_limit_events_bucket_expiry_idx", + "columns": [ + { + "expression": "caller_bucket_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rate_limit_events_expiry_idx": { + "name": "rate_limit_events_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rate_limit_events_discovery_run_idx": { + "name": "rate_limit_events_discovery_run_idx", + "columns": [ + { + "expression": "discovery_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"rate_limit_events\".\"discovery_run_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "rate_limit_events_discovery_run_id_discovery_runs_id_fk": { + "name": "rate_limit_events_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "rate_limit_events", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.snapshot_characters": { + "name": "snapshot_characters", + "schema": "", + "columns": { + "snapshot_id": { + "name": "snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "character_id": { + "name": "character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "display_order": { + "name": "display_order", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "discovery_source": { + "name": "discovery_source", + "type": "discovery_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "class_name": { + "name": "class_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raider_io_url": { + "name": "raider_io_url", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "snapshot_characters_membership_idx": { + "name": "snapshot_characters_membership_idx", + "columns": [ + { + "expression": "snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "snapshot_characters_display_order_idx": { + "name": "snapshot_characters_display_order_idx", + "columns": [ + { + "expression": "snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "snapshot_characters_snapshot_id_snapshots_id_fk": { + "name": "snapshot_characters_snapshot_id_snapshots_id_fk", + "tableFrom": "snapshot_characters", + "tableTo": "snapshots", + "columnsFrom": [ + "snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snapshot_characters_character_id_characters_id_fk": { + "name": "snapshot_characters_character_id_characters_id_fk", + "tableFrom": "snapshot_characters", + "tableTo": "characters", + "columnsFrom": [ + "character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.snapshots": { + "name": "snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "root_character_id": { + "name": "root_character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "snapshot_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "limitation_code": { + "name": "limitation_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "snapshots_discovery_run_idx": { + "name": "snapshots_discovery_run_idx", + "columns": [ + { + "expression": "discovery_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "snapshots_root_refreshed_idx": { + "name": "snapshots_root_refreshed_idx", + "columns": [ + { + "expression": "root_character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "refreshed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "snapshots_root_character_id_characters_id_fk": { + "name": "snapshots_root_character_id_characters_id_fk", + "tableFrom": "snapshots", + "tableTo": "characters", + "columnsFrom": [ + "root_character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "snapshots_discovery_run_id_discovery_runs_id_fk": { + "name": "snapshots_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "snapshots", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "snapshots_state_limitation_check": { + "name": "snapshots_state_limitation_check", + "value": "(\"snapshots\".\"state\" = 'complete' AND \"snapshots\".\"limitation_code\" IS NULL) OR (\"snapshots\".\"state\" = 'partial' AND \"snapshots\".\"limitation_code\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.suppressed_characters": { + "name": "suppressed_characters", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suppressed_at": { + "name": "suppressed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "suppressed_characters_expiry_idx": { + "name": "suppressed_characters_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "suppressed_characters_pkey": { + "name": "suppressed_characters_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.caller_class": { + "name": "caller_class", + "schema": "public", + "values": [ + "anonymous", + "bot" + ] + }, + "public.discovery_run_status": { + "name": "discovery_run_status", + "schema": "public", + "values": [ + "queued", + "running", + "retrying", + "complete", + "failed" + ] + }, + "public.discovery_source": { + "name": "discovery_source", + "schema": "public", + "values": [ + "input", + "claimed", + "declared_main", + "profile_guess", + "fingerprint" + ] + }, + "public.snapshot_state": { + "name": "snapshot_state", + "schema": "public", + "values": [ + "complete", + "partial" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/database/drizzle/meta/0003_snapshot.json b/packages/database/drizzle/meta/0003_snapshot.json new file mode 100644 index 0000000..6958ff9 --- /dev/null +++ b/packages/database/drizzle/meta/0003_snapshot.json @@ -0,0 +1,1218 @@ +{ + "id": "7e4136da-1a2d-464a-87d7-c5b87121851d", + "prevId": "f5f059bd-f78a-47d5-bfaa-9874bdffacd5", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.characters": { + "name": "characters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "class_name": { + "name": "class_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raider_io_url": { + "name": "raider_io_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "characters_canonical_key_idx": { + "name": "characters_canonical_key_idx", + "columns": [ + { + "expression": "region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.discovery_runs": { + "name": "discovery_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "root_region": { + "name": "root_region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_realm_slug": { + "name": "root_realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_normalized_name": { + "name": "root_normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_character_id": { + "name": "root_character_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "queue_job_id": { + "name": "queue_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "discovery_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "caller_class": { + "name": "caller_class", + "type": "caller_class", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "discovery_runs_one_active_root_idx": { + "name": "discovery_runs_one_active_root_idx", + "columns": [ + { + "expression": "root_region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"discovery_runs\".\"status\" in ('queued', 'running', 'retrying')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "discovery_runs_root_character_id_characters_id_fk": { + "name": "discovery_runs_root_character_id_characters_id_fk", + "tableFrom": "discovery_runs", + "tableTo": "characters", + "columnsFrom": [ + "root_character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "discovery_runs_snapshot_id_snapshots_id_fk": { + "name": "discovery_runs_snapshot_id_snapshots_id_fk", + "tableFrom": "discovery_runs", + "tableTo": "snapshots", + "columnsFrom": [ + "snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_admissions": { + "name": "fingerprint_sweep_admissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "queue_order": { + "name": "queue_order", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_cap": { + "name": "request_cap", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hourly_budget": { + "name": "hourly_budget", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cadence_cutoff": { + "name": "cadence_cutoff", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'waiting'" + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fingerprint_sweep_admissions_waiting_idx": { + "name": "fingerprint_sweep_admissions_waiting_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queue_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_admissions_dispatch_idx": { + "name": "fingerprint_sweep_admissions_dispatch_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dispatched_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queue_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_admissions_root_idx": { + "name": "fingerprint_sweep_admissions_root_idx", + "columns": [ + { + "expression": "region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fingerprint_sweep_admissions_discovery_run_id_discovery_runs_id_fk": { + "name": "fingerprint_sweep_admissions_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "fingerprint_sweep_admissions", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "fingerprint_sweep_admissions_request_cap_check": { + "name": "fingerprint_sweep_admissions_request_cap_check", + "value": "\"fingerprint_sweep_admissions\".\"request_cap\" > 0" + }, + "fingerprint_sweep_admissions_hourly_budget_check": { + "name": "fingerprint_sweep_admissions_hourly_budget_check", + "value": "\"fingerprint_sweep_admissions\".\"hourly_budget\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_reservations": { + "name": "fingerprint_sweep_reservations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "admission_id": { + "name": "admission_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_cap": { + "name": "request_cap", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "used_count": { + "name": "used_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "admitted_at": { + "name": "admitted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "limitation_code": { + "name": "limitation_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fingerprint_sweep_reservations_admission_idx": { + "name": "fingerprint_sweep_reservations_admission_idx", + "columns": [ + { + "expression": "admission_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_reservations_expiry_idx": { + "name": "fingerprint_sweep_reservations_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fingerprint_sweep_reservations_admission_id_fingerprint_sweep_admissions_id_fk": { + "name": "fingerprint_sweep_reservations_admission_id_fingerprint_sweep_admissions_id_fk", + "tableFrom": "fingerprint_sweep_reservations", + "tableTo": "fingerprint_sweep_admissions", + "columnsFrom": [ + "admission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "fingerprint_sweep_reservations_request_cap_check": { + "name": "fingerprint_sweep_reservations_request_cap_check", + "value": "\"fingerprint_sweep_reservations\".\"request_cap\" > 0" + }, + "fingerprint_sweep_reservations_used_count_check": { + "name": "fingerprint_sweep_reservations_used_count_check", + "value": "\"fingerprint_sweep_reservations\".\"used_count\" >= 0 AND \"fingerprint_sweep_reservations\".\"used_count\" <= \"fingerprint_sweep_reservations\".\"request_cap\"" + }, + "fingerprint_sweep_reservations_expiry_check": { + "name": "fingerprint_sweep_reservations_expiry_check", + "value": "\"fingerprint_sweep_reservations\".\"expires_at\" > \"fingerprint_sweep_reservations\".\"admitted_at\"" + } + }, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_states": { + "name": "fingerprint_sweep_states", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_published_at": { + "name": "last_published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "fingerprint_sweep_states_pkey": { + "name": "fingerprint_sweep_states_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negative_character_cache": { + "name": "negative_character_cache", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "negative_character_cache_expiry_idx": { + "name": "negative_character_cache_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "negative_character_cache_pkey": { + "name": "negative_character_cache_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_events": { + "name": "rate_limit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "caller_bucket_hash": { + "name": "caller_bucket_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "rate_limit_events_bucket_expiry_idx": { + "name": "rate_limit_events_bucket_expiry_idx", + "columns": [ + { + "expression": "caller_bucket_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rate_limit_events_expiry_idx": { + "name": "rate_limit_events_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rate_limit_events_discovery_run_idx": { + "name": "rate_limit_events_discovery_run_idx", + "columns": [ + { + "expression": "discovery_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"rate_limit_events\".\"discovery_run_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "rate_limit_events_discovery_run_id_discovery_runs_id_fk": { + "name": "rate_limit_events_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "rate_limit_events", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.snapshot_characters": { + "name": "snapshot_characters", + "schema": "", + "columns": { + "snapshot_id": { + "name": "snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "character_id": { + "name": "character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "display_order": { + "name": "display_order", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "discovery_source": { + "name": "discovery_source", + "type": "discovery_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "class_name": { + "name": "class_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raider_io_url": { + "name": "raider_io_url", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "snapshot_characters_membership_idx": { + "name": "snapshot_characters_membership_idx", + "columns": [ + { + "expression": "snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "snapshot_characters_display_order_idx": { + "name": "snapshot_characters_display_order_idx", + "columns": [ + { + "expression": "snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "snapshot_characters_snapshot_id_snapshots_id_fk": { + "name": "snapshot_characters_snapshot_id_snapshots_id_fk", + "tableFrom": "snapshot_characters", + "tableTo": "snapshots", + "columnsFrom": [ + "snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snapshot_characters_character_id_characters_id_fk": { + "name": "snapshot_characters_character_id_characters_id_fk", + "tableFrom": "snapshot_characters", + "tableTo": "characters", + "columnsFrom": [ + "character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.snapshots": { + "name": "snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "root_character_id": { + "name": "root_character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "snapshot_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "limitation_code": { + "name": "limitation_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "snapshots_discovery_run_idx": { + "name": "snapshots_discovery_run_idx", + "columns": [ + { + "expression": "discovery_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "snapshots_root_refreshed_idx": { + "name": "snapshots_root_refreshed_idx", + "columns": [ + { + "expression": "root_character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "refreshed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "snapshots_root_character_id_characters_id_fk": { + "name": "snapshots_root_character_id_characters_id_fk", + "tableFrom": "snapshots", + "tableTo": "characters", + "columnsFrom": [ + "root_character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "snapshots_discovery_run_id_discovery_runs_id_fk": { + "name": "snapshots_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "snapshots", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "snapshots_state_limitation_check": { + "name": "snapshots_state_limitation_check", + "value": "(\"snapshots\".\"state\" = 'complete' AND \"snapshots\".\"limitation_code\" IS NULL) OR (\"snapshots\".\"state\" = 'partial' AND \"snapshots\".\"limitation_code\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.suppressed_characters": { + "name": "suppressed_characters", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suppressed_at": { + "name": "suppressed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "suppressed_characters_expiry_idx": { + "name": "suppressed_characters_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "suppressed_characters_pkey": { + "name": "suppressed_characters_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.caller_class": { + "name": "caller_class", + "schema": "public", + "values": [ + "anonymous", + "bot" + ] + }, + "public.discovery_run_status": { + "name": "discovery_run_status", + "schema": "public", + "values": [ + "queued", + "running", + "retrying", + "complete", + "failed" + ] + }, + "public.discovery_source": { + "name": "discovery_source", + "schema": "public", + "values": [ + "input", + "claimed", + "declared_main", + "profile_guess", + "fingerprint" + ] + }, + "public.snapshot_state": { + "name": "snapshot_state", + "schema": "public", + "values": [ + "complete", + "partial" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/database/drizzle/meta/0004_snapshot.json b/packages/database/drizzle/meta/0004_snapshot.json new file mode 100644 index 0000000..d3b2de6 --- /dev/null +++ b/packages/database/drizzle/meta/0004_snapshot.json @@ -0,0 +1,1280 @@ +{ + "id": "7c58c66d-0128-4cd2-a5c2-f3f7c68338c3", + "prevId": "7e4136da-1a2d-464a-87d7-c5b87121851d", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.characters": { + "name": "characters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "class_name": { + "name": "class_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raider_io_url": { + "name": "raider_io_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "characters_canonical_key_idx": { + "name": "characters_canonical_key_idx", + "columns": [ + { + "expression": "region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.discovery_runs": { + "name": "discovery_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "root_region": { + "name": "root_region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_realm_slug": { + "name": "root_realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_normalized_name": { + "name": "root_normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "root_character_id": { + "name": "root_character_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "queue_job_id": { + "name": "queue_job_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "discovery_run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "caller_class": { + "name": "caller_class", + "type": "caller_class", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "snapshot_id": { + "name": "snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "discovery_runs_one_active_root_idx": { + "name": "discovery_runs_one_active_root_idx", + "columns": [ + { + "expression": "root_region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "root_normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"discovery_runs\".\"status\" in ('queued', 'running', 'retrying')", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "discovery_runs_root_character_id_characters_id_fk": { + "name": "discovery_runs_root_character_id_characters_id_fk", + "tableFrom": "discovery_runs", + "tableTo": "characters", + "columnsFrom": [ + "root_character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "discovery_runs_snapshot_id_snapshots_id_fk": { + "name": "discovery_runs_snapshot_id_snapshots_id_fk", + "tableFrom": "discovery_runs", + "tableTo": "snapshots", + "columnsFrom": [ + "snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_admissions": { + "name": "fingerprint_sweep_admissions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "queue_order": { + "name": "queue_order", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_cap": { + "name": "request_cap", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "hourly_budget": { + "name": "hourly_budget", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cadence_cutoff": { + "name": "cadence_cutoff", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'waiting'" + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "fingerprint_sweep_admissions_waiting_idx": { + "name": "fingerprint_sweep_admissions_waiting_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queue_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_admissions_root_idx": { + "name": "fingerprint_sweep_admissions_root_idx", + "columns": [ + { + "expression": "region", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "realm_slug", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "normalized_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_admissions_dispatch_idx": { + "name": "fingerprint_sweep_admissions_dispatch_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dispatched_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "queue_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fingerprint_sweep_admissions_discovery_run_id_discovery_runs_id_fk": { + "name": "fingerprint_sweep_admissions_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "fingerprint_sweep_admissions", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "fingerprint_sweep_admissions_request_cap_check": { + "name": "fingerprint_sweep_admissions_request_cap_check", + "value": "\"fingerprint_sweep_admissions\".\"request_cap\" > 0" + }, + "fingerprint_sweep_admissions_hourly_budget_check": { + "name": "fingerprint_sweep_admissions_hourly_budget_check", + "value": "\"fingerprint_sweep_admissions\".\"hourly_budget\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_request_events": { + "name": "fingerprint_sweep_request_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "reservation_id": { + "name": "reservation_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "fingerprint_sweep_request_events_window_idx": { + "name": "fingerprint_sweep_request_events_window_idx", + "columns": [ + { + "expression": "requested_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fingerprint_sweep_request_events_reservation_id_fingerprint_sweep_reservations_id_fk": { + "name": "fingerprint_sweep_request_events_reservation_id_fingerprint_sweep_reservations_id_fk", + "tableFrom": "fingerprint_sweep_request_events", + "tableTo": "fingerprint_sweep_reservations", + "columnsFrom": [ + "reservation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_reservations": { + "name": "fingerprint_sweep_reservations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "admission_id": { + "name": "admission_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_cap": { + "name": "request_cap", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "used_count": { + "name": "used_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "admitted_at": { + "name": "admitted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "published": { + "name": "published", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "limitation_code": { + "name": "limitation_code", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "fingerprint_sweep_reservations_admission_idx": { + "name": "fingerprint_sweep_reservations_admission_idx", + "columns": [ + { + "expression": "admission_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "fingerprint_sweep_reservations_expiry_idx": { + "name": "fingerprint_sweep_reservations_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "fingerprint_sweep_reservations_admission_id_fingerprint_sweep_admissions_id_fk": { + "name": "fingerprint_sweep_reservations_admission_id_fingerprint_sweep_admissions_id_fk", + "tableFrom": "fingerprint_sweep_reservations", + "tableTo": "fingerprint_sweep_admissions", + "columnsFrom": [ + "admission_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "fingerprint_sweep_reservations_request_cap_check": { + "name": "fingerprint_sweep_reservations_request_cap_check", + "value": "\"fingerprint_sweep_reservations\".\"request_cap\" > 0" + }, + "fingerprint_sweep_reservations_used_count_check": { + "name": "fingerprint_sweep_reservations_used_count_check", + "value": "\"fingerprint_sweep_reservations\".\"used_count\" >= 0 AND \"fingerprint_sweep_reservations\".\"used_count\" <= \"fingerprint_sweep_reservations\".\"request_cap\"" + }, + "fingerprint_sweep_reservations_expiry_check": { + "name": "fingerprint_sweep_reservations_expiry_check", + "value": "\"fingerprint_sweep_reservations\".\"expires_at\" > \"fingerprint_sweep_reservations\".\"admitted_at\"" + } + }, + "isRLSEnabled": false + }, + "public.fingerprint_sweep_states": { + "name": "fingerprint_sweep_states", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_published_at": { + "name": "last_published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "fingerprint_sweep_states_pkey": { + "name": "fingerprint_sweep_states_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.negative_character_cache": { + "name": "negative_character_cache", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "negative_character_cache_expiry_idx": { + "name": "negative_character_cache_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "negative_character_cache_pkey": { + "name": "negative_character_cache_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.rate_limit_events": { + "name": "rate_limit_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "caller_bucket_hash": { + "name": "caller_bucket_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "rate_limit_events_bucket_expiry_idx": { + "name": "rate_limit_events_bucket_expiry_idx", + "columns": [ + { + "expression": "caller_bucket_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rate_limit_events_expiry_idx": { + "name": "rate_limit_events_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "rate_limit_events_discovery_run_idx": { + "name": "rate_limit_events_discovery_run_idx", + "columns": [ + { + "expression": "discovery_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"rate_limit_events\".\"discovery_run_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "rate_limit_events_discovery_run_id_discovery_runs_id_fk": { + "name": "rate_limit_events_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "rate_limit_events", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.snapshot_characters": { + "name": "snapshot_characters", + "schema": "", + "columns": { + "snapshot_id": { + "name": "snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "character_id": { + "name": "character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "display_order": { + "name": "display_order", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "discovery_source": { + "name": "discovery_source", + "type": "discovery_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "class_name": { + "name": "class_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "level": { + "name": "level", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "raider_io_url": { + "name": "raider_io_url", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "snapshot_characters_membership_idx": { + "name": "snapshot_characters_membership_idx", + "columns": [ + { + "expression": "snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "snapshot_characters_display_order_idx": { + "name": "snapshot_characters_display_order_idx", + "columns": [ + { + "expression": "snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "display_order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "snapshot_characters_snapshot_id_snapshots_id_fk": { + "name": "snapshot_characters_snapshot_id_snapshots_id_fk", + "tableFrom": "snapshot_characters", + "tableTo": "snapshots", + "columnsFrom": [ + "snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "snapshot_characters_character_id_characters_id_fk": { + "name": "snapshot_characters_character_id_characters_id_fk", + "tableFrom": "snapshot_characters", + "tableTo": "characters", + "columnsFrom": [ + "character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.snapshots": { + "name": "snapshots", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "root_character_id": { + "name": "root_character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "discovery_run_id": { + "name": "discovery_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "snapshot_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "limitation_code": { + "name": "limitation_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "character_count": { + "name": "character_count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "snapshots_discovery_run_idx": { + "name": "snapshots_discovery_run_idx", + "columns": [ + { + "expression": "discovery_run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "snapshots_root_refreshed_idx": { + "name": "snapshots_root_refreshed_idx", + "columns": [ + { + "expression": "root_character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "refreshed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "snapshots_root_character_id_characters_id_fk": { + "name": "snapshots_root_character_id_characters_id_fk", + "tableFrom": "snapshots", + "tableTo": "characters", + "columnsFrom": [ + "root_character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "snapshots_discovery_run_id_discovery_runs_id_fk": { + "name": "snapshots_discovery_run_id_discovery_runs_id_fk", + "tableFrom": "snapshots", + "tableTo": "discovery_runs", + "columnsFrom": [ + "discovery_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "snapshots_state_limitation_check": { + "name": "snapshots_state_limitation_check", + "value": "(\"snapshots\".\"state\" = 'complete' AND \"snapshots\".\"limitation_code\" IS NULL) OR (\"snapshots\".\"state\" = 'partial' AND \"snapshots\".\"limitation_code\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.suppressed_characters": { + "name": "suppressed_characters", + "schema": "", + "columns": { + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "realm_slug": { + "name": "realm_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "normalized_name": { + "name": "normalized_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suppressed_at": { + "name": "suppressed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "suppressed_characters_expiry_idx": { + "name": "suppressed_characters_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "suppressed_characters_pkey": { + "name": "suppressed_characters_pkey", + "columns": [ + "region", + "realm_slug", + "normalized_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.caller_class": { + "name": "caller_class", + "schema": "public", + "values": [ + "anonymous", + "bot" + ] + }, + "public.discovery_run_status": { + "name": "discovery_run_status", + "schema": "public", + "values": [ + "queued", + "running", + "retrying", + "complete", + "failed" + ] + }, + "public.discovery_source": { + "name": "discovery_source", + "schema": "public", + "values": [ + "input", + "claimed", + "declared_main", + "profile_guess", + "fingerprint" + ] + }, + "public.snapshot_state": { + "name": "snapshot_state", + "schema": "public", + "values": [ + "complete", + "partial" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/database/drizzle/meta/_journal.json b/packages/database/drizzle/meta/_journal.json index 1783dfb..516327c 100644 --- a/packages/database/drizzle/meta/_journal.json +++ b/packages/database/drizzle/meta/_journal.json @@ -15,6 +15,27 @@ "when": 1785927934514, "tag": "0001_search_reservations", "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1786365831105, + "tag": "0002_fingerprint_sweeps", + "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1786371100000, + "tag": "0003_fingerprint_admission_dispatch", + "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1786375079385, + "tag": "0004_simple_venom", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/database/src/index.ts b/packages/database/src/index.ts index 9bfcc59..2d3d714 100644 --- a/packages/database/src/index.ts +++ b/packages/database/src/index.ts @@ -4,6 +4,7 @@ export { createDiscoveryQueue, DiscoveryQueueStopTimeoutError, discoverCharacterQueueName, + fingerprintAdmissionQueueName, maintenanceCleanupQueueName } from "./queue"; export type { @@ -17,6 +18,9 @@ export type { CreateSnapshotInput, DiscoveryRun, DiscoverySource, + FingerprintAdmission, + FingerprintAdmissionDispatch, + FingerprintSweepRepository, NegativeCacheEntry, NegativeCacheRepository, RateLimitRepository, diff --git a/packages/database/src/postgres-repositories.ts b/packages/database/src/postgres-repositories.ts index be8aba6..6a2aeaf 100644 --- a/packages/database/src/postgres-repositories.ts +++ b/packages/database/src/postgres-repositories.ts @@ -3,7 +3,9 @@ import type { CharacterKey } from "@slashwho/domain"; import type { Pool, PoolClient } from "pg"; import type { CallerClass, + CreateSnapshotInput, DiscoveryRun, + FingerprintAdmission, Repositories, SnapshotHistoryItem, SnapshotHistoryPage, @@ -64,6 +66,134 @@ async function lockRoot(client: Queryable, key: CharacterKey): Promise { ]); } +async function lockFingerprintSweeps(client: Queryable): Promise { + await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [ + "fingerprint-sweeps" + ]); +} + +function assertFingerprintAdmissionInput(input: { + requestCap: number; + hourlyBudget: number; + cadenceCutoff: Date; + at: Date; +}): void { + if (!Number.isInteger(input.requestCap) || input.requestCap < 1) { + throw new RangeError("fingerprint_request_cap_out_of_range"); + } + if (!Number.isInteger(input.hourlyBudget) || input.hourlyBudget < 1) { + throw new RangeError("fingerprint_hourly_budget_out_of_range"); + } + if (input.requestCap > input.hourlyBudget) { + throw new RangeError("fingerprint_request_cap_exceeds_hourly_budget"); + } + if ( + Number.isNaN(input.cadenceCutoff.valueOf()) || + Number.isNaN(input.at.valueOf()) + ) { + throw new RangeError("fingerprint_admission_time_invalid"); + } +} + +async function fingerprintRetryAt(client: Queryable, at: Date): Promise { + const result = await client.query<{ retry_at: Date | null }>( + `SELECT min(retry_at) AS retry_at FROM ( + SELECT expires_at AS retry_at + FROM fingerprint_sweep_reservations + WHERE expires_at > $1 AND released_at IS NULL + UNION ALL + SELECT requested_at + interval '1 hour' AS retry_at + FROM fingerprint_sweep_request_events + WHERE requested_at + interval '1 hour' > $1 + ) retained`, + [at] + ); + return result.rows[0]?.retry_at ?? at; +} + +async function admitFingerprintWaitingRun( + client: Queryable, + admissionId: string, + at: Date +): Promise> { + const head = await client.query<{ + id: string; + request_cap: number; + hourly_budget: number; + requested_at: Date; + }>( + `SELECT admission.id, admission.request_cap, admission.hourly_budget, admission.requested_at + FROM fingerprint_sweep_admissions admission + LEFT JOIN fingerprint_sweep_states state + ON state.region = admission.region + AND state.realm_slug = admission.realm_slug + AND state.normalized_name = admission.normalized_name + WHERE admission.status = 'waiting' + AND ( + state.last_published_at IS NULL + OR state.last_published_at <= admission.cadence_cutoff + ) + ORDER BY admission.requested_at, admission.queue_order + LIMIT 1 + FOR UPDATE OF admission` + ); + const candidate = head.rows[0]; + if (!candidate || candidate.id !== admissionId) { + const requested = await client.query<{ requested_at: Date }>( + `SELECT requested_at FROM fingerprint_sweep_admissions WHERE id = $1`, + [admissionId] + ); + return { + kind: "waiting", + retryAt: await fingerprintRetryAt(client, at), + blockedSince: requested.rows[0]?.requested_at + }; + } + + const usage = await client.query<{ commitment: string }>( + `SELECT ( + SELECT count(*) FROM fingerprint_sweep_request_events + WHERE requested_at > $1::timestamptz - interval '1 hour' + ) + coalesce(sum(request_cap - used_count) FILTER ( + WHERE released_at IS NULL AND expires_at > $1 + ), 0)::bigint AS commitment + FROM fingerprint_sweep_reservations`, + [at] + ); + if ( + Number(usage.rows[0]!.commitment) + candidate.request_cap > + candidate.hourly_budget + ) { + return { + kind: "waiting", + retryAt: await fingerprintRetryAt(client, at), + blockedSince: candidate.requested_at + }; + } + + const reservation = await client.query<{ id: string }>( + `INSERT INTO fingerprint_sweep_reservations + (admission_id, request_cap, admitted_at, expires_at) + VALUES ($1, $2, $3::timestamptz, $3::timestamptz + interval '1 hour') + RETURNING id`, + [admissionId, candidate.request_cap, at] + ); + await client.query( + `UPDATE fingerprint_sweep_admissions + SET status = 'admitted', dispatched_at = NULL + WHERE id = $1`, + [admissionId] + ); + return { + kind: "admitted", + reservationId: reservation.rows[0]!.id, + requestCap: candidate.request_cap, + committedRequests: + Number(usage.rows[0]!.commitment) + candidate.request_cap, + hourlyBudget: candidate.hourly_budget + }; +} + function mapRun(row: RunRow): DiscoveryRun { return { id: row.id, @@ -206,6 +336,175 @@ async function loadSnapshot( }; } +async function createSnapshot( + client: PoolClient, + input: CreateSnapshotInput, + options?: { signal?: AbortSignal } +): Promise { + const runResult = await client.query( + `SELECT 1 FROM discovery_runs + WHERE id = $1 + AND root_region = $2 + AND root_realm_slug = $3 + AND root_normalized_name = $4 + AND status IN ${activeRunSql} + FOR UPDATE`, + [input.runId, input.rootKey.region, input.rootKey.realm, input.rootKey.name] + ); + if (runResult.rowCount !== 1) { + throw new Error("discovery_run_root_mismatch"); + } + + const characterIds = new Map(); + const charactersByCanonicalKey = [...input.characters].sort((left, right) => { + const leftKey = `${left.key.region}\0${left.key.realm}\0${left.key.name}`; + const rightKey = `${right.key.region}\0${right.key.realm}\0${right.key.name}`; + return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0; + }); + for (const character of charactersByCanonicalKey) { + const result = await client.query<{ id: string }>( + `INSERT INTO characters + (region, realm_slug, normalized_name, display_name, class_name, + level, raider_io_url) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (region, realm_slug, normalized_name) + DO UPDATE SET + display_name = EXCLUDED.display_name, + class_name = EXCLUDED.class_name, + level = EXCLUDED.level, + raider_io_url = EXCLUDED.raider_io_url, + updated_at = now() + RETURNING id`, + [ + character.key.region, + character.key.realm, + character.key.name, + character.displayName, + character.className, + character.level, + character.raiderIoUrl + ] + ); + characterIds.set( + `${character.key.region}/${character.key.realm}/${character.key.name}`, + result.rows[0]!.id + ); + } + + const rootId = characterIds.get( + `${input.rootKey.region}/${input.rootKey.realm}/${input.rootKey.name}` + ); + if (!rootId) throw new Error("snapshot_root_missing"); + + const snapshotResult = await client.query<{ id: string }>( + `INSERT INTO snapshots + (root_character_id, discovery_run_id, state, limitation_code, + refreshed_at, character_count) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id`, + [ + rootId, + input.runId, + input.state, + input.limitationCode, + input.refreshedAt, + input.characters.length + ] + ); + const snapshotId = snapshotResult.rows[0]!.id; + + await client.query( + `UPDATE discovery_runs SET root_character_id = $2 WHERE id = $1`, + [input.runId, rootId] + ); + + for (const [displayOrder, character] of input.characters.entries()) { + const characterId = characterIds.get( + `${character.key.region}/${character.key.realm}/${character.key.name}` + )!; + await client.query( + `INSERT INTO snapshot_characters + (snapshot_id, character_id, display_order, discovery_source, + display_name, class_name, level, raider_io_url) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, + [ + snapshotId, + characterId, + displayOrder, + character.source, + character.displayName, + character.className, + character.level, + character.raiderIoUrl + ] + ); + } + + const publication = await client.query( + `UPDATE discovery_runs + SET status = 'complete', snapshot_id = $2, + completed_at = COALESCE(completed_at, now()), + next_retry_at = NULL, error_code = NULL + WHERE id = $1 AND status IN ${activeRunSql}`, + [input.runId, snapshotId] + ); + if (publication.rowCount !== 1) { + throw new Error("discovery_run_not_active"); + } + + const snapshot = await loadSnapshot(client, snapshotId); + if (!snapshot) throw new Error("snapshot_not_found"); + options?.signal?.throwIfAborted(); + return snapshot; +} + +async function finishFingerprintSweep( + client: PoolClient, + reservationId: string, + input: { published: boolean; at: Date; limitationCode: string | null } +): Promise { + const reservation = await client.query<{ + admission_id: string; + region: CharacterKey["region"]; + realm_slug: string; + normalized_name: string; + }>( + `UPDATE fingerprint_sweep_reservations reservation + SET released_at = $2, + finished_at = $2, + published = $3, + limitation_code = $4 + FROM fingerprint_sweep_admissions admission + WHERE reservation.id = $1 + AND reservation.admission_id = admission.id + AND reservation.released_at IS NULL + RETURNING reservation.admission_id, admission.region, + admission.realm_slug, admission.normalized_name`, + [reservationId, input.at, input.published, input.limitationCode] + ); + const row = reservation.rows[0]; + if (!row) throw new Error("fingerprint_reservation_not_active"); + await client.query( + `UPDATE fingerprint_sweep_admissions + SET status = 'finished' + WHERE id = $1`, + [row.admission_id] + ); + if (input.published) { + await client.query( + `INSERT INTO fingerprint_sweep_states + (region, realm_slug, normalized_name, last_published_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (region, realm_slug, normalized_name) + DO UPDATE SET last_published_at = greatest( + fingerprint_sweep_states.last_published_at, + EXCLUDED.last_published_at + )`, + [row.region, row.realm_slug, row.normalized_name, input.at] + ); + } +} + async function requireUpdated( client: Pool, text: string, @@ -664,6 +963,33 @@ export function createPostgresRepositories(pool: Pool): Repositories { } }, + async createAndFinishFingerprintSweep(input, fingerprint, options) { + if (Number.isNaN(fingerprint.finishedAt.valueOf())) { + throw new RangeError("fingerprint_finish_time_invalid"); + } + const client = await pool.connect(); + try { + options?.signal?.throwIfAborted(); + await client.query("BEGIN"); + await lockRoot(client, input.rootKey); + await lockFingerprintSweeps(client); + const snapshot = await createSnapshot(client, input, options); + await finishFingerprintSweep(client, fingerprint.reservationId, { + published: true, + at: fingerprint.finishedAt, + limitationCode: fingerprint.limitationCode + }); + options?.signal?.throwIfAborted(); + await client.query("COMMIT"); + return snapshot; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + }, + async getCurrent(key) { const result = await pool.query<{ id: string }>( `SELECT snapshot.id @@ -887,6 +1213,352 @@ export function createPostgresRepositories(pool: Pool): Repositories { } }, + fingerprintSweeps: { + async requestAdmission(input): Promise { + assertFingerprintAdmissionInput(input); + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await lockFingerprintSweeps(client); + + const existingAdmission = await client.query<{ + reservation_id: string; + request_cap: number; + }>( + `SELECT reservation.id AS reservation_id, reservation.request_cap + FROM fingerprint_sweep_admissions admission + JOIN fingerprint_sweep_reservations reservation + ON reservation.admission_id = admission.id + WHERE admission.discovery_run_id = $1 + AND admission.status = 'admitted' + AND reservation.released_at IS NULL + ORDER BY admission.requested_at DESC + LIMIT 1 + FOR UPDATE OF admission, reservation`, + [input.runId] + ); + const existing = existingAdmission.rows[0]; + if (existing) { + await client.query("COMMIT"); + return { + kind: "admitted", + reservationId: existing.reservation_id, + requestCap: existing.request_cap + }; + } + + const state = await client.query<{ last_published_at: Date | null }>( + `SELECT last_published_at + FROM fingerprint_sweep_states + WHERE region = $1 AND realm_slug = $2 AND normalized_name = $3`, + [input.key.region, input.key.realm, input.key.name] + ); + if ( + state.rows[0]?.last_published_at && + state.rows[0].last_published_at > input.cadenceCutoff + ) { + await client.query( + `UPDATE fingerprint_sweep_admissions + SET status = 'not_due' + WHERE discovery_run_id = $1 AND status = 'waiting'`, + [input.runId] + ); + await client.query("COMMIT"); + return { kind: "not_due" }; + } + + const waiting = await client.query<{ id: string }>( + `SELECT id + FROM fingerprint_sweep_admissions + WHERE discovery_run_id = $1 AND status = 'waiting' + ORDER BY requested_at, queue_order + LIMIT 1 + FOR UPDATE`, + [input.runId] + ); + let admissionId = waiting.rows[0]?.id; + if (admissionId) { + await client.query( + `UPDATE fingerprint_sweep_admissions + SET request_cap = $2, hourly_budget = $3, cadence_cutoff = $4 + WHERE id = $1`, + [ + admissionId, + input.requestCap, + input.hourlyBudget, + input.cadenceCutoff + ] + ); + } else { + const admission = await client.query<{ id: string }>( + `INSERT INTO fingerprint_sweep_admissions + (discovery_run_id, region, realm_slug, normalized_name, request_cap, + hourly_budget, cadence_cutoff, requested_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + RETURNING id`, + [ + input.runId, + input.key.region, + input.key.realm, + input.key.name, + input.requestCap, + input.hourlyBudget, + input.cadenceCutoff, + input.at + ] + ); + admissionId = admission.rows[0]!.id; + } + + const result = await admitFingerprintWaitingRun( + client, + admissionId, + input.at + ); + if (result.kind === "waiting") { + const deferred = await client.query( + `UPDATE discovery_runs + SET status = 'queued', attempt = greatest(attempt - 1, 0), + next_retry_at = NULL + WHERE id = $1 AND status IN ('running', 'queued')`, + [input.runId] + ); + if (deferred.rowCount !== 1) { + throw new Error("fingerprint_waiting_run_not_running"); + } + } + await client.query("COMMIT"); + return result; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + }, + + async recordRequest(reservationId, count, at) { + if (!Number.isInteger(count) || count < 1) { + throw new RangeError("fingerprint_request_count_out_of_range"); + } + if (Number.isNaN(at.valueOf())) { + throw new RangeError("fingerprint_request_time_invalid"); + } + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await lockFingerprintSweeps(client); + const result = await client.query( + `UPDATE fingerprint_sweep_reservations + SET used_count = used_count + $2 + WHERE id = $1 + AND released_at IS NULL + AND expires_at > $3 + AND used_count + $2 <= request_cap + RETURNING id`, + [reservationId, count, at] + ); + if (result.rowCount !== 1) { + throw new Error("fingerprint_reservation_not_active"); + } + await client.query( + `INSERT INTO fingerprint_sweep_request_events (reservation_id, requested_at) + SELECT $1, $3::timestamptz FROM generate_series(1, $2)`, + [reservationId, count, at] + ); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + }, + + async finish(reservationId, input) { + if (Number.isNaN(input.at.valueOf())) { + throw new RangeError("fingerprint_finish_time_invalid"); + } + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await lockFingerprintSweeps(client); + await finishFingerprintSweep(client, reservationId, input); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + }, + + async release(reservationId, at) { + if (Number.isNaN(at.valueOf())) { + throw new RangeError("fingerprint_release_time_invalid"); + } + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await lockFingerprintSweeps(client); + const result = await client.query<{ admission_id: string }>( + `UPDATE fingerprint_sweep_reservations + SET released_at = $2 + WHERE id = $1 AND released_at IS NULL + RETURNING admission_id`, + [reservationId, at] + ); + const row = result.rows[0]; + if (!row) throw new Error("fingerprint_reservation_not_active"); + await client.query( + `UPDATE fingerprint_sweep_admissions + SET status = 'released' + WHERE id = $1`, + [row.admission_id] + ); + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + }, + + async listWaiting(limit, offset = 0) { + if ( + !Number.isInteger(limit) || + limit < 1 || + limit > 1_000 || + !Number.isInteger(offset) || + offset < 0 + ) { + throw new RangeError("fingerprint_waiting_limit_out_of_range"); + } + const result = await pool.query<{ discovery_run_id: string }>( + `SELECT discovery_run_id + FROM fingerprint_sweep_admissions + WHERE status = 'waiting' + ORDER BY requested_at, queue_order + LIMIT $1 OFFSET $2`, + [limit, offset] + ); + return result.rows.map((row) => row.discovery_run_id); + }, + + async listAdmittedUndispatched(limit) { + if (!Number.isInteger(limit) || limit < 1 || limit > 1_000) { + throw new RangeError( + "fingerprint_admission_dispatch_limit_out_of_range" + ); + } + const result = await pool.query<{ discovery_run_id: string }>( + `SELECT discovery_run_id + FROM fingerprint_sweep_admissions + WHERE status = 'admitted' AND dispatched_at IS NULL + ORDER BY requested_at, queue_order + LIMIT $1`, + [limit] + ); + return result.rows.map((row) => row.discovery_run_id); + }, + + async markDispatched(runId, at) { + if (Number.isNaN(at.valueOf())) { + throw new RangeError("fingerprint_admission_time_invalid"); + } + await pool.query( + `UPDATE fingerprint_sweep_admissions + SET dispatched_at = $2 + WHERE discovery_run_id = $1 + AND status = 'admitted' + AND dispatched_at IS NULL`, + [runId, at] + ); + }, + + async admitWaiting(runId, at) { + if (Number.isNaN(at.valueOf())) { + throw new RangeError("fingerprint_admission_time_invalid"); + } + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await lockFingerprintSweeps(client); + const waiting = await client.query<{ + id: string; + region: CharacterKey["region"]; + realm_slug: string; + normalized_name: string; + cadence_cutoff: Date; + }>( + `SELECT id, region, realm_slug, normalized_name, cadence_cutoff + FROM fingerprint_sweep_admissions + WHERE discovery_run_id = $1 AND status = 'waiting' + ORDER BY requested_at, queue_order + LIMIT 1 + FOR UPDATE`, + [runId] + ); + const admission = waiting.rows[0]; + if (!admission) { + await client.query("COMMIT"); + return { kind: "settled" }; + } + + const state = await client.query<{ last_published_at: Date | null }>( + `SELECT last_published_at + FROM fingerprint_sweep_states + WHERE region = $1 AND realm_slug = $2 AND normalized_name = $3`, + [admission.region, admission.realm_slug, admission.normalized_name] + ); + if ( + state.rows[0]?.last_published_at && + state.rows[0].last_published_at > admission.cadence_cutoff + ) { + await client.query( + `UPDATE fingerprint_sweep_admissions + SET status = 'not_due' + WHERE id = $1`, + [admission.id] + ); + await client.query("COMMIT"); + return { kind: "not_due" }; + } + + const result = await admitFingerprintWaitingRun( + client, + admission.id, + at + ); + await client.query("COMMIT"); + return result.kind === "admitted" ? { kind: "admitted" } : result; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + }, + + async cleanupExpired(at = new Date()) { + if (Number.isNaN(at.valueOf())) { + throw new RangeError("fingerprint_cleanup_time_invalid"); + } + // Each physical Blizzard request leaves one row, so the table would + // grow by the whole hourly budget every hour. A request stops counting + // towards the rolling hour once its own timestamp leaves the window, so + // prune on requested_at: deleting by reservation would drop events the + // admission accounting still has to see. + const result = await pool.query( + `DELETE FROM fingerprint_sweep_request_events + WHERE requested_at <= $1::timestamptz - interval '1 hour'`, + [at] + ); + return result.rowCount ?? 0; + } + }, + negativeCache: { async put(key, expiresAt) { const client = await pool.connect(); diff --git a/packages/database/src/public-api.typecheck.ts b/packages/database/src/public-api.typecheck.ts index 48a4135..e652edb 100644 --- a/packages/database/src/public-api.typecheck.ts +++ b/packages/database/src/public-api.typecheck.ts @@ -1,6 +1,8 @@ import { createPostgresRepositories, runMigrations, + type FingerprintAdmission, + type FingerprintSweepRepository, type Repositories } from "."; @@ -12,5 +14,7 @@ import { schema } from "."; void createPostgresRepositories; void runMigrations; void (undefined as Repositories | undefined); +void (undefined as FingerprintAdmission | undefined); +void (undefined as FingerprintSweepRepository | undefined); void createDatabase; void schema; diff --git a/packages/database/src/queue.test.ts b/packages/database/src/queue.test.ts index a18bbb4..111d366 100644 --- a/packages/database/src/queue.test.ts +++ b/packages/database/src/queue.test.ts @@ -1,6 +1,46 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { updateActiveRetryDelay } from "./queue"; +const queueFakes = vi.hoisted(() => { + const workers: Array<{ + name: string; + handler: (jobs: Array<{ data: { runId: string } }>) => Promise; + }> = []; + const db = { + executeSql: vi.fn(async () => ({ rows: [] })) + }; + return { + createQueue: vi.fn(async () => {}), + updateQueue: vi.fn(async () => {}), + send: vi.fn(async () => "job-id"), + start: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + work: vi.fn(async (name, _options, handler) => { + workers.push({ name, handler }); + }), + getDb: vi.fn(() => db), + db, + workers + }; +}); + +vi.mock("pg-boss", () => ({ + PgBoss: class { + start = queueFakes.start; + stop = queueFakes.stop; + createQueue = queueFakes.createQueue; + updateQueue = queueFakes.updateQueue; + send = queueFakes.send; + work = queueFakes.work; + getDb = queueFakes.getDb; + } +})); + +import { + createDiscoveryQueue, + discoverCharacterQueueName, + fingerprintAdmissionQueueName, + updateActiveRetryDelay +} from "./queue"; describe("pg-boss retry delay update", () => { it("fails safely when the active pg-boss row is not updated", async () => { @@ -16,3 +56,49 @@ describe("pg-boss retry delay update", () => { ).rejects.toThrow("retry_delay_update_failed"); }); }); + +describe("fingerprint admission queue", () => { + it("uses separate generated job ids while retaining a per-run singleton key", async () => { + // Break caught: admission work could be duplicated or leak a discovery payload into the private queue. + const queue = createDiscoveryQueue({ + connectionString: "postgres://worker:secret@database/slashwho" + }); + const runId = "00000000-0000-4000-8000-000000000004"; + const delivered: string[] = []; + + await queue.start(); + await queue.enqueue({ + runId, + key: { region: "eu", realm: "silvermoon", name: "root" } + }); + await queue.enqueueFingerprintAdmission(runId); + await queue.workFingerprintAdmissions(async (deliveredRunId) => { + delivered.push(deliveredRunId); + }); + + const worker = queueFakes.workers.find( + ({ name }) => name === fingerprintAdmissionQueueName + ); + await worker?.handler([{ data: { runId } }]); + + expect(queueFakes.createQueue).toHaveBeenCalledWith( + fingerprintAdmissionQueueName, + expect.any(Object) + ); + expect(queueFakes.send).toHaveBeenCalledWith( + fingerprintAdmissionQueueName, + { runId }, + { singletonKey: runId } + ); + expect(queueFakes.send).toHaveBeenCalledWith( + discoverCharacterQueueName, + expect.objectContaining({ runId }), + { singletonKey: runId } + ); + expect(delivered).toEqual([runId]); + + await queue.stop({ graceful: true, timeoutMs: 1 }); + await worker?.handler([{ data: { runId } }]); + expect(delivered).toEqual([runId]); + }); +}); diff --git a/packages/database/src/queue.ts b/packages/database/src/queue.ts index 8469a95..8aed2cc 100644 --- a/packages/database/src/queue.ts +++ b/packages/database/src/queue.ts @@ -3,12 +3,17 @@ import { PgBoss } from "pg-boss"; export const discoverCharacterQueueName = "discover-character"; export const maintenanceCleanupQueueName = "maintenance-cleanup"; +export const fingerprintAdmissionQueueName = "fingerprint-admission"; export type DiscoverCharacterJob = { runId: string; key: CharacterKey; }; +type FingerprintAdmissionJob = { + runId: string; +}; + export type DiscoveryWorkContext = { attempt: number; maxAttempts: number; @@ -27,12 +32,16 @@ export class DiscoveryQueueStopTimeoutError extends Error { export interface DiscoveryQueue { start(): Promise; enqueue(payload: DiscoverCharacterJob): Promise; + enqueueFingerprintAdmission(runId: string): Promise; work( handler: ( payload: DiscoverCharacterJob, context: DiscoveryWorkContext ) => Promise ): Promise; + workFingerprintAdmissions( + handler: (runId: string) => Promise + ): Promise; scheduleMaintenanceCleanup(handler: () => Promise): Promise; stop(options: { graceful: boolean; timeoutMs: number }): Promise; isReady(): boolean; @@ -50,7 +59,47 @@ const queueOptions = { expireInSeconds: 1_800 } as const; -function requestedRetryDelaySeconds(error: unknown): number | null { +const exclusiveQueuePolicyMigration = ` +DO $slashwho_queue_upgrade$ +BEGIN + LOCK TABLE pgboss.queue IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE pgboss.job IN SHARE ROW EXCLUSIVE MODE; + + WITH ranked AS ( + SELECT name, id, + row_number() OVER ( + PARTITION BY name, COALESCE(singleton_key, '') + ORDER BY (state = 'active') DESC, created_on, id + ) AS position + FROM pgboss.job + WHERE name IN ('discover-character', 'fingerprint-admission') + AND state < 'completed' + ) + UPDATE pgboss.job AS job + SET state = 'cancelled', completed_on = now() + FROM ranked + WHERE job.name = ranked.name + AND job.id = ranked.id + AND ranked.position > 1; + + UPDATE pgboss.job + SET policy = 'exclusive' + WHERE name IN ('discover-character', 'fingerprint-admission') + AND state < 'completed' + AND policy <> 'exclusive'; + + UPDATE pgboss.queue + SET policy = 'exclusive', updated_on = now() + WHERE name IN ('discover-character', 'fingerprint-admission') + AND policy <> 'exclusive'; +END +$slashwho_queue_upgrade$; +`; + +function requestedRetryDelaySeconds( + error: unknown, + maximumDelaySeconds: number = queueOptions.retryDelayMax +): number | null { if ( typeof error !== "object" || error === null || @@ -65,7 +114,7 @@ function requestedRetryDelaySeconds(error: unknown): number | null { const retryDelaySeconds = error.retryAfterMs / 1_000; return Number.isInteger(retryDelaySeconds) && retryDelaySeconds >= 1 && - retryDelaySeconds <= queueOptions.retryDelayMax + retryDelaySeconds <= maximumDelaySeconds ? retryDelaySeconds : null; } @@ -80,7 +129,8 @@ type SqlExecutor = { export async function updateActiveRetryDelay( db: SqlExecutor, jobId: string, - retryDelaySeconds: number + retryDelaySeconds: number, + queueName = discoverCharacterQueueName ): Promise { const result = await db.executeSql( `UPDATE pgboss.job @@ -90,7 +140,7 @@ export async function updateActiveRetryDelay( AND name = $3 AND state = 'active' RETURNING id`, - [jobId, retryDelaySeconds, discoverCharacterQueueName] + [jobId, retryDelaySeconds, queueName] ); if (result.rows.length !== 1) throw new Error("retry_delay_update_failed"); } @@ -102,6 +152,8 @@ export function createDiscoveryQueue( const inFlight = new Set>(); let ready = false; let maintenanceRegistered = false; + let fingerprintAdmissionsRegistered = false; + let acceptingFingerprintAdmissions = false; async function settleInFlight(timeoutMs: number): Promise { const executions = [...inFlight]; @@ -122,21 +174,83 @@ export function createDiscoveryQueue( } } + async function existingSingletonJobId( + queueName: string, + singletonKey: string + ): Promise { + const result = await boss.getDb().executeSql( + `SELECT id::text AS id FROM pgboss.job + WHERE name = $1 AND singleton_key = $2 + AND state IN ('created', 'retry', 'active') + ORDER BY created_on DESC LIMIT 1`, + [queueName, singletonKey] + ); + const id = result.rows[0]?.id; + return typeof id === "string" ? id : null; + } + return { async start() { await boss.start(); - await boss.createQueue(discoverCharacterQueueName, queueOptions); + await boss.createQueue(discoverCharacterQueueName, { + ...queueOptions, + // pg-boss persists this policy and its singleton-key index, so duplicate + // recovery sends from a restarted worker remain one durable delivery. + policy: "exclusive" + }); await boss.updateQueue(discoverCharacterQueueName, queueOptions); + await boss.createQueue(fingerprintAdmissionQueueName, { + policy: "exclusive", + retryLimit: 2_147_483_647, + retryDelay: 60, + expireInSeconds: 300 + }); + // pg-boss deliberately makes createQueue idempotent and forbids changing + // policy through updateQueue. Migrate deployed queues and their runnable + // jobs atomically before this worker accepts sends or registers work. + await boss.getDb().executeSql(exclusiveQueuePolicyMigration); + await boss.updateQueue(fingerprintAdmissionQueueName, { + retryLimit: 2_147_483_647, + retryDelay: 60, + expireInSeconds: 300 + }); + acceptingFingerprintAdmissions = true; ready = true; }, async enqueue(payload) { if (!ready) throw new Error("discovery_queue_not_ready"); const id = await boss.send(discoverCharacterQueueName, payload, { - id: payload.runId, singletonKey: payload.runId }); - return id ?? payload.runId; + return ( + id ?? + (await existingSingletonJobId( + discoverCharacterQueueName, + payload.runId + )) ?? + (() => { + throw new Error("discovery_queue_enqueue_not_created"); + })() + ); + }, + + async enqueueFingerprintAdmission(runId) { + if (!ready) throw new Error("discovery_queue_not_ready"); + const id = await boss.send( + fingerprintAdmissionQueueName, + { runId }, + { + singletonKey: runId + } + ); + return ( + id ?? + (await existingSingletonJobId(fingerprintAdmissionQueueName, runId)) ?? + (() => { + throw new Error("fingerprint_admission_enqueue_not_created"); + })() + ); }, async work(handler) { @@ -179,6 +293,48 @@ export function createDiscoveryQueue( ); }, + async workFingerprintAdmissions(handler) { + if (!ready) throw new Error("discovery_queue_not_ready"); + if (fingerprintAdmissionsRegistered) return; + await boss.work< + FingerprintAdmissionJob, + void, + { pollingIntervalSeconds: number; includeMetadata: true } + >( + fingerprintAdmissionQueueName, + { pollingIntervalSeconds: 0.5, includeMetadata: true }, + async ([job]) => { + if (!job || !acceptingFingerprintAdmissions) return; + const execution = (async () => { + try { + await handler(job.data.runId); + } catch (error) { + const retryDelaySeconds = requestedRetryDelaySeconds( + error, + 86_400 + ); + if (retryDelaySeconds !== null) { + await updateActiveRetryDelay( + boss.getDb(), + job.id, + retryDelaySeconds, + fingerprintAdmissionQueueName + ); + } + throw error; + } + })(); + inFlight.add(execution); + try { + await execution; + } finally { + inFlight.delete(execution); + } + } + ); + fingerprintAdmissionsRegistered = true; + }, + async scheduleMaintenanceCleanup(handler) { if (!ready) throw new Error("discovery_queue_not_ready"); if (maintenanceRegistered) return; @@ -217,6 +373,8 @@ export function createDiscoveryQueue( async stop({ graceful, timeoutMs }) { ready = false; maintenanceRegistered = false; + fingerprintAdmissionsRegistered = false; + acceptingFingerprintAdmissions = false; let stopError: unknown; try { await boss.stop({ graceful, timeout: timeoutMs }); diff --git a/packages/database/src/repositories.ts b/packages/database/src/repositories.ts index f02f630..ab964c2 100644 --- a/packages/database/src/repositories.ts +++ b/packages/database/src/repositories.ts @@ -7,7 +7,7 @@ import type { CharacterKey } from "@slashwho/domain"; export type CallerClass = "anonymous" | "bot"; export type DiscoverySource = - "input" | "claimed" | "declared_main" | "profile_guess"; + "input" | "claimed" | "declared_main" | "profile_guess" | "fingerprint"; export interface DiscoveryRun { id: string; @@ -76,6 +76,15 @@ export interface SnapshotRepository { input: CreateSnapshotInput, options?: { signal?: AbortSignal } ): Promise; + createAndFinishFingerprintSweep( + input: CreateSnapshotInput, + fingerprint: { + reservationId: string; + finishedAt: Date; + limitationCode: string | null; + }, + options?: { signal?: AbortSignal } + ): Promise; getCurrent(key: CharacterKey): Promise; find(id: string): Promise; listHistory( @@ -123,6 +132,45 @@ export interface NegativeCacheRepository { cleanupExpired(at?: Date): Promise; } +export type FingerprintAdmission = + | { kind: "not_due" } + | { kind: "waiting"; retryAt: Date; blockedSince?: Date } + | { + kind: "admitted"; + reservationId: string; + requestCap: number; + committedRequests?: number; + hourlyBudget?: number; + }; + +export type FingerprintAdmissionDispatch = + | { kind: "admitted" } + | { kind: "waiting"; retryAt: Date; blockedSince?: Date } + | { kind: "not_due" } + | { kind: "settled" }; + +export interface FingerprintSweepRepository { + requestAdmission(input: { + runId: string; + key: CharacterKey; + requestCap: number; + hourlyBudget: number; + cadenceCutoff: Date; + at: Date; + }): Promise; + recordRequest(reservationId: string, count: number, at: Date): Promise; + finish( + reservationId: string, + input: { published: boolean; at: Date; limitationCode: string | null } + ): Promise; + release(reservationId: string, at: Date): Promise; + listWaiting(limit: number, offset?: number): Promise; + listAdmittedUndispatched(limit: number): Promise; + markDispatched(runId: string, at: Date): Promise; + admitWaiting(runId: string, at: Date): Promise; + cleanupExpired(at?: Date): Promise; +} + export type SearchReservationResult = | { kind: "active"; run: DiscoveryRun } | { kind: "reserved"; run: DiscoveryRun } @@ -167,4 +215,5 @@ export interface Repositories { suppressions: SuppressionRepository; rateLimits: RateLimitRepository; negativeCache: NegativeCacheRepository; + fingerprintSweeps: FingerprintSweepRepository; } diff --git a/packages/database/src/schema.ts b/packages/database/src/schema.ts index 27015d4..d992be4 100644 --- a/packages/database/src/schema.ts +++ b/packages/database/src/schema.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm"; import { bigserial, + boolean, check, index, integer, @@ -39,7 +40,8 @@ export const discoverySource = pgEnum("discovery_source", [ "input", "claimed", "declared_main", - "profile_guess" + "profile_guess", + "fingerprint" ]); export const characters = pgTable( @@ -214,3 +216,123 @@ export const negativeCharacterCache = pgTable( index("negative_character_cache_expiry_idx").on(table.expiresAt) ] ); + +export const fingerprintSweepStates = pgTable( + "fingerprint_sweep_states", + { + region: text("region").notNull(), + realmSlug: text("realm_slug").notNull(), + normalizedName: text("normalized_name").notNull(), + lastPublishedAt: timestamp("last_published_at", { + withTimezone: true + }) + }, + (table) => [ + primaryKey({ + name: "fingerprint_sweep_states_pkey", + columns: [table.region, table.realmSlug, table.normalizedName] + }) + ] +); + +export const fingerprintSweepAdmissions = pgTable( + "fingerprint_sweep_admissions", + { + id: uuid("id").defaultRandom().primaryKey(), + queueOrder: bigserial("queue_order", { mode: "number" }).notNull(), + discoveryRunId: uuid("discovery_run_id") + .notNull() + .references(() => discoveryRuns.id, { onDelete: "cascade" }), + region: text("region").notNull(), + realmSlug: text("realm_slug").notNull(), + normalizedName: text("normalized_name").notNull(), + requestCap: integer("request_cap").notNull(), + hourlyBudget: integer("hourly_budget").notNull(), + cadenceCutoff: timestamp("cadence_cutoff", { + withTimezone: true + }).notNull(), + status: text("status").default("waiting").notNull(), + dispatchedAt: timestamp("dispatched_at", { withTimezone: true }), + requestedAt: timestamp("requested_at", { withTimezone: true }) + .defaultNow() + .notNull() + }, + (table) => [ + index("fingerprint_sweep_admissions_waiting_idx").on( + table.status, + table.requestedAt, + table.queueOrder + ), + index("fingerprint_sweep_admissions_root_idx").on( + table.region, + table.realmSlug, + table.normalizedName + ), + index("fingerprint_sweep_admissions_dispatch_idx").on( + table.status, + table.dispatchedAt, + table.requestedAt, + table.queueOrder + ), + check( + "fingerprint_sweep_admissions_request_cap_check", + sql`${table.requestCap} > 0` + ), + check( + "fingerprint_sweep_admissions_hourly_budget_check", + sql`${table.hourlyBudget} > 0` + ) + ] +); + +export const fingerprintSweepReservations = pgTable( + "fingerprint_sweep_reservations", + { + id: uuid("id").defaultRandom().primaryKey(), + admissionId: uuid("admission_id") + .notNull() + .references(() => fingerprintSweepAdmissions.id, { onDelete: "cascade" }), + requestCap: integer("request_cap").notNull(), + usedCount: integer("used_count").default(0).notNull(), + admittedAt: timestamp("admitted_at", { withTimezone: true }).notNull(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + releasedAt: timestamp("released_at", { withTimezone: true }), + finishedAt: timestamp("finished_at", { withTimezone: true }), + published: boolean("published"), + limitationCode: text("limitation_code") + }, + (table) => [ + uniqueIndex("fingerprint_sweep_reservations_admission_idx").on( + table.admissionId + ), + index("fingerprint_sweep_reservations_expiry_idx").on(table.expiresAt), + check( + "fingerprint_sweep_reservations_request_cap_check", + sql`${table.requestCap} > 0` + ), + check( + "fingerprint_sweep_reservations_used_count_check", + sql`${table.usedCount} >= 0 AND ${table.usedCount} <= ${table.requestCap}` + ), + check( + "fingerprint_sweep_reservations_expiry_check", + sql`${table.expiresAt} > ${table.admittedAt}` + ) + ] +); + +export const fingerprintSweepRequestEvents = pgTable( + "fingerprint_sweep_request_events", + { + id: uuid("id").defaultRandom().primaryKey(), + reservationId: uuid("reservation_id") + .notNull() + .references(() => fingerprintSweepReservations.id, { + onDelete: "cascade" + }), + requestedAt: timestamp("requested_at", { withTimezone: true }).notNull() + }, + (table) => [ + index("fingerprint_sweep_request_events_window_idx").on(table.requestedAt) + ] +); diff --git a/packages/domain/src/deduplicate.ts b/packages/domain/src/deduplicate.ts index a81288c..128a11c 100644 --- a/packages/domain/src/deduplicate.ts +++ b/packages/domain/src/deduplicate.ts @@ -1,7 +1,7 @@ import type { CharacterKey } from "./character-key"; export type DiscoverySource = - "input" | "claimed" | "declared_main" | "profile_guess"; + "input" | "claimed" | "declared_main" | "profile_guess" | "fingerprint"; export interface DiscoveredCharacter { readonly key: CharacterKey; diff --git a/packages/domain/src/discovery.test.ts b/packages/domain/src/discovery.test.ts index 646f954..1bc6568 100644 --- a/packages/domain/src/discovery.test.ts +++ b/packages/domain/src/discovery.test.ts @@ -240,6 +240,26 @@ describe("discoverCharacter", () => { ).toEqual([altKey]); }); + it("retains a hidden-ownership signal when the request cap wins the limitation", async () => { + // Break caught: a cap-first outcome could erase the sole privacy signal + // needed to prevent later fingerprint-derived links for this root. + const outcome = await discoverCharacter( + altKey, + scriptedGateway({ + characters: [[altKey, character(altKey, { profileGuess: "alias" })]], + profiles: { alias: null, alt: null } + }), + { ...options, requestCap: 1 } + ); + + expect(outcome).toMatchObject({ + kind: "snapshot", + state: "partial", + limitationCode: "request_cap", + privacyHiddenObserved: true + }); + }); + it("treats a non-finite request cap as an exhausted budget", async () => { // Break caught: an invalid cap could silently permit an unbounded crawl, or // publish a rootless snapshot that no read can ever anchor. diff --git a/packages/domain/src/discovery.ts b/packages/domain/src/discovery.ts index fffce6b..e7f487a 100644 --- a/packages/domain/src/discovery.ts +++ b/packages/domain/src/discovery.ts @@ -53,6 +53,8 @@ export type DiscoveryOutcome = kind: "snapshot"; state: "partial"; limitationCode: "privacy_hidden" | "request_cap" | "unsupported_member"; + /** Privacy-hidden ownership was observed even when another limitation won. */ + privacyHiddenObserved?: true; characters: readonly DiscoveredCharacter[]; } | { @@ -340,6 +342,7 @@ export async function discoverCharacter( kind: "snapshot", state: "partial", limitationCode: "request_cap", + ...(privacyHidden ? { privacyHiddenObserved: true as const } : {}), characters }; } diff --git a/packages/domain/src/fingerprint-discovery.test.ts b/packages/domain/src/fingerprint-discovery.test.ts new file mode 100644 index 0000000..bf22317 --- /dev/null +++ b/packages/domain/src/fingerprint-discovery.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, it } from "vitest"; + +import type { CharacterKey } from "./character-key"; +import { + discoverFingerprintMatches, + type FingerprintCandidate, + type FingerprintGateway +} from "."; + +const root: CharacterKey = { + region: "eu", + realm: "silvermoon", + name: "root" +}; + +const matchingKey: CharacterKey = { + region: "eu", + realm: "silvermoon", + name: "matching" +}; + +function fingerprint( + common: number, + identical: number = common +): ReadonlyMap { + return new Map( + Array.from({ length: common }, (_, id) => [id, id < identical ? 1 : 2]) + ); +} + +function candidate(key: CharacterKey): FingerprintCandidate { + return { + key, + displayName: key.name, + className: "Mage", + level: 80 + }; +} + +function keyId(key: CharacterKey): string { + return `${key.region}/${key.realm}/${key.name}`; +} + +function gatewayFor( + roster: readonly FingerprintCandidate[], + fingerprints: Readonly>> +): FingerprintGateway { + return { + async getGuildRoster() { + return roster; + }, + async getAchievementFingerprint(key) { + const value = fingerprints[keyId(key)]; + if (!value) + throw Object.assign(new Error("missing"), { kind: "not_found" }); + return value; + } + }; +} + +const options = { + requestCap: 3, + minimumCommon: 200, + minimumIdenticalPercent: 20, + isSuppressed: async (key: CharacterKey) => key.name === "a-suppressed", + isPrivacyHidden: async (key: CharacterKey) => key.name === "b-hidden" +}; + +describe("discoverFingerprintMatches", () => { + it("fetches the root once, skips suppressed, privacy-hidden, and cross-region candidates, and stops at its cap", async () => { + // Break caught: roster order or excluded candidates could consume the sweep + // budget, preventing an otherwise matching same-region character from being + // admitted before the cap. + const outcome = await discoverFingerprintMatches( + root, + gatewayFor( + [ + candidate({ region: "eu", realm: "silvermoon", name: "z-last" }), + candidate(matchingKey), + candidate({ region: "eu", realm: "silvermoon", name: "b-hidden" }), + candidate({ + region: "eu", + realm: "silvermoon", + name: "a-suppressed" + }), + candidate({ region: "us", realm: "area-52", name: "other-region" }), + candidate(root) + ], + { + [keyId(root)]: fingerprint(200), + [keyId(matchingKey)]: fingerprint(200), + "eu/silvermoon/z-last": fingerprint(200, 0) + } + ), + options + ); + + expect(outcome).toEqual({ + kind: "capped", + requestsUsed: 3, + characters: [ + { + key: matchingKey, + displayName: "matching", + className: "Mage", + level: 80, + raiderIoUrl: "https://raider.io/characters/eu/silvermoon/matching", + source: "fingerprint" + } + ] + }); + }); + + it("enforces the non-configurable matching floors", async () => { + // Break caught: lower caller-provided thresholds could admit a weak + // fingerprint-derived relationship. + const outcome = await discoverFingerprintMatches( + root, + gatewayFor([candidate(matchingKey)], { + [keyId(root)]: fingerprint(199), + [keyId(matchingKey)]: fingerprint(199) + }), + { + ...options, + requestCap: 3, + minimumCommon: 1, + minimumIdenticalPercent: 0, + isSuppressed: async () => false, + isPrivacyHidden: async () => false + } + ); + + expect(outcome).toEqual({ + kind: "matched", + requestsUsed: 3, + characters: [] + }); + }); + + it("does not report a cap when the roster is exhausted exactly at the budget", async () => { + // Break caught: consuming the final allowed request could be mistaken for a + // measured cap stop despite there being no further work to perform. + await expect( + discoverFingerprintMatches( + root, + gatewayFor( + [candidate({ region: "us", realm: "area-52", name: "other-region" })], + { [keyId(root)]: fingerprint(200) } + ), + { ...options, requestCap: 2 } + ) + ).resolves.toEqual({ + kind: "matched", + requestsUsed: 2, + characters: [] + }); + }); + + it("rechecks privacy immediately before admitting a matched candidate", async () => { + // Break caught: a privacy-hidden designation that lands while the candidate + // fingerprint is being fetched could still be retained in the result. + let privacyChecks = 0; + const outcome = await discoverFingerprintMatches( + root, + gatewayFor([candidate(matchingKey)], { + [keyId(root)]: fingerprint(200), + [keyId(matchingKey)]: fingerprint(200) + }), + { + ...options, + isSuppressed: async () => false, + isPrivacyHidden: async () => { + privacyChecks += 1; + return privacyChecks > 1; + } + } + ); + + expect(outcome).toEqual({ + kind: "matched", + requestsUsed: 3, + characters: [] + }); + }); + + it("rechecks suppression immediately before admitting a matched candidate", async () => { + // Break caught: a removal that lands while the candidate fingerprint is + // being fetched could still be retained in the result. + let suppressionChecks = 0; + const outcome = await discoverFingerprintMatches( + root, + gatewayFor([candidate(matchingKey)], { + [keyId(root)]: fingerprint(200), + [keyId(matchingKey)]: fingerprint(200) + }), + { + ...options, + isSuppressed: async () => { + suppressionChecks += 1; + return suppressionChecks > 1; + }, + isPrivacyHidden: async () => false + } + ); + + expect(outcome).toEqual({ + kind: "matched", + requestsUsed: 3, + characters: [] + }); + }); + + it("throws the abort reason when a suppressing policy callback aborts", async () => { + // Break caught: a callback that excludes a candidate could bypass the next + // abort checkpoint and let an abandoned sweep return a result. + const aborted = new AbortController(); + const abortReason = new DOMException("drain timeout", "AbortError"); + const outcome = discoverFingerprintMatches( + root, + gatewayFor([candidate(matchingKey)], { + [keyId(root)]: fingerprint(200), + [keyId(matchingKey)]: fingerprint(200) + }), + { + ...options, + isSuppressed: async (key) => { + if (keyId(key) === keyId(matchingKey)) { + aborted.abort(abortReason); + return true; + } + return false; + }, + isPrivacyHidden: async () => false, + signal: aborted.signal + } + ); + + await expect(outcome).rejects.toBe(abortReason); + }); + + it("returns a retryable failure for a 429", async () => { + // Break caught: rate limiting could publish a partial match set instead of + // restarting the atomic sweep through the worker retry path. + const rateLimited = Object.assign(new Error("rate limited"), { + kind: "transient", + status: 429, + retryAfterMs: 30_000 + }); + const gateway = gatewayFor([], { [keyId(root)]: fingerprint(200) }); + gateway.getAchievementFingerprint = async () => { + throw rateLimited; + }; + + await expect( + discoverFingerprintMatches(root, gateway, options) + ).resolves.toEqual({ + kind: "failure", + code: "upstream_unavailable", + retryable: true, + retryAfterMs: 30_000 + }); + }); + + it("throws the abort reason without returning a partial result", async () => { + // Break caught: cancellation after an upstream response could continue the + // sweep and expose observations from an abandoned atomic run. + const aborted = new AbortController(); + const gateway = gatewayFor([], { [keyId(root)]: fingerprint(200) }); + gateway.getGuildRoster = async () => { + aborted.abort(new DOMException("drain timeout", "AbortError")); + return []; + }; + + await expect( + discoverFingerprintMatches(root, gateway, { + ...options, + signal: aborted.signal + }) + ).rejects.toBe(aborted.signal.reason); + }); +}); diff --git a/packages/domain/src/fingerprint-discovery.ts b/packages/domain/src/fingerprint-discovery.ts new file mode 100644 index 0000000..58eef1b --- /dev/null +++ b/packages/domain/src/fingerprint-discovery.ts @@ -0,0 +1,290 @@ +import { toRaiderIoUrl, type CharacterKey } from "./character-key"; +import { canonicalCharacterId, type DiscoveredCharacter } from "./deduplicate"; + +const mandatoryMinimumCommon = 200; +const mandatoryMinimumIdenticalPercent = 20; +const budgetExhausted = Symbol("budget_exhausted"); + +export type FingerprintCandidate = Readonly<{ + key: CharacterKey; + displayName: string; + className: string; + level: number; +}>; + +export interface FingerprintGateway { + getGuildRoster( + root: CharacterKey, + signal?: AbortSignal + ): Promise; + getAchievementFingerprint( + key: CharacterKey, + signal?: AbortSignal + ): Promise>; +} + +export type FingerprintSweepOutcome = + | { + kind: "matched"; + characters: readonly DiscoveredCharacter[]; + requestsUsed: number; + } + | { + kind: "capped"; + characters: readonly DiscoveredCharacter[]; + requestsUsed: number; + } + | { + kind: "failure"; + code: "upstream_unavailable" | "upstream_schema_changed"; + retryable: boolean; + retryAfterMs?: number; + }; + +export type DiscoverFingerprintMatchesOptions = { + requestCap: number; + minimumCommon: number; + minimumIdenticalPercent: number; + isSuppressed(key: CharacterKey): Promise; + isPrivacyHidden(key: CharacterKey): Promise; + signal?: AbortSignal; +}; + +function isCharacterKey(value: unknown): value is CharacterKey { + if (typeof value !== "object" || value === null) return false; + + return ( + "region" in value && + typeof value.region === "string" && + "realm" in value && + typeof value.realm === "string" && + "name" in value && + typeof value.name === "string" + ); +} + +function isCandidate(value: unknown): value is FingerprintCandidate { + if (typeof value !== "object" || value === null) return false; + + return ( + "key" in value && + isCharacterKey(value.key) && + "displayName" in value && + typeof value.displayName === "string" && + "className" in value && + typeof value.className === "string" && + "level" in value && + typeof value.level === "number" + ); +} + +function isCandidateList( + value: unknown +): value is readonly FingerprintCandidate[] { + return Array.isArray(value) && value.every(isCandidate); +} + +function isFingerprint(value: unknown): value is ReadonlyMap { + return ( + value instanceof Map && + [...value].every( + ([achievementId, timestamp]) => + typeof achievementId === "number" && + Number.isFinite(achievementId) && + typeof timestamp === "number" && + Number.isFinite(timestamp) + ) + ); +} + +function compareCandidates( + left: FingerprintCandidate, + right: FingerprintCandidate +): number { + return canonicalCharacterId(left.key).localeCompare( + canonicalCharacterId(right.key) + ); +} + +function fingerprintMatches( + root: ReadonlyMap, + candidate: ReadonlyMap, + options: DiscoverFingerprintMatchesOptions +): boolean { + let common = 0; + let identical = 0; + + for (const [achievementId, timestamp] of root) { + const candidateTimestamp = candidate.get(achievementId); + if (candidateTimestamp === undefined) continue; + + common += 1; + if (candidateTimestamp === timestamp) identical += 1; + } + + const identicalPercent = common === 0 ? 0 : (identical / common) * 100; + return ( + common >= Math.max(mandatoryMinimumCommon, options.minimumCommon) && + identicalPercent >= + Math.max( + mandatoryMinimumIdenticalPercent, + options.minimumIdenticalPercent + ) + ); +} + +function discoveredCharacter( + candidate: FingerprintCandidate +): DiscoveredCharacter { + return { + key: candidate.key, + displayName: candidate.displayName, + className: candidate.className, + level: candidate.level, + raiderIoUrl: toRaiderIoUrl(candidate.key), + source: "fingerprint" + }; +} + +function failureOutcome(error: unknown): FingerprintSweepOutcome { + const kind = + typeof error === "object" && error !== null && "kind" in error + ? error.kind + : undefined; + + if (kind === "schema_drift") { + return { + kind: "failure", + code: "upstream_schema_changed", + retryable: false + }; + } + + const retryAfterMs = + typeof error === "object" && + error !== null && + "retryAfterMs" in error && + typeof error.retryAfterMs === "number" && + Number.isFinite(error.retryAfterMs) && + error.retryAfterMs >= 0 + ? error.retryAfterMs + : undefined; + + return { + kind: "failure", + code: "upstream_unavailable", + retryable: true, + ...(retryAfterMs === undefined ? {} : { retryAfterMs }) + }; +} + +export async function discoverFingerprintMatches( + root: CharacterKey, + gateway: FingerprintGateway, + options: DiscoverFingerprintMatchesOptions +): Promise { + let remainingRequests = Number.isFinite(options.requestCap) + ? Math.max(0, Math.floor(options.requestCap)) + : 0; + let requestsUsed = 0; + let capped = false; + const matches: DiscoveredCharacter[] = []; + + function throwIfAborted(): void { + options.signal?.throwIfAborted(); + } + + async function request( + operation: () => Promise + ): Promise { + throwIfAborted(); + if (remainingRequests === 0) { + capped = true; + return budgetExhausted; + } + + remainingRequests -= 1; + requestsUsed += 1; + const result = await operation(); + throwIfAborted(); + return result; + } + + try { + const roster = await request(() => + gateway.getGuildRoster(root, options.signal) + ); + if (roster === budgetExhausted) { + return { kind: "capped", characters: [], requestsUsed }; + } + if (!isCandidateList(roster)) throw { kind: "schema_drift" }; + + const rootFingerprint = await request(() => + gateway.getAchievementFingerprint(root, options.signal) + ); + if (rootFingerprint === budgetExhausted) { + return { kind: "capped", characters: [], requestsUsed }; + } + if (!isFingerprint(rootFingerprint)) throw { kind: "schema_drift" }; + + const rootId = canonicalCharacterId(root); + const candidates = [...roster].sort(compareCandidates); + const seen = new Set(); + for (const candidate of candidates) { + throwIfAborted(); + const candidateId = canonicalCharacterId(candidate.key); + if ( + candidateId === rootId || + seen.has(candidateId) || + candidate.key.region !== root.region + ) { + continue; + } + seen.add(candidateId); + + const isSuppressed = await options.isSuppressed(candidate.key); + throwIfAborted(); + if (isSuppressed) continue; + const isPrivacyHidden = await options.isPrivacyHidden(candidate.key); + throwIfAborted(); + if (isPrivacyHidden) continue; + + const candidateFingerprint = await request(() => + gateway.getAchievementFingerprint(candidate.key, options.signal) + ); + if (candidateFingerprint === budgetExhausted) break; + if (!isFingerprint(candidateFingerprint)) throw { kind: "schema_drift" }; + + if (!fingerprintMatches(rootFingerprint, candidateFingerprint, options)) { + continue; + } + const isSuppressedBeforeAdmission = await options.isSuppressed( + candidate.key + ); + throwIfAborted(); + if (isSuppressedBeforeAdmission) continue; + const isPrivacyHiddenBeforeAdmission = await options.isPrivacyHidden( + candidate.key + ); + throwIfAborted(); + if (isPrivacyHiddenBeforeAdmission) continue; + + matches.push(discoveredCharacter(candidate)); + } + } catch (error) { + if (options.signal?.aborted) throw options.signal.reason; + if ( + typeof error === "object" && + error !== null && + "kind" in error && + error.kind === "fingerprint_cap_reached" + ) { + return { kind: "capped", characters: matches, requestsUsed }; + } + return failureOutcome(error); + } + + return capped + ? { kind: "capped", characters: matches, requestsUsed } + : { kind: "matched", characters: matches, requestsUsed }; +} diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index d835918..c15aa31 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -15,3 +15,10 @@ export type { RaiderIoGateway, RaiderIoProfile } from "./discovery"; +export { discoverFingerprintMatches } from "./fingerprint-discovery"; +export type { + DiscoverFingerprintMatchesOptions, + FingerprintCandidate, + FingerprintGateway, + FingerprintSweepOutcome +} from "./fingerprint-discovery"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f4c0b34..e7c1ee1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -120,6 +120,9 @@ importers: '@slashwho/application': specifier: workspace:* version: link:../../packages/application + '@slashwho/blizzard': + specifier: workspace:* + version: link:../../packages/blizzard '@slashwho/database': specifier: workspace:* version: link:../../packages/database @@ -145,6 +148,9 @@ importers: packages/application: dependencies: + '@slashwho/blizzard': + specifier: workspace:* + version: link:../blizzard '@slashwho/contracts': specifier: workspace:* version: link:../contracts @@ -158,6 +164,12 @@ importers: specifier: ^4.3.5 version: 4.4.3 + packages/blizzard: + dependencies: + '@slashwho/domain': + specifier: workspace:* + version: link:../domain + packages/contracts: dependencies: zod: diff --git a/tests/e2e/support/fake-blizzard.ts b/tests/e2e/support/fake-blizzard.ts new file mode 100644 index 0000000..dd3ab69 --- /dev/null +++ b/tests/e2e/support/fake-blizzard.ts @@ -0,0 +1,60 @@ +import { createServer, type Server } from "node:http"; + +type FakeBlizzard = Readonly<{ + baseUrl: string; + close(): Promise; +}>; + +async function listen(server: Server): Promise { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("fake_blizzard_address_unavailable"); + } + return address.port; +} + +export async function startFakeBlizzard(): Promise { + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://fixture.invalid"); + response.setHeader("content-type", "application/json"); + + if (request.method === "POST" && url.pathname === "/token") { + response.end( + JSON.stringify({ access_token: "e2e-access-token", expires_in: 3600 }) + ); + return; + } + + if (request.method === "GET" && url.pathname.endsWith("/achievements")) { + response.end(JSON.stringify({ achievements: [] })); + return; + } + + if ( + request.method === "GET" && + url.pathname.startsWith("/profile/wow/character/") + ) { + // No guild means the sweep only fingerprints its root character. + response.end(JSON.stringify({})); + return; + } + + response.statusCode = 404; + response.end(JSON.stringify({ status: 404 })); + }); + const port = await listen(server); + return { + baseUrl: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }) + }; +} diff --git a/tests/e2e/support/global-setup.ts b/tests/e2e/support/global-setup.ts index 30439fb..694ff1f 100644 --- a/tests/e2e/support/global-setup.ts +++ b/tests/e2e/support/global-setup.ts @@ -5,6 +5,7 @@ import { import { spawn, type ChildProcess } from "node:child_process"; import process from "node:process"; +import { startFakeBlizzard } from "./fake-blizzard"; import { startFakeRaiderIo } from "./fake-raiderio"; type ManagedProcess = Readonly<{ @@ -89,6 +90,7 @@ async function stopProcess(processHandle: ManagedProcess): Promise { export default async function globalSetup(): Promise<() => Promise> { let postgres: StartedPostgreSqlContainer | undefined; let fixture: Awaited> | undefined; + let blizzard: Awaited> | undefined; const processes: ManagedProcess[] = []; try { @@ -98,6 +100,7 @@ export default async function globalSetup(): Promise<() => Promise> { .withPassword("slashwho") .start(); fixture = await startFakeRaiderIo(); + blizzard = await startFakeBlizzard(); const databaseUrl = postgres.getConnectionUri(); process.env.E2E_DATABASE_URL = databaseUrl; process.env.E2E_RAIDER_IO_BASE_URL = fixture.baseUrl; @@ -117,7 +120,13 @@ export default async function globalSetup(): Promise<() => Promise> { RAIDER_IO_BASE_URL: fixture.baseUrl, RAIDER_IO_TIMEOUT_MS: "30000", DATABASE_STARTUP_ATTEMPTS: "10", - DATABASE_STARTUP_RETRY_MS: "250" + DATABASE_STARTUP_RETRY_MS: "250", + // These are deliberately inert, non-secret fixtures. The worker validates + // its Blizzard sweep credentials before it can expose readiness. + BLIZZARD_CLIENT_ID: "e2e-blizzard-client-id", + BLIZZARD_CLIENT_SECRET: "e2e-blizzard-client-secret", + BLIZZARD_SWEEP_REQUEST_CAP: "12", + BLIZZARD_BASE_URL: blizzard.baseUrl }; const worker = startPnpm(["--filter", "@slashwho/worker", "dev"], { @@ -145,12 +154,17 @@ export default async function globalSetup(): Promise<() => Promise> { return async () => { await Promise.allSettled(processes.map(stopProcess)); - await Promise.allSettled([fixture!.close(), postgres!.stop()]); + await Promise.allSettled([ + fixture!.close(), + blizzard!.close(), + postgres!.stop() + ]); }; } catch (error) { await Promise.allSettled(processes.map(stopProcess)); await Promise.allSettled([ ...(fixture ? [fixture.close()] : []), + ...(blizzard ? [blizzard.close()] : []), ...(postgres ? [postgres.stop()] : []) ]); throw error; diff --git a/tests/integration/migrations.test.ts b/tests/integration/migrations.test.ts index e599913..19aed6d 100644 --- a/tests/integration/migrations.test.ts +++ b/tests/integration/migrations.test.ts @@ -28,6 +28,10 @@ describe("database migrations", () => { expect(result.rows.map(({ name }) => name)).toEqual([ "characters", "discovery_runs", + "fingerprint_sweep_admissions", + "fingerprint_sweep_request_events", + "fingerprint_sweep_reservations", + "fingerprint_sweep_states", "negative_character_cache", "rate_limit_events", "snapshot_characters", diff --git a/tests/integration/queue.test.ts b/tests/integration/queue.test.ts index 36ffbe2..48f7ab2 100644 --- a/tests/integration/queue.test.ts +++ b/tests/integration/queue.test.ts @@ -13,6 +13,7 @@ import { startPostgres } from "./postgres"; const queueName = "discover-character"; const maintenanceQueueName = "maintenance-cleanup"; +const fingerprintAdmissionQueueName = "fingerprint-admission"; const key: CharacterKey = { region: "eu", realm: "silvermoon", @@ -46,8 +47,8 @@ describe("durable discovery queue", () => { afterEach(async () => { await Promise.allSettled(cleanup.splice(0).map((stop) => stop())); - await applicationPool.query("DELETE FROM pgboss.job WHERE name = $1", [ - queueName + await applicationPool.query("DELETE FROM pgboss.job WHERE name = ANY($1)", [ + [queueName, fingerprintAdmissionQueueName] ]); }); @@ -55,6 +56,86 @@ describe("durable discovery queue", () => { await stopPostgres(); }); + it("upgrades deployed standard and stately queues to exclusive without losing work", async () => { + // Break caught: createQueue is a no-op for deployed queues and updateQueue + // cannot change policy, leaving singleton-key enqueue retries duplicated. + const runId = "00000000-0000-4000-8000-000000000020"; + const admissionRunId = "00000000-0000-4000-8000-000000000021"; + const legacy = new PgBoss(connectionString); + cleanup.push(() => legacy.stop({ graceful: false, timeout: 1_000 })); + await legacy.start(); + await legacy.createQueue(queueName, { policy: "standard" }); + await legacy.createQueue(fingerprintAdmissionQueueName, { + policy: "stately" + }); + const legacyJobIds = await Promise.all([ + legacy.send(queueName, { runId, key }, { singletonKey: runId }), + legacy.send(queueName, { runId, key }, { singletonKey: runId }) + ]); + expect(new Set(legacyJobIds).size).toBe(2); + const legacyAdmissionId = await legacy.send( + fingerprintAdmissionQueueName, + { runId: admissionRunId }, + { singletonKey: admissionRunId } + ); + await legacy.stop({ graceful: false, timeout: 1_000 }); + + const queue = createDiscoveryQueue({ connectionString }); + cleanup.push(() => queue.stop({ graceful: false, timeoutMs: 1_000 })); + await queue.start(); + + const deployedQueues = await applicationPool.query<{ + name: string; + policy: string; + }>( + `SELECT name, policy + FROM pgboss.queue + WHERE name = ANY($1) + ORDER BY name`, + [[queueName, fingerprintAdmissionQueueName]] + ); + expect(deployedQueues.rows).toEqual([ + { name: queueName, policy: "exclusive" }, + { name: fingerprintAdmissionQueueName, policy: "exclusive" } + ]); + + const runnableJobs = await applicationPool.query<{ + id: string; + name: string; + policy: string; + }>( + `SELECT id::text, name, policy + FROM pgboss.job + WHERE name = ANY($1) + AND state IN ('created', 'retry', 'active') + ORDER BY name`, + [[queueName, fingerprintAdmissionQueueName]] + ); + expect(runnableJobs.rows).toEqual([ + { + id: expect.any(String), + name: queueName, + policy: "exclusive" + }, + { + id: legacyAdmissionId, + name: fingerprintAdmissionQueueName, + policy: "exclusive" + } + ]); + + const recoveredId = await queue.enqueue({ runId, key }); + expect(legacyJobIds).toContain(recoveredId); + const duplicateCount = await applicationPool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM pgboss.job + WHERE name = $1 AND singleton_key = $2 + AND state IN ('created', 'retry', 'active')`, + [queueName, runId] + ); + expect(duplicateCount.rows[0]?.count).toBe("1"); + }); + it("delivers one job once across two concurrent worker processes", async () => { // Break caught: separate workers could both execute the same durable job. const first = createDiscoveryQueue({ connectionString }); @@ -117,6 +198,37 @@ describe("durable discovery queue", () => { ).resolves.toHaveLength(1); }); + it("keeps one private admission job across concurrent restarted queues", async () => { + // Break caught: startup recovery could create an extra private admission + // delivery after another worker already persisted the same singleton key. + const runId = "00000000-0000-4000-8000-000000000010"; + const first = createDiscoveryQueue({ connectionString }); + cleanup.push(() => first.stop({ graceful: false, timeoutMs: 1_000 })); + await first.start(); + await first.enqueueFingerprintAdmission(runId); + await first.stop({ graceful: false, timeoutMs: 1_000 }); + + const restarted = createDiscoveryQueue({ connectionString }); + const replica = createDiscoveryQueue({ connectionString }); + cleanup.push( + () => restarted.stop({ graceful: false, timeoutMs: 1_000 }), + () => replica.stop({ graceful: false, timeoutMs: 1_000 }) + ); + await Promise.all([restarted.start(), replica.start()]); + await Promise.all([ + restarted.enqueueFingerprintAdmission(runId), + replica.enqueueFingerprintAdmission(runId) + ]); + + const rows = await applicationPool.query<{ count: string }>( + `SELECT count(*)::text AS count + FROM pgboss.job + WHERE name = $1 AND singleton_key = $2 AND state IN ('created', 'active')`, + [fingerprintAdmissionQueueName, runId] + ); + expect(rows.rows[0]?.count).toBe("1"); + }); + it("keeps repeated maintenance scheduling idempotent", async () => { // Break caught: restarts or replicas could create duplicate cleanup schedules. const queue = createDiscoveryQueue({ connectionString }); diff --git a/tests/integration/repositories.test.ts b/tests/integration/repositories.test.ts index cae9862..eb97f62 100644 --- a/tests/integration/repositories.test.ts +++ b/tests/integration/repositories.test.ts @@ -178,6 +178,93 @@ describe("PostgreSQL repositories", () => { expect(counts.rows[0]).toEqual({ characters: "0", snapshots: "0" }); }); + it("rolls back fingerprint cadence completion when merged snapshot publication cannot finish", async () => { + // Break caught: a crash between snapshot completion and cadence advancement + // could make the public snapshot visible while the sweep stayed reusable. + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + await repositories.runs.markRunning(run.id); + const admission = await repositories.fingerprintSweeps.requestAdmission({ + runId: run.id, + key: rootKey, + requestCap: 1, + hourlyBudget: 2, + cadenceCutoff: new Date("2026-08-01T12:00:00.000Z"), + at: new Date("2026-08-08T12:00:00.000Z") + }); + if (admission.kind !== "admitted") throw new Error("sweep_not_admitted"); + + await expect( + repositories.snapshots.createAndFinishFingerprintSweep( + { + runId: run.id, + rootKey, + state: "complete", + limitationCode: null, + refreshedAt: new Date("2026-08-08T12:00:00.000Z"), + characters: [observation(rootKey, "Ryii")] + }, + { + reservationId: "00000000-0000-4000-8000-000000000999", + finishedAt: new Date("2026-08-08T12:00:00.000Z"), + limitationCode: null + } + ) + ).rejects.toThrow("fingerprint_reservation_not_active"); + + await expect( + repositories.snapshots.getCurrent(rootKey) + ).resolves.toBeNull(); + await expect(repositories.runs.find(run.id)).resolves.toMatchObject({ + status: "running", + snapshotId: null + }); + }); + + it("publishes the snapshot and advances fingerprint cadence together", async () => { + // Break caught: a successful combined publication could commit the snapshot + // but leave the next run eligible for another sweep immediately. + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + await repositories.runs.markRunning(run.id); + const at = new Date("2026-08-08T12:00:00.000Z"); + const admission = await repositories.fingerprintSweeps.requestAdmission({ + runId: run.id, + key: rootKey, + requestCap: 1, + hourlyBudget: 2, + cadenceCutoff: new Date("2026-08-01T12:00:00.000Z"), + at + }); + if (admission.kind !== "admitted") throw new Error("sweep_not_admitted"); + + await repositories.snapshots.createAndFinishFingerprintSweep( + { + runId: run.id, + rootKey, + state: "complete", + limitationCode: null, + refreshedAt: at, + characters: [observation(rootKey, "Ryii")] + }, + { + reservationId: admission.reservationId, + finishedAt: at, + limitationCode: null + } + ); + + const nextRun = await repositories.runs.createOrReuse(rootKey, "anonymous"); + await expect( + repositories.fingerprintSweeps.requestAdmission({ + runId: nextRun.id, + key: rootKey, + requestCap: 1, + hourlyBudget: 2, + cadenceCutoff: new Date("2026-08-01T12:00:00.000Z"), + at: new Date("2026-08-08T12:01:00.000Z") + }) + ).resolves.toEqual({ kind: "not_due" }); + }); + it("avoids deadlocks for overlapping snapshots with inverse display order", async () => { const firstRun = await repositories.runs.createOrReuse( rootKey, @@ -492,4 +579,377 @@ describe("PostgreSQL repositories", () => { await repositories.rateLimits.countActive("sha256:active", now) ).toBe(1); }); + + it("admits only the FIFO head when two caps would exceed the rolling budget", async () => { + // Break caught: later sweeps could jump the queue or oversubscribe the global hourly budget. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const at = new Date("2026-08-10T12:00:00.000Z"); + const firstKey = rootKey; + const secondKey = altKey; + const firstRun = await repositories.runs.createOrReuse( + firstKey, + "anonymous" + ); + const secondRun = await repositories.runs.createOrReuse( + secondKey, + "anonymous" + ); + const first = { + runId: firstRun.id, + key: firstKey, + requestCap: 3, + hourlyBudget: 5, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }; + const second = { ...first, runId: secondRun.id, key: secondKey }; + + const admitted = + await repositories.fingerprintSweeps.requestAdmission(first); + expect(admitted).toMatchObject({ kind: "admitted", requestCap: 3 }); + if (admitted.kind !== "admitted") + throw new Error("first_sweep_not_admitted"); + + await expect( + repositories.fingerprintSweeps.requestAdmission(second) + ).resolves.toMatchObject({ kind: "waiting" }); + await expect( + repositories.fingerprintSweeps.listWaiting(10) + ).resolves.toEqual([secondRun.id]); + + await repositories.fingerprintSweeps.finish(admitted.reservationId, { + published: true, + at, + limitationCode: null + }); + + await expect( + repositories.fingerprintSweeps.requestAdmission(second) + ).resolves.toMatchObject({ kind: "admitted", requestCap: 3 }); + }); + + it("atomically returns a budget-waiting discovery run to its unconsumed delivery", async () => { + // Break caught: a crash after persisting private admission could leave the + // run running, or its redispatch could start past the original retry count. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const at = new Date("2026-08-10T12:00:00.000Z"); + const blockerRun = await repositories.runs.createOrReuse( + rootKey, + "anonymous" + ); + const waitingRun = await repositories.runs.createOrReuse( + altKey, + "anonymous" + ); + await repositories.runs.claim(waitingRun.id, 1); + const blocker = await repositories.fingerprintSweeps.requestAdmission({ + runId: blockerRun.id, + key: rootKey, + requestCap: 3, + hourlyBudget: 3, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }); + expect(blocker.kind).toBe("admitted"); + + await expect( + repositories.fingerprintSweeps.requestAdmission({ + runId: waitingRun.id, + key: altKey, + requestCap: 1, + hourlyBudget: 3, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }) + ).resolves.toMatchObject({ kind: "waiting" }); + + await expect(repositories.runs.find(waitingRun.id)).resolves.toMatchObject({ + status: "queued", + attempt: 0 + }); + }); + + it("admits a durable waiting run through private admission dispatch after budget frees", async () => { + // Break caught: waiting sweeps could need another discovery delivery instead of being admitted privately. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const at = new Date("2026-08-10T12:00:00.000Z"); + const firstRun = await repositories.runs.createOrReuse( + rootKey, + "anonymous" + ); + const waitingRun = await repositories.runs.createOrReuse( + altKey, + "anonymous" + ); + const first = { + runId: firstRun.id, + key: rootKey, + requestCap: 3, + hourlyBudget: 5, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }; + const waiting = { ...first, runId: waitingRun.id, key: altKey }; + const admitted = + await repositories.fingerprintSweeps.requestAdmission(first); + if (admitted.kind !== "admitted") + throw new Error("first_sweep_not_admitted"); + await expect( + repositories.fingerprintSweeps.requestAdmission(waiting) + ).resolves.toMatchObject({ kind: "waiting" }); + + await repositories.fingerprintSweeps.release(admitted.reservationId, at); + + await expect( + repositories.fingerprintSweeps.admitWaiting( + waitingRun.id, + new Date("2026-08-10T12:01:00.000Z") + ) + ).resolves.toEqual({ kind: "admitted" }); + await expect( + repositories.fingerprintSweeps.requestAdmission({ + ...waiting, + at: new Date("2026-08-10T12:01:00.000Z") + }) + ).resolves.toMatchObject({ kind: "admitted", requestCap: 3 }); + }); + + it("keeps an admitted sweep dispatch-pending until its discovery job is durably enqueued", async () => { + // Break caught: a crash after budget reservation could lose a run before discovery is re-enqueued. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const at = new Date("2026-08-10T12:00:00.000Z"); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + await expect( + repositories.fingerprintSweeps.requestAdmission({ + runId: run.id, + key: rootKey, + requestCap: 3, + hourlyBudget: 5, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }) + ).resolves.toMatchObject({ kind: "admitted" }); + + await expect( + repositories.fingerprintSweeps.listAdmittedUndispatched(10) + ).resolves.toEqual([run.id]); + await repositories.fingerprintSweeps.markDispatched(run.id, at); + await expect( + repositories.fingerprintSweeps.listAdmittedUndispatched(10) + ).resolves.toEqual([]); + }); + + it("does not advance cadence or retain unused capacity after an aborted sweep", async () => { + // Break caught: aborts could consume future cadence or the entire unused reservation. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const at = new Date("2026-08-10T12:00:00.000Z"); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + const input = { + runId: run.id, + key: rootKey, + requestCap: 5, + hourlyBudget: 8, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }; + const admitted = + await repositories.fingerprintSweeps.requestAdmission(input); + expect(admitted).toMatchObject({ kind: "admitted" }); + if (admitted.kind !== "admitted") throw new Error("sweep_not_admitted"); + + await repositories.fingerprintSweeps.recordRequest( + admitted.reservationId, + 3, + at + ); + await repositories.fingerprintSweeps.release(admitted.reservationId, at); + + await expect( + repositories.fingerprintSweeps.requestAdmission({ + ...input, + at: new Date("2026-08-10T12:01:00.000Z") + }) + ).resolves.toMatchObject({ kind: "admitted", requestCap: 5 }); + }); + + it("prunes fingerprint request events only once they leave the rolling hour", async () => { + // Break caught: one row per Blizzard request accumulates without limit, and + // a prune keyed on the reservation would delete events the rolling-hour + // budget still has to count. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_request_events, + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const sweptRun = await repositories.runs.createOrReuse( + rootKey, + "anonymous" + ); + const admitted = await repositories.fingerprintSweeps.requestAdmission({ + runId: sweptRun.id, + key: rootKey, + requestCap: 3, + hourlyBudget: 3, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at: new Date("2026-08-10T12:00:00.000Z") + }); + if (admitted.kind !== "admitted") throw new Error("sweep_not_admitted"); + await repositories.fingerprintSweeps.recordRequest( + admitted.reservationId, + 1, + new Date("2026-08-10T12:10:00.000Z") + ); + const lastRequestedAt = new Date("2026-08-10T12:55:00.000Z"); + await repositories.fingerprintSweeps.recordRequest( + admitted.reservationId, + 2, + lastRequestedAt + ); + await repositories.fingerprintSweeps.release( + admitted.reservationId, + lastRequestedAt + ); + await repositories.runs.fail(sweptRun.id, "upstream_unavailable"); + + const at = new Date("2026-08-10T13:20:00.000Z"); + await expect( + repositories.fingerprintSweeps.cleanupExpired(at) + ).resolves.toBe(1); + const retained = await pool.query<{ requested_at: Date }>( + `SELECT requested_at FROM fingerprint_sweep_request_events + ORDER BY requested_at` + ); + expect(retained.rows.map((row) => row.requested_at)).toEqual([ + lastRequestedAt, + lastRequestedAt + ]); + + const nextRun = await repositories.runs.createOrReuse(rootKey, "anonymous"); + await expect( + repositories.fingerprintSweeps.requestAdmission({ + runId: nextRun.id, + key: rootKey, + requestCap: 2, + hourlyBudget: 3, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }) + ).resolves.toMatchObject({ + kind: "waiting", + retryAt: new Date("2026-08-10T13:55:00.000Z") + }); + }); + + it("retains each physical fingerprint request for its own rolling hour", async () => { + // Break caught: extending a reservation expiry from its admission time can + // undercount late Profile API requests and admit a budget-overlapping sweep. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_request_events, + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const admittedAt = new Date("2026-08-10T12:00:00.000Z"); + const firstRun = await repositories.runs.createOrReuse( + rootKey, + "anonymous" + ); + const admitted = await repositories.fingerprintSweeps.requestAdmission({ + runId: firstRun.id, + key: rootKey, + requestCap: 3, + hourlyBudget: 3, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at: admittedAt + }); + if (admitted.kind !== "admitted") throw new Error("sweep_not_admitted"); + const usedAt = new Date("2026-08-10T12:55:00.000Z"); + await repositories.fingerprintSweeps.recordRequest( + admitted.reservationId, + 3, + usedAt + ); + await repositories.fingerprintSweeps.release( + admitted.reservationId, + usedAt + ); + await repositories.runs.fail(firstRun.id, "upstream_unavailable"); + + const secondRun = await repositories.runs.createOrReuse( + rootKey, + "anonymous" + ); + await expect( + repositories.fingerprintSweeps.requestAdmission({ + runId: secondRun.id, + key: rootKey, + requestCap: 1, + hourlyBudget: 3, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at: new Date("2026-08-10T13:10:00.000Z") + }) + ).resolves.toMatchObject({ + kind: "waiting", + retryAt: new Date("2026-08-10T13:55:00.000Z") + }); + }); + + it("returns not due only after a published sweep within its cadence", async () => { + // Break caught: a partial, unpublished, or aborted sweep could suppress a later sweep. + await pool.query(`TRUNCATE TABLE + fingerprint_sweep_reservations, + fingerprint_sweep_admissions, + fingerprint_sweep_states + CASCADE`); + const at = new Date("2026-08-10T12:00:00.000Z"); + const run = await repositories.runs.createOrReuse(rootKey, "anonymous"); + const admitted = await repositories.fingerprintSweeps.requestAdmission({ + runId: run.id, + key: rootKey, + requestCap: 1, + hourlyBudget: 2, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at + }); + if (admitted.kind !== "admitted") throw new Error("sweep_not_admitted"); + await repositories.fingerprintSweeps.finish(admitted.reservationId, { + published: true, + at, + limitationCode: null + }); + await repositories.runs.fail(run.id, "upstream_unavailable"); + + const nextRun = await repositories.runs.createOrReuse(rootKey, "anonymous"); + await expect( + repositories.fingerprintSweeps.requestAdmission({ + runId: nextRun.id, + key: rootKey, + requestCap: 1, + hourlyBudget: 2, + cadenceCutoff: new Date("2026-08-03T12:00:00.000Z"), + at: new Date("2026-08-10T12:01:00.000Z") + }) + ).resolves.toEqual({ kind: "not_due" }); + }); }); diff --git a/tests/integration/suppression.test.ts b/tests/integration/suppression.test.ts index 3b6965d..807e581 100644 --- a/tests/integration/suppression.test.ts +++ b/tests/integration/suppression.test.ts @@ -226,7 +226,8 @@ describe("application suppression policy", () => { await expect(service.cleanupExpired(now)).resolves.toEqual({ rateLimits: 0, negativeCache: 1, - suppressions: 1 + suppressions: 1, + fingerprintRequests: 0 }); await expect(repositories.suppressions.isActive(root, now)).resolves.toBe( true