From 6d52c8c14d757db6d55b8e54c8451c2ca88439f3 Mon Sep 17 00:00:00 2001 From: zmuhls Date: Tue, 6 Oct 2026 14:39:21 -0400 Subject: [PATCH 01/25] Add durable operation and allowance ledger schema --- migrations/0022-reliable-operations.sql | 107 ++++++++++++++++++++++++ schema.sql | 107 ++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 migrations/0022-reliable-operations.sql diff --git a/migrations/0022-reliable-operations.sql b/migrations/0022-reliable-operations.sql new file mode 100644 index 0000000..61d179f --- /dev/null +++ b/migrations/0022-reliable-operations.sql @@ -0,0 +1,107 @@ +-- Deliberately separate from historical app_request_reservations. No inferred +-- successful charges/backfill. Apply only with the queue-drain rollback plan. +CREATE TABLE IF NOT EXISTS app_operations ( + id TEXT PRIMARY KEY, + subject TEXT NOT NULL REFERENCES app_users(subject), + course_id TEXT, + kind TEXT NOT NULL CHECK(kind IN ('split','chat','guide')), + idempotency_key TEXT NOT NULL, + fingerprint TEXT NOT NULL, + day TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('queued','running','starting','processing','reconciling','succeeded','partial','failed','cancelled')), + phase TEXT NOT NULL CHECK(phase IN ('fetch','split','chat','guide')), + job_id TEXT, + request_json TEXT NOT NULL DEFAULT '{}', + result_json TEXT, + provider_id TEXT, + lease_owner TEXT, + lease_until INTEGER NOT NULL DEFAULT 0, + fence INTEGER NOT NULL DEFAULT 0, + not_before INTEGER NOT NULL DEFAULT 0, + deadline INTEGER NOT NULL, + cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK(cancel_requested IN (0,1)), + error_code TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(subject,kind,idempotency_key) +); +CREATE INDEX IF NOT EXISTS app_operations_allowance ON app_operations(subject,kind,day,state); +CREATE INDEX IF NOT EXISTS app_operations_queue ON app_operations(kind,phase,state,not_before,created_at); +CREATE UNIQUE INDEX IF NOT EXISTS app_operations_split_job ON app_operations(job_id) WHERE kind='split'; +CREATE TABLE IF NOT EXISTS operation_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + operation_id TEXT NOT NULL REFERENCES app_operations(id), + state TEXT NOT NULL, + phase TEXT NOT NULL, + fence INTEGER NOT NULL, + code TEXT, + at INTEGER NOT NULL +); +CREATE TRIGGER IF NOT EXISTS operation_created AFTER INSERT ON app_operations BEGIN + INSERT INTO operation_events(operation_id,state,phase,fence,code,at) + VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at); +END; +CREATE TRIGGER IF NOT EXISTS operation_changed AFTER UPDATE ON app_operations +WHEN OLD.state<>NEW.state OR OLD.phase<>NEW.phase OR OLD.fence<>NEW.fence BEGIN + INSERT INTO operation_events(operation_id,state,phase,fence,code,at) + VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at); +END; +CREATE TRIGGER IF NOT EXISTS operation_terminal_fence BEFORE UPDATE ON app_operations +WHEN OLD.state IN ('succeeded','partial','failed','cancelled') AND NEW.state<>OLD.state +BEGIN SELECT RAISE(ABORT,'terminal operation'); END; +CREATE TRIGGER IF NOT EXISTS operation_identity_fence BEFORE UPDATE ON app_operations +WHEN NEW.subject<>OLD.subject OR NEW.kind<>OLD.kind OR NEW.day<>OLD.day OR + NEW.fingerprint<>OLD.fingerprint OR NEW.idempotency_key<>OLD.idempotency_key OR NEW.course_id IS NOT OLD.course_id +BEGIN SELECT RAISE(ABORT,'immutable operation identity'); END; +CREATE TABLE IF NOT EXISTS operation_attempts ( + id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL REFERENCES app_operations(id), + phase TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT, + outcome TEXT NOT NULL, + external_id TEXT, + http_status INTEGER, + code TEXT, + -- Provider spend is separate from allowance. NULL means unreported, not free. + usage_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS operation_attempts_operation ON operation_attempts(operation_id,phase,created_at); +CREATE TABLE IF NOT EXISTS operation_fairness ( + subject TEXT NOT NULL, + phase TEXT NOT NULL, + last_started INTEGER NOT NULL, + PRIMARY KEY(subject,phase) +); +CREATE TABLE IF NOT EXISTS provider_cooldowns ( + provider TEXT PRIMARY KEY, + until_ms INTEGER NOT NULL +); +-- Only same subject AND exact authorized course may coalesce/reuse imports. +-- Sources with no affirmative rights evidence (including YouTube) never cache. +CREATE TABLE IF NOT EXISTS import_cache ( + scope TEXT NOT NULL, + source TEXT NOT NULL, + operation_id TEXT NOT NULL REFERENCES app_operations(id), + object_key TEXT, + metadata_json TEXT, + expires_at INTEGER NOT NULL, + PRIMARY KEY(scope,source) +); +CREATE TRIGGER IF NOT EXISTS operation_job_done_guard BEFORE UPDATE OF status ON jobs +WHEN NEW.status='done' AND EXISTS(SELECT 1 FROM app_operations o WHERE o.job_id=NEW.id AND o.kind='split' + AND (o.state IN ('failed','cancelled') OR o.cancel_requested=1)) +BEGIN SELECT RAISE(ABORT,'operation no longer accepts success'); END; +CREATE TRIGGER IF NOT EXISTS operation_job_delete_guard BEFORE DELETE ON jobs +WHEN EXISTS(SELECT 1 FROM app_operations WHERE job_id=OLD.id AND kind='split' + AND state NOT IN ('succeeded','partial','failed','cancelled')) +BEGIN SELECT RAISE(ABORT,'cancel and reconcile operation before deleting job'); END; +CREATE TRIGGER IF NOT EXISTS operation_job_settlement AFTER UPDATE OF status ON jobs +WHEN NEW.status IN ('done','failed') BEGIN + UPDATE app_operations SET state=CASE WHEN NEW.status='done' THEN 'succeeded' + WHEN cancel_requested=1 THEN 'cancelled' ELSE 'failed' END, + lease_owner=NULL,lease_until=0,updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 + WHERE job_id=NEW.id AND kind='split' AND state NOT IN ('succeeded','partial','failed','cancelled'); +END; diff --git a/schema.sql b/schema.sql index 159c2d7..c249792 100644 --- a/schema.sql +++ b/schema.sql @@ -679,3 +679,110 @@ BEGIN INSERT INTO course_folder_events(folder_id,course_id,actor,permission,revision) VALUES(NEW.id,NEW.course_id,NEW.created_by,NEW.permission,NEW.revision); END; +-- Deliberately separate from historical app_request_reservations. No inferred +-- successful charges/backfill. Apply only with the queue-drain rollback plan. +CREATE TABLE IF NOT EXISTS app_operations ( + id TEXT PRIMARY KEY, + subject TEXT NOT NULL REFERENCES app_users(subject), + course_id TEXT, + kind TEXT NOT NULL CHECK(kind IN ('split','chat','guide')), + idempotency_key TEXT NOT NULL, + fingerprint TEXT NOT NULL, + day TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('queued','running','starting','processing','reconciling','succeeded','partial','failed','cancelled')), + phase TEXT NOT NULL CHECK(phase IN ('fetch','split','chat','guide')), + job_id TEXT, + request_json TEXT NOT NULL DEFAULT '{}', + result_json TEXT, + provider_id TEXT, + lease_owner TEXT, + lease_until INTEGER NOT NULL DEFAULT 0, + fence INTEGER NOT NULL DEFAULT 0, + not_before INTEGER NOT NULL DEFAULT 0, + deadline INTEGER NOT NULL, + cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK(cancel_requested IN (0,1)), + error_code TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(subject,kind,idempotency_key) +); +CREATE INDEX IF NOT EXISTS app_operations_allowance ON app_operations(subject,kind,day,state); +CREATE INDEX IF NOT EXISTS app_operations_queue ON app_operations(kind,phase,state,not_before,created_at); +CREATE UNIQUE INDEX IF NOT EXISTS app_operations_split_job ON app_operations(job_id) WHERE kind='split'; +CREATE TABLE IF NOT EXISTS operation_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + operation_id TEXT NOT NULL REFERENCES app_operations(id), + state TEXT NOT NULL, + phase TEXT NOT NULL, + fence INTEGER NOT NULL, + code TEXT, + at INTEGER NOT NULL +); +CREATE TRIGGER IF NOT EXISTS operation_created AFTER INSERT ON app_operations BEGIN + INSERT INTO operation_events(operation_id,state,phase,fence,code,at) + VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at); +END; +CREATE TRIGGER IF NOT EXISTS operation_changed AFTER UPDATE ON app_operations +WHEN OLD.state<>NEW.state OR OLD.phase<>NEW.phase OR OLD.fence<>NEW.fence BEGIN + INSERT INTO operation_events(operation_id,state,phase,fence,code,at) + VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at); +END; +CREATE TRIGGER IF NOT EXISTS operation_terminal_fence BEFORE UPDATE ON app_operations +WHEN OLD.state IN ('succeeded','partial','failed','cancelled') AND NEW.state<>OLD.state +BEGIN SELECT RAISE(ABORT,'terminal operation'); END; +CREATE TRIGGER IF NOT EXISTS operation_identity_fence BEFORE UPDATE ON app_operations +WHEN NEW.subject<>OLD.subject OR NEW.kind<>OLD.kind OR NEW.day<>OLD.day OR + NEW.fingerprint<>OLD.fingerprint OR NEW.idempotency_key<>OLD.idempotency_key OR NEW.course_id IS NOT OLD.course_id +BEGIN SELECT RAISE(ABORT,'immutable operation identity'); END; +CREATE TABLE IF NOT EXISTS operation_attempts ( + id TEXT PRIMARY KEY, + operation_id TEXT NOT NULL REFERENCES app_operations(id), + phase TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT, + outcome TEXT NOT NULL, + external_id TEXT, + http_status INTEGER, + code TEXT, + -- Provider spend is separate from allowance. NULL means unreported, not free. + usage_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS operation_attempts_operation ON operation_attempts(operation_id,phase,created_at); +CREATE TABLE IF NOT EXISTS operation_fairness ( + subject TEXT NOT NULL, + phase TEXT NOT NULL, + last_started INTEGER NOT NULL, + PRIMARY KEY(subject,phase) +); +CREATE TABLE IF NOT EXISTS provider_cooldowns ( + provider TEXT PRIMARY KEY, + until_ms INTEGER NOT NULL +); +-- Only same subject AND exact authorized course may coalesce/reuse imports. +-- Sources with no affirmative rights evidence (including YouTube) never cache. +CREATE TABLE IF NOT EXISTS import_cache ( + scope TEXT NOT NULL, + source TEXT NOT NULL, + operation_id TEXT NOT NULL REFERENCES app_operations(id), + object_key TEXT, + metadata_json TEXT, + expires_at INTEGER NOT NULL, + PRIMARY KEY(scope,source) +); +CREATE TRIGGER IF NOT EXISTS operation_job_done_guard BEFORE UPDATE OF status ON jobs +WHEN NEW.status='done' AND EXISTS(SELECT 1 FROM app_operations o WHERE o.job_id=NEW.id AND o.kind='split' + AND (o.state IN ('failed','cancelled') OR o.cancel_requested=1)) +BEGIN SELECT RAISE(ABORT,'operation no longer accepts success'); END; +CREATE TRIGGER IF NOT EXISTS operation_job_delete_guard BEFORE DELETE ON jobs +WHEN EXISTS(SELECT 1 FROM app_operations WHERE job_id=OLD.id AND kind='split' + AND state NOT IN ('succeeded','partial','failed','cancelled')) +BEGIN SELECT RAISE(ABORT,'cancel and reconcile operation before deleting job'); END; +CREATE TRIGGER IF NOT EXISTS operation_job_settlement AFTER UPDATE OF status ON jobs +WHEN NEW.status IN ('done','failed') BEGIN + UPDATE app_operations SET state=CASE WHEN NEW.status='done' THEN 'succeeded' + WHEN cancel_requested=1 THEN 'cancelled' ELSE 'failed' END, + lease_owner=NULL,lease_until=0,updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 + WHERE job_id=NEW.id AND kind='split' AND state NOT IN ('succeeded','partial','failed','cancelled'); +END; From 002fcb9c8448c4a2e143d3fb2afaca35ad44b556 Mon Sep 17 00:00:00 2001 From: zmuhls Date: Tue, 6 Oct 2026 14:39:21 -0400 Subject: [PATCH 02/25] Coordinate split lifecycles and ordered Gateway recovery --- cloudflare/gateway.ts | 39 +++++- cloudflare/worker.ts | 15 ++- cloudflare/wrangler.jsonc | 3 +- replicate-yt-audio/cog.yaml | 7 +- replicate-yt-audio/predict.py | 212 ++++++++++++++++++++------------- src/assistant/tools.ts | 9 +- src/assistant/types.ts | 2 + src/daily-allowance.ts | 21 +++- src/env.ts | 4 + src/index.ts | 70 ++++++++--- src/reliability/assistant.ts | 33 ++++++ src/reliability/ledger.ts | 93 +++++++++++++++ src/reliability/queue.ts | 70 +++++++++++ src/reliability/retention.ts | 13 ++ src/reliability/retry.ts | 28 +++++ src/reliability/splits.ts | 215 ++++++++++++++++++++++++++++++++++ src/separation/replicate.ts | 36 +++--- src/separation/types.ts | 2 + src/youtube.ts | 67 ++++++++--- 19 files changed, 782 insertions(+), 157 deletions(-) create mode 100644 src/reliability/assistant.ts create mode 100644 src/reliability/ledger.ts create mode 100644 src/reliability/queue.ts create mode 100644 src/reliability/retention.ts create mode 100644 src/reliability/retry.ts create mode 100644 src/reliability/splits.ts diff --git a/cloudflare/gateway.ts b/cloudflare/gateway.ts index b4c441b..49811d4 100644 --- a/cloudflare/gateway.ts +++ b/cloudflare/gateway.ts @@ -4,9 +4,18 @@ import { AssistantError, COACH_DOWN, COACH_UNCONFIGURED, type OpenRouterParams, import type { WireStreamChunk, WireToolCall } from '../src/assistant/types.ts'; import type { Env } from '../src/env.ts'; import { boundedText, cancel, deadline, withSignal } from './bounded.ts'; +import { beginAttempt, finishAttempt, readOperation } from '../src/reliability/ledger.ts'; +import { cooldown, retryAt, setCooldown } from '../src/reliability/retry.ts'; export const REQUEST_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[47][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; export const canonicalModel = (value: unknown): value is string => typeof value === 'string' && /^[a-z0-9][a-z0-9._-]{0,127}$/.test(value); +export const GATEWAY_MODELS = ['glm-5.2', 'deepseek-v4-flash-0731'] as const; +export const gatewayModelsConfigured = (primary: unknown, backup: unknown) => primary === GATEWAY_MODELS[0] && backup === GATEWAY_MODELS[1]; +export function eligibleGatewayFallback(error: unknown) { + return error instanceof CailError && error.extras.should_retry === true && + (error.code==='upstream_rate_limited' && error.status===429 || + ['upstream_unavailable','model_unavailable'].includes(error.code) && error.status===503); +} const catalog = defineEventCatalog({ 'stem-splitter.gateway.completed': { source: 'tenant', severity: 'outcome', required: ['request_id', 'terminal'], optional: [] }, }); @@ -45,16 +54,27 @@ export function gatewayForRequest(binding: Fetcher | undefined, token: string | const stream = async (env: Env, params: OpenRouterParams, onDelta: (text: string) => void | Promise): Promise => deadline(async signal => { + if (!binding || !token || !canonicalModel(env.ASSISTANT_MODEL) || + (env.ASSISTANT_FALLBACK_MODELS !== undefined && !gatewayModelsConfigured(env.ASSISTANT_MODEL, env.ASSISTANT_FALLBACK_MODELS))) throw new AssistantError(503,COACH_UNCONFIGURED); + const models = env.ASSISTANT_FALLBACK_MODELS ? GATEWAY_MODELS : [env.ASSISTANT_MODEL]; + for (const [modelIndex, model] of models.entries()) { let reader: ReadableStreamDefaultReader | undefined; + let accepted = false, outputStarted = false; + let attemptId: string | null = null, usage: unknown; + const operation = env.ASSISTANT_OPERATION_ID ? await readOperation(env.DB,env.ASSISTANT_OPERATION_ID) : null; let requestId = correlation.request_id; let outcome: 'ok' | 'cancelled' | 'error' = 'error'; try { - if (!binding || !token || !canonicalModel(env.ASSISTANT_MODEL)) throw new AssistantError(503, COACH_UNCONFIGURED); + if (operation) { + if (await cooldown(env.DB,`gateway:${model}`)>Date.now()) throw new AssistantError(503,COACH_DOWN,{code:'model_cooldown',shouldRetry:true}); + attemptId=await beginAttempt(env.DB,operation,'cail-gateway',model); + } const response = await client.chatCompletions({ - model: env.ASSISTANT_MODEL, messages: params.messages.map(({ role, content }) => ({ role, content })), + model, messages: params.messages.map(({ role, content }) => ({ role, content })), ...(params.tools?.length ? { tools: params.tools.map(tool => ({ type: tool.type, function: { ...tool.function, parameters: JSON.parse(JSON.stringify(tool.function.parameters)) } })), tool_choice: 'auto' } : {}), stream: true, stream_options: { include_usage: true }, max_tokens: params.maxTokens, temperature: params.temperature, }, { kind: 'jwt', token }, { signal, correlation }); + accepted = true; const responseId = response.headers.get('x-request-id') ?? response.headers.get('x-cail-request-id'); if (responseId && REQUEST_ID.test(responseId)) requestId = responseId; reader = response.body?.getReader(); @@ -71,16 +91,19 @@ export function gatewayForRequest(binding: Fetcher | undefined, token: string | if (error) throw error; if (!value || typeof value !== 'object' || 'error' in value) throw new AssistantError(502, COACH_DOWN); const chunk = value as WireStreamChunk; + if ('usage' in value) usage=(value as {usage:unknown}).usage; const choice = chunk.choices?.[0]; if (!choice) return; // Trailing accounting events still get drained. if (choice.finish_reason === 'error') throw new AssistantError(502, COACH_DOWN); if (choice.finish_reason) finishReason = choice.finish_reason; if (choice.delta?.content) { if (typeof choice.delta.content !== 'string') throw new AssistantError(502, COACH_DOWN); + outputStarted = true; content += choice.delta.content; await withSignal(Promise.resolve(onDelta(choice.delta.content)), signal); } for (const delta of choice.delta?.tool_calls ?? []) { + outputStarted = true; if (!Number.isSafeInteger(delta.index) || delta.index! < 0 || delta.index! >= 32) throw new AssistantError(502, COACH_DOWN); const slot = calls.get(delta.index!) ?? { type: 'function', function: { name: '', arguments: '' } }; if (delta.id) slot.id = delta.id; @@ -101,9 +124,17 @@ export function gatewayForRequest(binding: Fetcher | undefined, token: string | } if (buffer.trim()) await consume(buffer); if (!finishReason) throw new AssistantError(502, COACH_DOWN); + const toolIds=[...calls.values()].map(call=>call.id).filter(Boolean); + if (new Set(toolIds).size!==toolIds.length) throw new AssistantError(502,COACH_DOWN,{code:'invalid_tool_calls'}); outcome = 'ok'; - return { content: content.trim(), model: env.ASSISTANT_MODEL, toolCalls: [...calls.entries()].sort(([a], [b]) => a - b).map(([, call]) => call), finishReason }; + if (attemptId) await finishAttempt(env.DB,attemptId,'succeeded',{usage}); + return { content: content.trim(), model, toolCalls: [...calls.entries()].sort(([a], [b]) => a - b).map(([, call]) => call), finishReason }; } catch (error) { + if (attemptId) await finishAttempt(env.DB,attemptId,outputStarted?'partial':accepted?'uncertain':'rejected',{code:error instanceof CailError?error.code:'gateway_failure',usage}); + if (!signal.aborted && !accepted && !outputStarted && modelIndex===0 && models.length===2 && eligibleGatewayFallback(error)) { + if (operation) await setCooldown(env.DB,`gateway:${model}`,retryAt(error instanceof CailError ? String(error.extras.retry_after ?? error.extras.retry_after_seconds ?? '30') : '30')); + continue; + } if (signal.aborted) { if (signal.reason?.name === 'TimeoutError') throw new AssistantError(503, COACH_DOWN, { requestId, code: 'upstream_timeout', shouldRetry: true }); outcome = 'cancelled'; throw signal.reason; @@ -117,6 +148,8 @@ export function gatewayForRequest(binding: Fetcher | undefined, token: string | : { outcome: 'error', reason: 'upstream_failure' } as const; log.emit('stem-splitter.gateway.completed', { request_id: requestId, terminal }); } + } + throw new AssistantError(503,COACH_DOWN); }, env.ASSISTANT_ABORT_SIGNAL ?? request.signal, 60000); return { stream, quota: () => deadline(async signal => { if (!binding || !token) throw new AssistantError(503, COACH_UNCONFIGURED); diff --git a/cloudflare/worker.ts b/cloudflare/worker.ts index 3895537..ab29414 100644 --- a/cloudflare/worker.ts +++ b/cloudflare/worker.ts @@ -1,13 +1,14 @@ -import app from '../src/index.ts'; +import app, { runReliableJobs } from '../src/index.ts'; import type { Env } from '../src/env.ts'; import { verifyCailIdentity } from './verify.ts'; -import { gatewayForRequest, canonicalModel } from './gateway.ts'; +import { gatewayForRequest, canonicalModel, gatewayModelsConfigured } from './gateway.ts'; import { authenticatedRequest, handleAuth, type WorkerIdentity } from './sso.ts'; import { purgeExpiredListeningConversations } from './retention.ts'; +import { purgeOperationContent } from '../src/reliability/retention.ts'; export type WorkerEnv = Omit & Pick & Partial> & - { IDENTITY?: WorkerIdentity; PREVIEW_IDENTITY?: WorkerIdentity; GATEWAY?: Fetcher; GATEWAY_MODEL?: string }; + { IDENTITY?: WorkerIdentity; PREVIEW_IDENTITY?: WorkerIdentity; GATEWAY?: Fetcher; GATEWAY_MODEL?: string; GATEWAY_FALLBACK_MODEL?: string }; const HEADERS = { 'X-Content-Type-Options': 'nosniff', @@ -29,12 +30,14 @@ async function serveApi(request: Request, env: WorkerEnv, ctx: ExecutionContext) } const gateway = gatewayForRequest(env.GATEWAY, request.headers.get('x-cail-gateway-identity-jwt'), request, env.RELEASE || 'candidate'); return app.fetch(request, { ...env, ASSISTANT_MODEL: env.GATEWAY_MODEL, + ASSISTANT_FALLBACK_MODELS: env.GATEWAY_FALLBACK_MODEL ?? '', assistantTransport: gateway.stream, assistantQuota: gateway.quota, verifyCailIdentity }, ctx); } export default { async scheduled(_controller: ScheduledController, env: WorkerEnv, ctx: ExecutionContext): Promise { - ctx.waitUntil(purgeExpiredListeningConversations(env.DB)); + ctx.waitUntil(runReliableJobs(env)); + if (_controller.cron === '0 8 * * *') ctx.waitUntil(Promise.all([purgeExpiredListeningConversations(env.DB),purgeOperationContent(env.DB)])); }, async fetch(request: Request, env: WorkerEnv, ctx: ExecutionContext): Promise { const url = new URL(request.url); @@ -50,8 +53,8 @@ export default { const db = await env.DB.prepare('SELECT 1 AS ready').first(); response = Response.json({ ok: Boolean(db), release: env.RELEASE || 'preview', authMode: env.AUTH_MODE, remixer: env.REMIXER_ENABLED === 'true', production: false, - listeningGuide: { configured: Boolean(env.GATEWAY && canonicalModel(env.GATEWAY_MODEL)), - transport: 'cail-gateway', fallbackConfigured: false } }); + listeningGuide: { configured: Boolean(env.GATEWAY && gatewayModelsConfigured(env.GATEWAY_MODEL,env.GATEWAY_FALLBACK_MODEL)), + transport: 'cail-gateway', fallbackConfigured: gatewayModelsConfigured(env.GATEWAY_MODEL,env.GATEWAY_FALLBACK_MODEL) } }); } else if (env.AUTH_MODE !== 'cail') { response = Response.json({ error: 'Service configuration is incomplete.' }, { status: 503 }); } else if (url.pathname.startsWith('/auth/')) { diff --git a/cloudflare/wrangler.jsonc b/cloudflare/wrangler.jsonc index b815fa6..bfa7be7 100644 --- a/cloudflare/wrangler.jsonc +++ b/cloudflare/wrangler.jsonc @@ -8,7 +8,7 @@ "compatibility_flags": ["nodejs_compat"], "workers_dev": true, "preview_urls": false, - "triggers": { "crons": ["0 8 * * *"] }, + "triggers": { "crons": ["* * * * *", "0 8 * * *"] }, "assets": { "directory": "../public", "binding": "ASSETS", "run_worker_first": true }, "observability": { "enabled": true, "logs": { "invocation_logs": false, "head_sampling_rate": 1 } }, "d1_databases": [{ "binding": "DB", "database_name": "cail-stem-splitter-preview", "database_id": "ae01c5db-c19e-48c8-91bd-132876276f22" }], @@ -22,6 +22,7 @@ "ratelimits": [{ "name": "REQUEST_LIMIT", "namespace_id": "350609", "simple": { "limit": 30, "period": 60 } }], "vars": { "GATEWAY_MODEL": "glm-5.2", + "GATEWAY_FALLBACK_MODEL": "deepseek-v4-flash-0731", "AUTH_MODE": "cail", "CAIL_BROWSER_ORIGIN": "https://tools.ailab.gc.cuny.edu", "PUBLIC_BASE_URL": "https://cail-stem-splitter-preview.ailab-452.workers.dev", diff --git a/replicate-yt-audio/cog.yaml b/replicate-yt-audio/cog.yaml index efe642e..5f2f19d 100644 --- a/replicate-yt-audio/cog.yaml +++ b/replicate-yt-audio/cog.yaml @@ -1,10 +1,11 @@ -# Tiny CPU model: fetch a YouTube video's audio track with yt-dlp. -# Push target: r8.im//yt-audio (see README in this dir). +# Source maintenance only. A new image/version requires a separately approved canary. build: gpu: false python_version: "3.11" system_packages: - ffmpeg python_packages: - - yt-dlp + - "yt-dlp[default,pin]==2026.8.19" + - "yt-dlp-ejs==0.8.0" + - "deno==2.9.5" predict: "predict.py:Predictor" diff --git a/replicate-yt-audio/predict.py b/replicate-yt-audio/predict.py index 617c27d..feea1ff 100644 --- a/replicate-yt-audio/predict.py +++ b/replicate-yt-audio/predict.py @@ -1,18 +1,56 @@ -"""YouTube audio fetcher for the stem-splitter app. - -Downloads the best audio-only track as M4A (Demucs ingests M4A directly). -Raises ValueError with student-readable messages; the Worker surfaces -prediction errors verbatim. -""" - +"""Bounded import of permitted YouTube audio. No client/cookie/proxy rotation.""" +import atexit import json import os +import re +import shutil +import signal import subprocess import tempfile -from urllib.parse import urlparse +import time +from pathlib import Path as FilePath +from urllib.parse import parse_qs, urlparse from cog import BaseModel, BasePredictor, Input, Path +MAX_BYTES = 100 * 1024 * 1024 +MAX_SECONDS = 900 +TOTAL_SECONDS = 230 +ROOT = FilePath(tempfile.gettempdir()) / "stem-youtube-artifacts" +DENIAL = re.compile(r"private|sign.?in|age.?restrict|not available in your country|region|geo.?restrict|removed|deleted|drm|captcha|confirm.*bot|challenge|403|401", re.I) +_active = set() + + +def canonical_url(url): + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or parsed.username or parsed.password or parsed.port: + raise ValueError("invalid_youtube_url") + host = (parsed.hostname or "").lower() + if host.startswith("www.") or host.startswith("m."): + host = host.split(".", 1)[1] + video = None + if host == "youtu.be" and len(parsed.path.strip("/").split("/")) == 1: + video = parsed.path.strip("/") + elif host in ("youtube.com", "music.youtube.com", "youtube-nocookie.com"): + if parsed.path == "/watch": + video = parse_qs(parsed.query).get("v", [None])[0] + elif re.fullmatch(r"/(shorts|embed|live)/[\w-]{11}/?", parsed.path): + video = parsed.path.split("/")[2] + if not video or not re.fullmatch(r"[A-Za-z0-9_-]{11}", video): + raise ValueError("invalid_youtube_url") + return "https://www.youtube.com/watch?v=" + video + + +def cleanup_stale(now=None): + """Finished output is retained briefly for Cog's uploader, then collected. + Failed/cancelled work is removed immediately; startup also clears old roots. + """ + now = time.time() if now is None else now + ROOT.mkdir(mode=0o700, parents=True, exist_ok=True) + for child in ROOT.iterdir(): + if child.is_dir() and str(child) not in _active and now - child.stat().st_mtime > 3600: + shutil.rmtree(child, ignore_errors=True) + class Output(BaseModel): audio: Path @@ -21,83 +59,89 @@ class Output(BaseModel): class Predictor(BasePredictor): - def predict( - self, - url: str = Input(description="YouTube video URL"), - max_duration: int = Input( - description="Reject videos longer than this many seconds", default=900 - ), - ) -> Output: - # The URL is untrusted input; never let it be parsed as a yt-dlp flag. - parsed = urlparse(url) - if url.startswith("-") or parsed.scheme not in ("http", "https") or not parsed.netloc: - raise ValueError("Not a valid http(s) URL.") + def setup(self): + cleanup_stale() + for command in ("yt-dlp", "deno", "ffmpeg", "ffprobe"): + if shutil.which(command) is None: + raise RuntimeError("extractor_dependency_missing") + atexit.register(self._cleanup_active) - info = self._probe(url) + def _cleanup_active(self): + for directory in list(_active): + shutil.rmtree(directory, ignore_errors=True) - if info.get("is_live"): - raise ValueError("Live streams cannot be imported.") - duration = float(info.get("duration") or 0) - if duration > max_duration: - minutes = max_duration // 60 - raise ValueError(f"Video is longer than {minutes} minutes — pick a shorter one.") + def predict(self, url: str = Input(description="Permitted YouTube video URL"), + max_duration: int = Input(description="Duration ceiling", default=900)) -> Output: + url = canonical_url(url) + ceiling = min(MAX_SECONDS, max(1, int(max_duration))) + cleanup_stale() + directory = tempfile.mkdtemp(prefix="import-", dir=ROOT) + _active.add(directory) + keep = False + deadline = time.monotonic() + TOTAL_SECONDS + common = ["yt-dlp", "--ignore-config", "--no-playlist", "--no-cookies", "--no-cache-dir", + "--js-runtimes", "deno", "--no-remote-components", "--socket-timeout", "20", + "--retries", "0", "--fragment-retries", "0", "--extractor-retries", "0", + "--concurrent-fragments", "1", "--max-filesize", str(MAX_BYTES)] + try: + info = json.loads(self._run(common + ["-J", "--skip-download", "--", url], deadline)) + if info.get("is_live") or info.get("live_status") in ("is_live", "is_upcoming"): + raise ValueError("Live streams cannot be imported.") + duration = float(info.get("duration") or 0) + if not 0 < duration <= ceiling: + raise ValueError("Video is longer than 15 minutes or has no usable duration.") + if info.get("availability") in ("private", "premium_only", "subscriber_only", "needs_auth") or info.get("age_limit", 0) > 0: + raise ValueError("source_denied") + self._run(common + ["-f", "bestaudio[ext=m4a]/bestaudio", "-x", "--audio-format", "m4a", + "--postprocessor-args", "ffmpeg:-fs 104857600", "-o", os.path.join(directory, "audio.%(ext)s"), "--", url], deadline) + out = os.path.join(directory, "audio.m4a") + if not os.path.isfile(out) or not 1024 <= os.path.getsize(out) <= MAX_BYTES: + raise ValueError("invalid_audio_response") + probe = json.loads(self._run(["ffprobe", "-v", "error", "-show_entries", "format=duration:stream=codec_type,codec_name", "-of", "json", out], deadline)) + decoded_duration = float(probe.get("format", {}).get("duration") or 0) + if not 0 < decoded_duration <= ceiling or not any(s.get("codec_type") == "audio" for s in probe.get("streams", [])): + raise ValueError("invalid_audio_response") + # ffprobe metadata alone is not successful decoding. + self._run(["ffmpeg", "-v", "error", "-xerror", "-i", out, "-map", "0:a:0", "-f", "null", "-"], deadline) + for child in FilePath(directory).iterdir(): + if str(child) != out: + child.unlink(missing_ok=True) + keep = True + return Output(audio=Path(out), title=str(info.get("title") or "youtube-audio")[:200], duration=decoded_duration) + finally: + _active.discard(directory) + if not keep: + shutil.rmtree(directory, ignore_errors=True) - tmp = tempfile.mkdtemp() - # %(ext)s keeps a video-container download from colliding with the - # extracted M4A; -x --audio-format m4a makes the final name stable. - template = os.path.join(tmp, "audio.%(ext)s") - out = os.path.join(tmp, "audio.m4a") - # YouTube requires PO tokens for web-client format URLs from datacenter - # IPs (media downloads 403 even when metadata succeeds). These app - # clients are exempt from that requirement to varying degrees, so try - # them in order until one delivers audio. Some expose no audio-only - # formats, hence the /best fallback — extraction transcodes to M4A. - last_error = None - for client in ("android", "android_vr", "tv", "ios", "default"): + def _run(self, cmd, deadline=None): + timeout = min(90, (deadline or time.monotonic() + 90) - time.monotonic()) + if timeout <= 0: + raise ValueError("extractor_timeout") + # A process group lets timeout/cancellation kill ffmpeg descendants too. + with tempfile.TemporaryFile() as output, tempfile.TemporaryFile() as errors: + proc = subprocess.Popen(cmd, stdout=output, stderr=errors, start_new_session=True) try: - self._run( - [ - "yt-dlp", - "--no-playlist", - "--extractor-args", - f"youtube:player_client={client}", - "-f", - "bestaudio[ext=m4a]/bestaudio/best", - "-x", - "--audio-format", - "m4a", - "-o", - template, - "--", - url, - ] - ) - except ValueError as err: - last_error = err - print(f"[yt-audio] client {client} failed: {err}", flush=True) - continue - if os.path.exists(out): - print(f"[yt-audio] client {client} delivered audio", flush=True) - break - if not os.path.exists(out): - if last_error is not None: - raise last_error - raise ValueError("YouTube returned no audio track for this video.") - - return Output( - audio=Path(out), - title=info.get("title") or "youtube-audio", - duration=duration, - ) - - def _probe(self, url: str) -> dict: - stdout = self._run(["yt-dlp", "-J", "--no-playlist", "--", url]) - return json.loads(stdout) - - def _run(self, cmd: list) -> str: - proc = subprocess.run(cmd, capture_output=True, text=True) - if proc.returncode != 0: - # Last stderr line usually carries yt-dlp's human-readable reason. - reason = (proc.stderr or "").strip().splitlines()[-1:] or ["unknown error"] - raise ValueError(f"YouTube fetch failed: {reason[0][:300]}") - return proc.stdout + proc.wait(timeout=timeout) + except BaseException: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + raise ValueError("extractor_timeout_or_cancelled") from None + # Keep provider diagnostics bounded and private. Disk-backed capture + # prevents a giant metadata/error response exhausting process RAM. + output.seek(0, os.SEEK_END) + if output.tell() > 8 * 1024 * 1024: + raise ValueError("extractor_response_too_large") + output.seek(0) + stdout = output.read(8 * 1024 * 1024).decode("utf-8", errors="strict") + errors.seek(0) + stderr = errors.read(64 * 1024).decode("utf-8", errors="replace") + if proc.returncode: + if DENIAL.search(stderr): + raise ValueError("source_denied") + if re.search(r"429|too many requests|rate.?limit", stderr, re.I): + raise ValueError("source_rate_limited") + raise ValueError("extractor_failed") + return stdout diff --git a/src/assistant/tools.ts b/src/assistant/tools.ts index 3b4e156..cd0267c 100644 --- a/src/assistant/tools.ts +++ b/src/assistant/tools.ts @@ -71,7 +71,10 @@ export function sanitizeToolCalls( durationSec?: number ): AssistantToolCall[] { const out: AssistantToolCall[] = []; + const seen = new Set(); for (const call of raw.slice(0, MAX_TOOL_CALLS)) { + if (call.id && (typeof call.id !== 'string' || !/^[a-zA-Z0-9_-]{1,128}$/.test(call.id) || seen.has(call.id))) continue; + if (call.id) seen.add(call.id); const name = call.function?.name; let args: unknown; try { @@ -85,16 +88,16 @@ export function sanitizeToolCalls( if (name === 'solo' || name === 'set_mute') { if (typeof a.stem !== 'string' || !stemNames.includes(a.stem)) continue; if (name === 'set_mute' && typeof a.muted !== 'boolean') continue; - out.push({ name, args: name === 'solo' ? { stem: a.stem } : { stem: a.stem, muted: a.muted } }); + out.push({ ...(call.id ? { id: call.id } : {}), name, args: name === 'solo' ? { stem: a.stem } : { stem: a.stem, muted: a.muted } }); } else if (name === 'seek' || name === 'add_note') { const seconds = clampSeconds(a.seconds, durationSec); if (seconds === null) continue; if (name === 'add_note') { const text = String(a.text ?? '').trim().slice(0, MAX_NOTE_CHARS); if (!text) continue; - out.push({ name, args: { seconds, text } }); + out.push({ ...(call.id ? { id: call.id } : {}), name, args: { seconds, text } }); } else { - out.push({ name, args: { seconds } }); + out.push({ ...(call.id ? { id: call.id } : {}), name, args: { seconds } }); } } } diff --git a/src/assistant/types.ts b/src/assistant/types.ts index 7d801aa..4a6f34d 100644 --- a/src/assistant/types.ts +++ b/src/assistant/types.ts @@ -17,6 +17,8 @@ export interface ChatTurn { /** A validated mixer command the browser executes (fire-and-forget). */ export interface AssistantToolCall { + /** Stable within one provider attempt; never use it without the operation ID. */ + id?: string; name: 'solo' | 'set_mute' | 'seek' | 'add_note'; args: Record; } diff --git a/src/daily-allowance.ts b/src/daily-allowance.ts index 41e4419..96cb668 100644 --- a/src/daily-allowance.ts +++ b/src/daily-allowance.ts @@ -1,5 +1,6 @@ -/** Request-count guard, not a monetary balance or a substitute for Gateway authorization. */ -export const SIGNED_IN_SPLITS_PER_DAY = 10; +/** App allowances are independent of provider spend and Gateway authorization. */ +export const SIGNED_IN_SPLITS_PER_DAY = 15; +export const HUMAN_INPUTS_PER_DAY = 50; export function dailyWindow(now = new Date()) { const day = now.toISOString().slice(0, 10); @@ -9,12 +10,20 @@ export function dailyWindow(now = new Date()) { } export async function splitAllowance(db: D1Database, subject: string, now = new Date()) { + return operationAllowance(db, subject, 'split', now); +} + +export async function operationAllowance(db: D1Database, subject: string, kind: 'split' | 'chat', now = new Date()) { const { day, resetsAt } = dailyWindow(now); - const row = await db.prepare("SELECT COUNT(*) AS used FROM app_request_reservations WHERE scope = 'split' AND day = ? AND subject = ?") - .bind(day, subject).first<{ used: number }>(); + const row = await db.prepare(`SELECT + COALESCE(SUM(CASE WHEN state IN ('succeeded','partial') THEN 1 ELSE 0 END),0) AS completed, + COALESCE(SUM(CASE WHEN state NOT IN ('succeeded','partial','failed','cancelled') THEN 1 ELSE 0 END),0) AS inProgress + FROM app_operations WHERE kind=? AND day=? AND subject=?`) + .bind(kind, day, subject).first<{ completed: number; inProgress: number }>(); // A missing/corrupt read must never present a full allowance as verified. - if (!row || !Number.isSafeInteger(row.used) || row.used < 0) throw new Error('Allowance unavailable'); - return { limit: SIGNED_IN_SPLITS_PER_DAY, used: row.used, remaining: Math.max(0, SIGNED_IN_SPLITS_PER_DAY - row.used), resetsAt }; + if (!row || !Number.isSafeInteger(row.completed) || row.completed < 0 || !Number.isSafeInteger(row.inProgress) || row.inProgress < 0) throw new Error('Allowance unavailable'); + const limit = kind === 'split' ? SIGNED_IN_SPLITS_PER_DAY : HUMAN_INPUTS_PER_DAY; + return { limit, completed: row.completed, inProgress: row.inProgress, remaining: Math.max(0, limit-row.completed-row.inProgress), resetsAt }; } /** One atomic insert: parallel requests and different isolates share the same count. diff --git a/src/env.ts b/src/env.ts index d6d8ab6..90b6f5d 100644 --- a/src/env.ts +++ b/src/env.ts @@ -11,6 +11,10 @@ export type Env = { assistantTransport?: typeof import('./assistant/openrouter.ts').openRouterChatStream; assistantQuota?: () => Promise; ASSISTANT_ABORT_SIGNAL?: AbortSignal; + /** Trusted logical input, bound to authenticated subject/course by the route. */ + ASSISTANT_OPERATION_ID?: string; + /** Drain/reconcile existing operations while preventing new paid starts. */ + SPLIT_STARTS_DISABLED?: string; CAIL_BROWSER_ORIGIN?: string; CAIL_COURSE_IDS?: string; /** Literal false pauses course collaboration while retaining owner-only reads. */ diff --git a/src/index.ts b/src/index.ts index 9a78994..00a2efd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,11 @@ import { Hono } from 'hono'; import type { Context } from 'hono'; import { createMiddleware } from 'hono/factory'; import type { Env } from './env'; -import { reserveDailyRequest, splitAllowance } from './daily-allowance.ts'; +import { splitAllowance, operationAllowance, dailyWindow } from './daily-allowance.ts'; +import { submitSplit, drainSplitQueue, reconcileSplit, recoverCallback } from './reliability/splits.ts'; +import { readOperation, OperationError, fingerprint } from './reliability/ledger.ts'; +import { cancelOperation } from './reliability/queue.ts'; +import { reserveAssistant, assistantReceipt } from './reliability/assistant.ts'; import { getRetainedAudio, serveStemAudio, @@ -183,20 +187,6 @@ app.use('/api/*', async (c, next) => { c.header('Cache-Control', 'private, no-store'); const principal = c.get('principal'); if(principal)c.header('X-Stem-Account',principal.subject); - const scope = c.req.method !== 'POST' ? null : c.req.path === '/api/jobs' ? 'split' - : !c.env.assistantTransport && /^\/api\/jobs\/[^/]+\/(?:guide|chat)$/.test(c.req.path) ? 'guide' : null; - if (scope && principal) { - const reservation = await reserveDailyRequest(c.env.DB, principal.subject, scope); - if (!reservation.allowed) { - c.header('Retry-After', String(reservation.retryAfter)); - return c.json({ - error: scope === 'split' ? `You have used today's ${reservation.limit} runs. Your daily limit resets at midnight UTC.` - : 'The daily guide allowance has been reached. Please try again tomorrow.', - code: scope === 'split' ? 'split_daily_limit' : 'guide_daily_limit', - limit: reservation.limit, resetsAt: reservation.resetsAt, - }, 429); - } - } } else if (c.req.path.startsWith('/api/teacher/') && !['GET', 'HEAD'].includes(c.req.method)) { if (c.req.header('origin') && !validWriteOrigin(c.req.raw, c.env)) return c.json({ error: 'Request origin not allowed' }, 403); } @@ -262,7 +252,9 @@ app.get('/api/account', async (c) => { // Usage is informational; an unavailable count must not hide a valid account. let allowance = null; try { allowance = await splitAllowance(c.env.DB, principal.subject); } catch { /* Report unknown, not zero. */ } - return c.json({ account: principal, splitAllowance: allowance }); + let chatAllowance = null; + try { chatAllowance = await operationAllowance(c.env.DB, principal.subject, 'chat'); } catch {} + return c.json({ account: principal, splitAllowance: allowance, chatAllowance }); }); app.get('/api/model-quota', async (c) => { @@ -910,6 +902,12 @@ function archiveErrorResponse(c: Context, err: unknown, fallback: st app.post('/api/jobs', requireClassCode, async (c) => { const parsed = await boundedJson(c, MAX_JOB_JSON_BYTES); if ('response' in parsed) return parsed.response; + if (c.env.AUTH_MODE === 'cail' && c.get('principal')) { + const result=await submitSplit(c.env,c.get('principal')!,parsed.value,c.req.header('Idempotency-Key')); + try { c.executionCtx.waitUntil(runReliableJobs(c.env)); } catch { await runReliableJobs(c.env); } + const row=await c.env.DB.prepare('SELECT * FROM jobs WHERE id=?').bind(result.operation.job_id).first(); + return c.json({...jobResponse(row!), savedToAccount:true, operationId:result.operation.id, replay:!result.created}, result.created?202:200); + } const body = parsed.value as | { coursePolicy?: string; @@ -1233,8 +1231,18 @@ app.get('/api/jobs', async (c) => { nextCursor: results.length > 40 && last ? btoa(JSON.stringify({ createdAt: last.created_at, id: last.id })) : null }); }); +app.post('/api/jobs/:id/cancel', requireClassCode, async (c) => { + const principal=c.get('principal'); + if (!principal || !await cancelOperation(c.env.DB,c.req.param('id'),principal.subject)) return c.json({error:'Split not found.'},404); + return c.json({ok:true}); +}); + app.get('/api/jobs/:id', async (c) => { const id = c.req.param('id'); + if(c.env.AUTH_MODE==='cail') { + const operation=await readOperation(c.env.DB,id); + if(operation) { await reconcileSplit(c.env,operation,ingestResult); try {c.executionCtx.waitUntil(runReliableJobs(c.env));} catch {} } + } let row = await c.env.DB.prepare('SELECT * FROM jobs WHERE id = ?').bind(id).first(); if (!row) return c.json({ error: 'Job not found' }, 404); @@ -1252,7 +1260,7 @@ app.get('/api/jobs/:id', async (c) => { // Reconciliation fallback: if we're still 'processing', poll the provider // directly in case the completion webhook was missed (also makes local // dev work, where webhooks can't reach us). - if (row.status === 'processing' && row.external_id) { + if (row.status === 'processing' && row.external_id && !(c.env.AUTH_MODE==='cail' && await readOperation(c.env.DB,id))) { try { const result = await getBackend(c.env).fetchStatus(row.external_id); if (result.status !== 'processing') { @@ -1915,6 +1923,20 @@ app.post('/api/webhooks/separation', async (c) => { const jobId = c.req.query('job'); if (!(await equalSecret(token, c.env.WEBHOOK_SECRET))) return c.text('Forbidden', 403); if (!jobId) return c.text('Missing job', 400); + if(c.env.AUTH_MODE==='cail') { + let operation=await readOperation(c.env.DB,jobId); + if(operation) { + if(!operation.provider_id) { + const parsed=await boundedJson(c,MAX_WEBHOOK_JSON_BYTES); + const value=parsed.value as {id?:unknown}|undefined; + if(value && typeof value.id==='string') await recoverCallback(c.env,operation,value.id); + operation=await readOperation(c.env.DB,jobId); + } else await c.req.raw.body?.cancel().catch(()=>undefined); + if(operation?.provider_id) await reconcileSplit(c.env,{...operation,not_before:0},ingestResult); + try {c.executionCtx.waitUntil(runReliableJobs(c.env));} catch {} + return c.json({ok:true}); + } + } const row = await c.env.DB.prepare('SELECT * FROM jobs WHERE id = ?').bind(jobId).first(); if (!row) return c.text('Unknown job', 404); @@ -1972,11 +1994,16 @@ app.notFound((c) => c.json({ error: 'Not found' }, 404)); app.onError((_error, c) => { if(_error instanceof CourseError)return courseErrorResponse(_error); + if (_error instanceof OperationError) { + if(_error.retryAfter)c.header('Retry-After',String(_error.retryAfter)); + return c.json({error:_error.message,code:_error.code,resetsAt:dailyWindow().resetsAt},_error.status); + } console.error(JSON.stringify({ event: 'request_failed', method: c.req.method })); return c.json({ error: 'The service is temporarily unavailable. Please try again.' }, 503); }); export default app; +export async function runReliableJobs(env: Env) { return drainSplitQueue(env,ingestResult); } // --- helpers ------------------------------------------------------------ @@ -2037,18 +2064,19 @@ async function ingestResult(env: Env, jobId: string, result: SeparationResult): } } const audio = await downloadStem(stem.name, stem.url); - const key = `stems/${jobId}/${stem.name}.mp3`; + const key = `stems/${jobId}/${lease.split(':').at(-1)}/${stem.name}.mp3`; await env.AUDIO.put(key, audio, { httpMetadata: { contentType: 'audio/mpeg' }, }); stored.push({ name: stem.name, key }); } - await env.DB.prepare( + const committed = await env.DB.prepare( "UPDATE jobs SET status = ?, stems = ?, error = NULL WHERE id = ? AND status = 'ingesting' AND error = ?" ) .bind('done', JSON.stringify(stored), jobId, lease) .run(); + if (!committed.meta.changes) await Promise.allSettled(stored.map(({key})=>env.AUDIO.delete(key))); } catch (error) { const cleanup = await Promise.allSettled(stored.map(({ key }) => env.AUDIO.delete(key))); if (cleanup.some((result) => result.status === 'rejected')) { @@ -2062,6 +2090,10 @@ async function ingestResult(env: Env, jobId: string, result: SeparationResult): .run(); return; } + if(env.AUTH_MODE==='cail' && await readOperation(env.DB,jobId)) { + await env.DB.prepare("UPDATE jobs SET status='failed',error='The finished audio could not be saved. No successful-split allowance was used.' WHERE id=? AND status='ingesting' AND error=?").bind(jobId,lease).run(); + return; + } // Let a provider retry or the next browser poll make another attempt. await env.DB.prepare( "UPDATE jobs SET status = 'processing', error = NULL WHERE id = ? AND status = 'ingesting' AND error = ?" diff --git a/src/reliability/assistant.ts b/src/reliability/assistant.ts new file mode 100644 index 0000000..6f3a0bc --- /dev/null +++ b/src/reliability/assistant.ts @@ -0,0 +1,33 @@ +import type { Env } from '../env.ts'; +import { fingerprint, OperationError, reserveOperation, settleOperation, type Operation } from './ledger.ts'; + +/** One reservation surrounds the human message, tools, and both model attempts. + * This ledger contains no transcript text. Conversation storage owns readback. */ +export async function reserveAssistant(env: Env, subject: string, courseId: string | null, kind: 'chat'|'guide', key: string, jobId: string, input: unknown) { + const reserved=await reserveOperation(env.DB,{subject,courseId,kind,key,jobId,fingerprint:await fingerprint(input),phase:kind,request:{}}); + const op=reserved.operation; + if(!reserved.created) { + throw new OperationError('input_already_submitted',409,op.state==='failed'?'This input did not complete. Send a new message to try again.':'This input was already accepted. Reload the conversation before continuing.'); + } + await env.DB.prepare("UPDATE app_operations SET state='running',lease_until=?,updated_at=? WHERE id=? AND state='queued'") + .bind(Date.now()+90000,Date.now(),op.id).run(); + return {operation:op,replay:false}; +} +export function assistantReceipt(env: Env,operation: Operation) { + let delivered=false; + return { + /** Persist BEFORE handing output to the response writer. A process crash + * after this point must not turn a delivered partial into a free replay. */ + async delta(value: string) { + delivered ||= Boolean(value); + await env.DB.prepare("UPDATE app_operations SET result_json=?,updated_at=? WHERE id=? AND state='running'") + .bind(JSON.stringify({delivery:'partial'}),Date.now(),operation.id).run(); + }, + async complete(_reply: string) { + await settleOperation(env.DB,operation.id,'succeeded',null,{delivery:'completed'}); + }, + async failed() { + await settleOperation(env.DB,operation.id,delivered?'partial':'failed',delivered?'partial_response':'no_usable_response',{delivery:delivered?'partial':'none'}); + }, + }; +} diff --git a/src/reliability/ledger.ts b/src/reliability/ledger.ts new file mode 100644 index 0000000..5cdb615 --- /dev/null +++ b/src/reliability/ledger.ts @@ -0,0 +1,93 @@ +import { dailyWindow, SIGNED_IN_SPLITS_PER_DAY, HUMAN_INPUTS_PER_DAY } from '../daily-allowance.ts'; + +export type OperationKind = 'split' | 'chat' | 'guide'; +export type Phase = 'fetch' | 'split' | 'chat' | 'guide'; +export type State = 'queued' | 'running' | 'starting' | 'processing' | 'reconciling' | 'succeeded' | 'partial' | 'failed' | 'cancelled'; +export interface Operation { + id: string; subject: string; course_id: string | null; kind: OperationKind; + idempotency_key: string; fingerprint: string; day: string; state: State; phase: Phase; + job_id: string | null; request_json: string; result_json: string | null; provider_id: string | null; + lease_owner: string | null; lease_until: number; fence: number; not_before: number; + deadline: number; cancel_requested: number; error_code: string | null; created_at: number; updated_at: number; +} +export const TERMINAL = "'succeeded','partial','failed','cancelled'"; +export const MAX_QUEUED_SPLITS = 100; +export const MAX_SPLIT_ATTEMPTS_PER_DAY = 40; +export const MAX_CHAT_ATTEMPTS_PER_DAY = 80; +export const validOperationKey = (key: unknown): key is string => typeof key === 'string' && /^[a-zA-Z0-9_-]{16,80}$/.test(key); +export class OperationError extends Error { + code: string; status: 400|409|429|503; retryAfter: number; + constructor(code: string, status: 400 | 409 | 429 | 503, message: string, retryAfter = 0) { super(message);this.code=code;this.status=status;this.retryAfter=retryAfter; } +} +export async function fingerprint(value: unknown): Promise { + const data = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(JSON.stringify(value))); + return Array.from(new Uint8Array(data), b => b.toString(16).padStart(2, '0')).join(''); +} +export async function readOperation(db: D1Database, id: string): Promise { + return db.prepare('SELECT * FROM app_operations WHERE id=?').bind(id).first(); +} +export async function findOperation(db: D1Database, subject: string, kind: OperationKind, key: string) { + return db.prepare('SELECT * FROM app_operations WHERE subject=? AND kind=? AND idempotency_key=?').bind(subject, kind, key).first(); +} +/** The INSERT predicate and all associated job writes execute in one D1 batch. + * The unique key handles racing replays; quota includes active and settled work. + */ +export async function reserveOperation(db: D1Database, input: { + subject: string; courseId: string | null; kind: OperationKind; key: string; fingerprint: string; + phase: Phase; jobId?: string; request?: unknown; now?: number; + statements?: (id: string) => D1PreparedStatement[]; +}): Promise<{ operation: Operation; created: boolean }> { + if (!validOperationKey(input.key)) throw new OperationError('idempotency_required', 400, 'Reload and submit again with a request identifier.'); + const now = input.now ?? Date.now(); + const { day, retryAfter } = dailyWindow(new Date(now)); + const limit = input.kind === 'split' ? SIGNED_IN_SPLITS_PER_DAY : input.kind === 'chat' ? HUMAN_INPUTS_PER_DAY : 20; + const abuseLimit = input.kind === 'split' ? MAX_SPLIT_ATTEMPTS_PER_DAY : input.kind === 'chat' ? MAX_CHAT_ATTEMPTS_PER_DAY : 20; + const id = input.kind==='split' && input.jobId ? input.jobId : crypto.randomUUID(); + const insert = db.prepare(`INSERT INTO app_operations + (id,subject,course_id,kind,idempotency_key,fingerprint,day,state,phase,job_id,request_json,deadline,created_at,updated_at) + SELECT ?,?,?,?,?,?,?,'queued',?,?,?,?,?,? WHERE + (SELECT COUNT(*) FROM app_operations WHERE subject=? AND kind=? AND day=? AND state NOT IN ('failed','cancelled')) < ? + AND (SELECT COUNT(*) FROM app_operations WHERE subject=? AND kind=? AND day=?) < ? + AND (? <> 'split' OR (SELECT COUNT(*) FROM app_operations WHERE kind='split' AND state NOT IN (${TERMINAL})) < ?) + AND (? <> 'guide' OR (SELECT COUNT(*) FROM app_operations WHERE kind='guide' AND day=?) < 200) + ON CONFLICT(subject,kind,idempotency_key) DO NOTHING`) + .bind(id,input.subject,input.courseId,input.kind,input.key,input.fingerprint,day,input.phase,input.jobId ?? null, + JSON.stringify(input.request ?? {}),now+24*60*60*1000,now,now,input.subject,input.kind,day,limit, + input.subject,input.kind,day,abuseLimit,input.kind,MAX_QUEUED_SPLITS,input.kind,day); + await db.batch([insert,...(input.statements?.(id) ?? [])]); + const operation = await findOperation(db,input.subject,input.kind,input.key); + if (!operation) { + const counts=await db.prepare(`SELECT COUNT(*) AS attempts, + SUM(CASE WHEN state NOT IN ('failed','cancelled') THEN 1 ELSE 0 END) AS held + FROM app_operations WHERE subject=? AND kind=? AND day=?`).bind(input.subject,input.kind,day).first<{attempts:number;held:number}>(); + if ((counts?.held ?? 0)>=limit) throw new OperationError(`${input.kind}_daily_limit`,429, + input.kind==='split'?'Completed and in-progress splits fill today\'s 15 places. Failed splits do not count as completed.' + :'Today\'s Listening Guy input allowance is full.',retryAfter); + if ((counts?.attempts ?? 0)>=abuseLimit || input.kind==='guide') throw new OperationError(`${input.kind}_attempt_limit`,429, + 'Repeated requests have reached the daily protection limit. Failed splits did not use successful-split allowance. Please try again after the reset.',retryAfter); + throw new OperationError('split_queue_full',429,'The audio queue is full. Wait a minute and submit the same request again.',60); + } + if (operation.fingerprint !== input.fingerprint || operation.course_id !== input.courseId) { + throw new OperationError('idempotency_conflict',409,'This request identifier belongs to different input. Submit a new request.'); + } + return { operation, created: operation.id === id && operation.created_at === now }; +} +export async function settleOperation(db: D1Database, id: string, state: 'succeeded' | 'partial' | 'failed' | 'cancelled', code: string | null = null, result: unknown = null, fence?: number) { + const update = await db.prepare(`UPDATE app_operations SET state=?,error_code=?,result_json=?,lease_owner=NULL,lease_until=0,updated_at=? + WHERE id=? AND state NOT IN (${TERMINAL}) AND (? IS NULL OR fence=?) + AND (? NOT IN ('succeeded','partial') OR cancel_requested=0)`) + .bind(state,code,result === null ? null : JSON.stringify(result),Date.now(),id,fence ?? null,fence ?? null,state).run(); + return update.meta.changes === 1; +} +export async function beginAttempt(db: D1Database, operation: Operation, provider: string, model?: string) { + const id=crypto.randomUUID(), now=Date.now(); + const inserted=await db.prepare(`INSERT INTO operation_attempts(id,operation_id,phase,provider,model,outcome,created_at,updated_at) + SELECT ?,id,phase,?,?,'starting',?,? FROM app_operations WHERE id=? AND fence=? AND state NOT IN (${TERMINAL})`) + .bind(id,provider,model ?? null,now,now,operation.id,operation.fence).run(); + if(inserted.meta.changes!==1)throw new OperationError('operation_fenced',409,'This request is being recovered.'); + return id; +} +export async function finishAttempt(db: D1Database, id: string, outcome: string, detail: { externalId?: string; status?: number; code?: string; usage?: unknown } = {}) { + await db.prepare('UPDATE operation_attempts SET outcome=?,external_id=COALESCE(?,external_id),http_status=?,code=?,usage_json=?,updated_at=? WHERE id=?') + .bind(outcome,detail.externalId ?? null,detail.status ?? null,detail.code ?? null,detail.usage === undefined ? null : JSON.stringify(detail.usage),Date.now(),id).run(); +} diff --git a/src/reliability/queue.ts b/src/reliability/queue.ts new file mode 100644 index 0000000..913db0b --- /dev/null +++ b/src/reliability/queue.ts @@ -0,0 +1,70 @@ +import { type Operation, type Phase, TERMINAL, readOperation } from './ledger.ts'; + +// Conservative application ceilings; they do not assert an account entitlement. +// Unknown starts occupy a slot until authoritative evidence resolves them. +export const QUEUE_CAPS = { fetch: 1, split: 1, perSubject: 1 } as const; +export const WORK_LEASE_MS = 5*60*1000; + +export async function claimNext(db: D1Database, phase: 'fetch' | 'split', now=Date.now()): Promise { + const owner=crypto.randomUUID(); + // D1 serializes this statement. The subquery plus compare-and-set cannot + // oversubscribe even when unrelated Worker instances claim simultaneously. + const row=await db.prepare(`UPDATE app_operations SET state='running',lease_owner=?,lease_until=?,fence=fence+1,updated_at=? + WHERE id=(SELECT q.id FROM app_operations q + LEFT JOIN operation_fairness f ON f.subject=q.subject AND f.phase=q.phase + WHERE q.kind='split' AND q.phase=? AND q.state='queued' AND q.not_before<=? AND q.deadline>? + AND COALESCE((SELECT until_ms FROM provider_cooldowns WHERE provider='replicate'),0)<=? + AND (SELECT COUNT(*) FROM app_operations a WHERE a.kind='split' AND a.phase=q.phase AND a.state IN ('running','starting','processing','reconciling'))(); + if (row) await db.prepare(`INSERT INTO operation_fairness(subject,phase,last_started) VALUES(?,?,?) + ON CONFLICT(subject,phase) DO UPDATE SET last_started=excluded.last_started`).bind(row.subject,phase,now).run(); + return row; +} + +export async function transition(db: D1Database, operation: Operation, state: Operation['state'], changes: {phase?: Phase; providerId?: string | null; notBefore?: number; code?: string | null}={}) { + const result=await db.prepare(`UPDATE app_operations SET state=?,phase=?,provider_id=?,not_before=?,error_code=?,updated_at=? + WHERE id=? AND fence=? AND lease_owner IS ? AND state NOT IN (${TERMINAL})`) + .bind(state,changes.phase ?? operation.phase,changes.providerId === undefined ? operation.provider_id : changes.providerId, + changes.notBefore ?? 0,changes.code ?? null,Date.now(),operation.id,operation.fence,operation.lease_owner).run(); + return result.meta.changes===1; +} + +export async function recoverExpired(db: D1Database, now=Date.now()) { + // Never blindly repeat an accepted or ambiguously accepted POST. Work that + // died before an attempt record is safe to enqueue; starts reconcile. + await db.prepare(`UPDATE app_operations SET state=CASE + WHEN EXISTS(SELECT 1 FROM operation_attempts a WHERE a.operation_id=app_operations.id AND a.phase=app_operations.phase AND a.outcome IN ('starting','accepted','uncertain')) + THEN 'reconciling' ELSE 'queued' END, + fence=fence+1,lease_owner=NULL,lease_until=0,updated_at=? + WHERE kind='split' AND state IN ('running','starting') AND lease_until'{}' OR result_json IS NOT NULL)`).bind(now,cutoff), + db.prepare(`UPDATE jobs SET status='failed',error='This split expired before it finished.' + WHERE id IN(SELECT job_id FROM app_operations WHERE kind='split' AND state='queued' AND created_at<=?)`).bind(cutoff), + ]); +} diff --git a/src/reliability/retry.ts b/src/reliability/retry.ts new file mode 100644 index 0000000..e282f22 --- /dev/null +++ b/src/reliability/retry.ts @@ -0,0 +1,28 @@ +/** Absolute deadlines: Retry-After is a floor, never capped to a shorter wait. */ +export function retryAt(header: string | null, now = Date.now(), attempt = 0, random = Math.random) { + if (header && /^\d+(?:\.\d+)?$/.test(header.trim())) { + const seconds=Number(header); + if (Number.isFinite(seconds) && seconds >= 0) return now+Math.ceil(seconds*1000); + } + if (header && !/^\s*[-+]?\d/.test(header)) { + const date=Date.parse(header); + if (Number.isFinite(date)) return Math.max(now,date); + } + return now+Math.ceil(Math.min(30000,1000*2**Math.min(attempt,5))*(1+Math.max(0,Math.min(1,random())))); +} +export class UpstreamError extends Error { + code: string; outcome: 'rejected' | 'uncertain' | 'terminal'; status: number; notBefore: number; + constructor(code: string, outcome: 'rejected' | 'uncertain' | 'terminal', status = 0, notBefore = 0) { + super(outcome === 'uncertain' ? 'The provider may have accepted this request. We are checking its status; do not submit it again.' + : 'The audio service could not complete this request. Your successful-split allowance has not been used.'); + this.code=code;this.outcome=outcome;this.status=status;this.notBefore=notBefore; + } +} +export async function setCooldown(db: D1Database, provider: string, until: number) { + await db.prepare(`INSERT INTO provider_cooldowns(provider,until_ms) VALUES(?,?) + ON CONFLICT(provider) DO UPDATE SET until_ms=MAX(until_ms,excluded.until_ms)`).bind(provider,until).run(); +} +export async function cooldown(db: D1Database, provider: string, now=Date.now()) { + const row=await db.prepare('SELECT until_ms FROM provider_cooldowns WHERE provider=?').bind(provider).first<{until_ms:number}>(); + return Math.max(now,row?.until_ms ?? now); +} diff --git a/src/reliability/splits.ts b/src/reliability/splits.ts new file mode 100644 index 0000000..d822653 --- /dev/null +++ b/src/reliability/splits.ts @@ -0,0 +1,215 @@ +import type { Env } from '../env.ts'; +import type { AppPrincipal } from '../identity.ts'; +import { archiveContentType, ArchiveError, fetchArchiveAudio, fetchArchiveItem, parseArchiveIdentifier } from '../archive.ts'; +import { parseYouTubeVideoId, pollYouTubeImport, startYouTubeImport, YouTubeError } from '../youtube.ts'; +import { getBackend, type SeparationResult } from '../separation/index.ts'; +import { getSeparationOptions, modelIsAllowed } from '../separation/options.ts'; +import { presignDownload, presignAnalysisDownload } from '../r2.ts'; +import { serverAutoCapability, configuredAudioAnalysisProvider, resolveAutoRoutingWithSource, audioAnalysisTimeoutMs, AUTO_ROUTING_REQUEST, prepareAuthoritativeAutoSource } from '../analysis/index.ts'; +import { parseBrowserAutoSummary } from '../analysis/contract.ts'; +import { processingFeatureFlags } from '../features.ts'; +import { authorizeStoredCourseWork, courseAssignmentStatement } from '../classroom/access.ts'; +import { beginAttempt, finishAttempt, fingerprint, OperationError, readOperation, reserveOperation, TERMINAL, type Operation } from './ledger.ts'; +import { claimNext, recoverExpired, transition, WORK_LEASE_MS } from './queue.ts'; +import { cooldown, setCooldown, UpstreamError } from './retry.ts'; + +const MAX_BYTES=100*1024*1024; +interface SplitInput { + sourceType: 'upload'|'youtube'|'archive'; key?: string; filename: string; youtubeUrl?: string; + archiveId?: string; archiveFile?: string; model: string; auto: boolean; browserAnalysis?: unknown; +} +type Complete = (env: Env, jobId: string, result: SeparationResult) => Promise; +const title=(value: unknown) => typeof value==='string' ? value.replace(/[\x00-\x1f\x7f/\\]/g,' ').trim().slice(0,200) : ''; +export const operationWebhook=(env: Env,id: string) => `${env.PUBLIC_BASE_URL}/api/webhooks/separation?job=${id}&token=${env.WEBHOOK_SECRET}`; + +export async function submitSplit(env: Env, principal: AppPrincipal, body: unknown, key: string | undefined) { + if (env.SPLIT_STARTS_DISABLED==='true') throw new OperationError('splits_paused',503,'New splits are temporarily paused. Saved audio still plays.'); + if (!body || typeof body!=='object'||Array.isArray(body)) throw new OperationError('invalid_input',400,'Choose an audio file or a supported source.'); + const value=body as Record; + const courseId=principal.courseId ?? null; + if (courseId && value.coursePolicy!=='course-work-v1') throw new OperationError('course_policy_required',400,'Review the course sharing notice before submitting.'); + const options=getSeparationOptions(env.SEPARATION_BACKEND), submitted=value.model ?? options.defaultModel; + const capability=serverAutoCapability(env); + const auto=Boolean(capability&&(submitted===AUTO_ROUTING_REQUEST||value.routingRequest===AUTO_ROUTING_REQUEST)); + if (typeof submitted!=='string'||(submitted===AUTO_ROUTING_REQUEST ? !auto||capability?.mode!=='authoritative' : !modelIsAllowed(env.SEPARATION_BACKEND,submitted))) throw new OperationError('invalid_model',400,'Choose an available split model.'); + const input: SplitInput={sourceType:'upload',filename:'Audio',model:submitted===AUTO_ROUTING_REQUEST?options.defaultModel:submitted,auto}; + if (auto) input.browserAnalysis=parseBrowserAutoSummary(value.browserAnalysis,options.models); + if ([value.youtubeUrl,value.archiveId,value.key].filter(v=>v!==undefined).length!==1) throw new OperationError('invalid_input',400,'Choose exactly one audio source.'); + if (value.youtubeUrl!==undefined) { + const id=parseYouTubeVideoId(value.youtubeUrl as string); + if (!id) throw new OperationError('invalid_youtube_url',400,'Paste a full YouTube video link.'); + input.sourceType='youtube'; input.youtubeUrl=`https://www.youtube.com/watch?v=${id}`; input.filename='YouTube audio'; + } else if (value.archiveId!==undefined) { + const id=typeof value.archiveId==='string'&&parseArchiveIdentifier(value.archiveId); + if (!id||value.archiveFile!==undefined&&(typeof value.archiveFile!=='string'||value.archiveFile.length>1024)) throw new OperationError('invalid_archive_id',400,'Choose an available Archive audio file.'); + input.sourceType='archive';input.archiveId=id;input.archiveFile=value.archiveFile as string|undefined;input.filename='Archive audio'; + } else { + if (typeof value.key!=='string'||!/^uploads\/[a-zA-Z0-9-]+\/[^/]+$/.test(value.key)||!title(value.filename)) throw new OperationError('invalid_upload',400,'Choose and upload an audio file.'); + const owned=await env.DB.prepare("SELECT 1 FROM upload_owners WHERE object_key=? AND subject=? AND state='ready' AND expires_at>?") + .bind(value.key,principal.subject,new Date().toISOString()).first(); + const source=owned&&await env.AUDIO.head(value.key); + if (!source||source.size<=0||source.size>MAX_BYTES) throw new OperationError('invalid_upload',400,'The upload is unavailable, empty, or too large. Upload it again.'); + input.key=value.key;input.filename=title(value.filename); + } + const id=crypto.randomUUID(); + return reserveOperation(env.DB,{subject:principal.subject,courseId,kind:'split',key:key ?? '',fingerprint:await fingerprint(input),phase:input.sourceType==='upload'?'split':'fetch',jobId:id,request:input, + statements: opId=>{ + const statements=[env.DB.prepare(`INSERT INTO jobs(id,filename,source_key,status,model,source_type) + SELECT id,?,?,'queued',?,? FROM app_operations WHERE id=?`) + .bind(input.filename,input.key ?? `uploads/${id}/source.m4a`,input.model,input.sourceType,opId), + env.DB.prepare('INSERT INTO job_owners(job_id,subject) SELECT id,subject FROM app_operations WHERE id=?').bind(opId)]; + const course=courseAssignmentStatement(env,id,principal); + if (course) statements.push(course); + return statements; + }}); +} + +async function fail(env: Env, op: Operation, code: string, message='This split could not finish. No successful-split allowance was used. Try another source or upload an original or licensed audio file.') { + await env.DB.batch([ + env.DB.prepare(`UPDATE app_operations SET error_code=? WHERE id=? AND fence=? AND state NOT IN (${TERMINAL})`).bind(code,op.id,op.fence), + env.DB.prepare(`UPDATE jobs SET status='failed',error=? WHERE id=? AND EXISTS(SELECT 1 FROM app_operations WHERE id=? AND fence=? AND state NOT IN (${TERMINAL}))`).bind(message,op.job_id,op.id,op.fence), + ]); +} +async function imported(env: Env,op: Operation,input: SplitInput,audio: {data:ArrayBuffer;title:string;fileName?:string;attribution?:unknown}) { + if (!audio.data.byteLength||audio.data.byteLength>MAX_BYTES) throw new YouTubeError('Audio is empty or too large.','invalid_audio_response'); + const hash=await crypto.subtle.digest('SHA-256',audio.data); + const sha=Array.from(new Uint8Array(hash),b=>b.toString(16).padStart(2,'0')).join(''); + const extension=audio.fileName?.match(/\.[a-zA-Z0-9]+$/)?.[0] ?? '.m4a'; + const key=`uploads/${op.id}/source-${op.fence}${extension}`; + await env.AUDIO.put(key,audio.data,{httpMetadata:{contentType:audio.fileName?archiveContentType(audio.fileName):'audio/mp4'}}); + const current=await readOperation(env.DB,op.id); + if (!current||current.fence!==op.fence||current.cancel_requested||['failed','cancelled'].includes(current.state)) { + await env.AUDIO.delete(key);if(current)await fail(env,current,'cancelled');return; + } + input.key=key;input.filename=title(audio.title)||input.filename; + await env.DB.batch([ + env.DB.prepare(`UPDATE jobs SET source_key=?,source_hash=?,filename=?,status='queued' WHERE id=?`).bind(key,sha,input.filename,op.job_id), + env.DB.prepare(`UPDATE app_operations SET request_json=?,provider_id=NULL,phase='split',state='queued',lease_owner=NULL,lease_until=0,updated_at=? WHERE id=? AND fence=?`).bind(JSON.stringify(input),Date.now(),op.id,op.fence), + ...(audio.attribution?[env.DB.prepare('INSERT OR REPLACE INTO job_attributions(job_id,attribution) VALUES(?,?)').bind(op.job_id,JSON.stringify(audio.attribution))]:[]), + ]); +} +async function storeArchive(env: Env,op: Operation,input: SplitInput) { + const scope=await fingerprint([op.subject,op.course_id]),source=await fingerprint([input.archiveId,input.archiveFile]); + // Every cache hit revalidates the public rights floor with Archive metadata. + await fetchArchiveItem(input.archiveId!); + const cached=await env.DB.prepare('SELECT object_key,metadata_json FROM import_cache WHERE scope=? AND source=? AND expires_at>? AND object_key IS NOT NULL') + .bind(scope,source,Date.now()).first<{object_key:string;metadata_json:string}>(); + if (cached) { + const object=await env.AUDIO.get(cached.object_key); + if (object && object.size<=MAX_BYTES) { await imported(env,op,input,{...JSON.parse(cached.metadata_json),data:await object.arrayBuffer()});return; } + } + const audio=await fetchArchiveAudio(input.archiveId!,input.archiveFile,env); + await imported(env,op,input,audio); + const current=await readOperation(env.DB,op.id); + if (current?.phase==='split'&&input.key) await env.DB.prepare(`INSERT INTO import_cache(scope,source,operation_id,object_key,metadata_json,expires_at) + VALUES(?,?,?,?,?,?) ON CONFLICT(scope,source) DO UPDATE SET operation_id=excluded.operation_id,object_key=excluded.object_key,metadata_json=excluded.metadata_json,expires_at=excluded.expires_at`) + .bind(scope,source,op.id,input.key,JSON.stringify({title:audio.title,fileName:audio.fileName,attribution:audio.attribution}),op.created_at+24*60*60*1000).run(); +} + +async function start(env: Env,op: Operation) { + const input=JSON.parse(op.request_json) as SplitInput; + try { await authorizeStoredCourseWork(env,op.subject,op.course_id); } + catch { await fail(env,op,'access_unavailable','Course access could not be confirmed. No successful-split allowance was used.');return; } + if (op.cancel_requested) {await fail(env,op,'cancelled');return;} + if (op.phase==='fetch'&&input.sourceType==='archive') { + try {await storeArchive(env,op,input);} catch {await fail(env,op,'archive_import_failed');}return; + } + let attempt: string | null=null; + try { + let sourceUrl='';let model=input.model; + if (op.phase==='split') { + let sourceKey=input.key!; + const capability=serverAutoCapability(env); + if (input.auto&&capability) { + const options=getSeparationOptions(env.SEPARATION_BACKEND); + let expectedSourceBytes: number | undefined; + if (input.sourceType==='upload'&&capability.mode==='authoritative') { + const snapshot=await prepareAuthoritativeAutoSource(env,{jobId:op.id,sourceKey});sourceKey=snapshot.snapshotKey;expectedSourceBytes=snapshot.bytes; + } + const stored=await env.DB.prepare('SELECT source_hash FROM jobs WHERE id=?').bind(op.job_id).first<{source_hash:string|null}>(); + const head=stored?.source_hash?await env.AUDIO.head(sourceKey):null; + const resolution=await resolveAutoRoutingWithSource({sourceUrl:await presignAnalysisDownload(env,sourceKey),sourceType:input.sourceType,mode:capability.mode,currentModel:model,fallbackModel:options.defaultModel,coreModels:options.models, + ...(input.browserAnalysis?{browserAnalysis:parseBrowserAutoSummary(input.browserAnalysis,options.models)}:{}), + ...(stored?.source_hash&&head?{expectedSourceIdentity:{schemaVersion:'1' as const,sha256:stored.source_hash,bytes:head.size}}:{}), + ...(expectedSourceBytes!==undefined?{expectedSourceBytes}:{}), + provider:configuredAudioAnalysisProvider(env),timeoutMs:audioAnalysisTimeoutMs(env),instrumentDiscovery:processingFeatureFlags(env).instrumentDiscovery}); + model=resolution.decision.resolvedCoreModel; + await env.DB.prepare('UPDATE jobs SET model=?,routing_request=?,analysis=?,source_key=? WHERE id=?').bind(model,AUTO_ROUTING_REQUEST,JSON.stringify(resolution.decision),sourceKey,op.job_id).run(); + } + sourceUrl=await presignDownload(env,sourceKey); + } + // This write precedes every paid POST. A crash from here reconciles, never retries. + if (!await transition(env.DB,op,'starting')) return; + attempt=await beginAttempt(env.DB,op,'replicate',op.phase==='fetch'?env.REPLICATE_YT_MODEL_VERSION:model); + const providerId=op.phase==='fetch' + ? await startYouTubeImport(input.youtubeUrl!,env,operationWebhook(env,op.id)) + : (await getBackend(env).start({jobId:op.id,audioUrl:sourceUrl,webhookUrl:operationWebhook(env,op.id),model})).externalId; + await finishAttempt(env.DB,attempt,'accepted',{externalId:providerId}); + if (await transition(env.DB,op,'processing',{providerId})) { + await env.DB.prepare('UPDATE app_operations SET lease_owner=NULL,lease_until=0 WHERE id=? AND fence=?').bind(op.id,op.fence).run(); + await env.DB.prepare('UPDATE jobs SET status=?,external_id=? WHERE id=? AND status<>\'failed\'') + .bind(op.phase==='fetch'?'importing':'processing',op.phase==='fetch'?null:providerId,op.job_id).run(); + } + } catch(error) { + if (attempt) await finishAttempt(env.DB,attempt,error instanceof UpstreamError?error.outcome:'uncertain',{ + code:error instanceof UpstreamError?error.code:'start_uncertain',status:error instanceof UpstreamError?error.status:undefined}); + if (!attempt) {await fail(env,op,'preparation_failed');return;} + if (error instanceof UpstreamError&&error.outcome==='rejected') { + const count=await env.DB.prepare('SELECT COUNT(*) AS n FROM operation_attempts WHERE operation_id=? AND phase=?').bind(op.id,op.phase).first<{n:number}>(); + if (error.status===429) await setCooldown(env.DB,'replicate',error.notBefore); + if (error.status===429&&(count?.n ?? 0)<3&&error.notBefore'failed'").bind(op.job_id).run(); + } +} + +export async function reconcileSplit(env: Env,op: Operation,complete: Complete) { + if (!op.provider_id||!['processing','reconciling'].includes(op.state)) return; + const now=Date.now(),owner=crypto.randomUUID(); + const claimed=await env.DB.prepare(`UPDATE app_operations SET lease_owner=?,lease_until=?,fence=fence+1,updated_at=? + WHERE id=? AND fence=? AND lease_until<=? AND not_before<=? AND state IN ('processing','reconciling') RETURNING *`) + .bind(owner,now+WORK_LEASE_MS,now,op.id,op.fence,now,now).first(); + if(!claimed)return; + try { + if (claimed.phase==='fetch') { + const audio=await pollYouTubeImport(claimed.provider_id!,env); + if(audio)await imported(env,claimed,JSON.parse(claimed.request_json),audio); + } else { + const result=await getBackend(env).fetchStatus(claimed.provider_id!); + if(result.status!=='processing') { + if(claimed.cancel_requested)await fail(env,claimed,'cancelled','Split cancelled. No successful-split allowance was used.'); + else await complete(env,claimed.job_id!,result); + } + } + } catch(error) { + if(error instanceof YouTubeError||error instanceof ArchiveError) await fail(env,claimed,error.code); + else if(error instanceof UpstreamError&&error.notBefore) await setCooldown(env.DB,'replicate',error.notBefore); + // Provider/storage outage retains the exact prediction, never creates another. + } finally { + await env.DB.prepare('UPDATE app_operations SET lease_owner=NULL,lease_until=0,not_before=? WHERE id=? AND fence=?') + .bind(Math.max(now+5000,await cooldown(env.DB,'replicate')),claimed.id,claimed.fence).run(); + } +} +export async function drainSplitQueue(env: Env,complete: Complete) { + if(env.AUTH_MODE!=='cail')return; + await recoverExpired(env.DB); + const active=await env.DB.prepare("SELECT * FROM app_operations WHERE kind='split' AND state IN ('processing','reconciling') AND provider_id IS NOT NULL AND not_before<=? ORDER BY updated_at LIMIT 4") + .bind(Date.now()).all(); + for(const op of active.results)await reconcileSplit(env,op,complete); + if(env.SPLIT_STARTS_DISABLED==='true')return; + for(const phase of ['fetch','split'] as const) {const op=await claimNext(env.DB,phase);if(op)await start(env,op);} +} + +export async function recoverCallback(env: Env,operation: Operation,externalId: string) { + if(operation.provider_id||!['starting','reconciling'].includes(operation.state))return; + // Provider API evidence must bind its prediction to our exact callback. + if(!await getBackend(env).confirmStart?.(externalId,operationWebhook(env,operation.id)))return; + const accepted=await env.DB.prepare(`UPDATE app_operations SET provider_id=?,state='processing',lease_until=0,lease_owner=NULL,updated_at=? + WHERE id=? AND provider_id IS NULL AND state IN ('starting','reconciling')`).bind(externalId,Date.now(),operation.id).run(); + if(accepted.meta.changes&&operation.phase==='split')await env.DB.prepare("UPDATE jobs SET external_id=?,status='processing' WHERE id=? AND status<>'failed'").bind(externalId,operation.job_id).run(); +} diff --git a/src/separation/replicate.ts b/src/separation/replicate.ts index 3b609fb..b52dc30 100644 --- a/src/separation/replicate.ts +++ b/src/separation/replicate.ts @@ -2,6 +2,7 @@ import type { Env } from '../env'; import { DEFAULT_DEMUCS_MODEL, getReplicateRunner, replicateVersion } from './options'; import type { SeparationBackend, SeparationResult, SeparationStartRequest } from './types'; import { readBoundedResponse } from '../http/bounded-response.ts'; +import { retryAt, UpstreamError } from '../reliability/retry.ts'; // Replicate-hosted Demucs (ryan5453/demucs), running the htdemucs_ft // fine-tuned model with MP3 output. ~$0.04–0.05/song on A40. @@ -15,6 +16,7 @@ interface ReplicatePrediction { status: 'starting' | 'processing' | 'succeeded' | 'failed' | 'canceled'; output?: Record; error?: unknown; + webhook?: string; } const API = 'https://api.replicate.com/v1'; @@ -34,18 +36,6 @@ export function replicateBackend(env: Env): SeparationBackend { 'Content-Type': 'application/json', }; - // Low-credit accounts get "burst of 1" rate limits; a YouTube import makes - // two predictions back-to-back (fetch, then separate), so honor 429s. - const fetchRetrying429 = async (url: string, init: RequestInit): Promise => { - for (let attempt = 0; ; attempt++) { - const res = await fetch(url, { ...init, redirect: 'manual', signal: AbortSignal.timeout(30000) }); - if (res.status !== 429 || attempt >= 3) return res; - await res.body?.cancel().catch(() => undefined); - const retryAfter = Number(res.headers.get('retry-after')) || 5; - await new Promise((r) => setTimeout(r, Math.min(retryAfter + 1, 15) * 1000)); - } - }; - const backend: SeparationBackend = { async start(req: SeparationStartRequest): Promise<{ externalId: string }> { // The catalogue owns the version and the input shape; this backend only @@ -59,7 +49,8 @@ export function replicateBackend(env: Env): SeparationBackend { if (!version) { throw new Error(`${runner.versionVar} is not configured`); } - const res = await fetchRetrying429(`${API}/predictions`, { + let res: Response; + try { res = await fetch(`${API}/predictions`, { method: 'POST', headers, body: JSON.stringify({ @@ -68,13 +59,16 @@ export function replicateBackend(env: Env): SeparationBackend { webhook: req.webhookUrl, webhook_events_filter: ['completed'], }), - }); + redirect: 'manual', signal: AbortSignal.timeout(30000), + }); } catch { throw new UpstreamError('start_uncertain','uncertain'); } if (!res.ok) { await res.body?.cancel().catch(() => undefined); - throw new Error(`The separator could not start (${res.status}). Please try again.`); + throw new UpstreamError(res.status === 429 ? 'provider_capacity' : res.status === 402 ? 'provider_credit' : 'provider_rejected', + res.status >= 500 ? 'uncertain' : 'rejected',res.status,res.status === 429 ? retryAt(res.headers.get('retry-after')) : 0); } - const prediction = await predictionJson(res); - if (typeof prediction.id !== 'string' || !/^[a-zA-Z0-9_-]{1,128}$/.test(prediction.id)) throw new Error('The separator returned an invalid prediction'); + let prediction: ReplicatePrediction; + try { prediction = await predictionJson(res); } catch { throw new UpstreamError('start_response_unreadable','uncertain'); } + if (typeof prediction.id !== 'string' || !/^[a-zA-Z0-9_-]{1,128}$/.test(prediction.id)) throw new UpstreamError('start_id_missing','uncertain'); return { externalId: prediction.id }; }, @@ -101,6 +95,14 @@ export function replicateBackend(env: Env): SeparationBackend { } return backend.parseResult(await predictionJson(res)); }, + + async confirmStart(externalId: string, webhookUrl: string): Promise { + if (!/^[a-zA-Z0-9_-]{1,128}$/.test(externalId)) return false; + const res=await fetch(`${API}/predictions/${externalId}`,{headers,redirect:'manual',signal:AbortSignal.timeout(20000)}); + if (!res.ok) { await res.body?.cancel(); return false; } + const prediction=await predictionJson(res); + return prediction.id===externalId && prediction.webhook===webhookUrl; + }, }; return backend; diff --git a/src/separation/types.ts b/src/separation/types.ts index bfea9d5..7865831 100644 --- a/src/separation/types.ts +++ b/src/separation/types.ts @@ -33,4 +33,6 @@ export interface SeparationBackend { parseResult(payload: unknown): SeparationResult; /** Poll the provider directly (reconciliation fallback if a webhook is missed). */ fetchStatus(externalId: string): Promise; + /** Prove an otherwise unknown accepted start belongs to this operation. */ + confirmStart?(externalId: string, webhookUrl: string): Promise; } diff --git a/src/youtube.ts b/src/youtube.ts index 5d02a3d..c45aa60 100644 --- a/src/youtube.ts +++ b/src/youtube.ts @@ -5,6 +5,7 @@ import { Innertube, type Types } from 'youtubei.js/cf-worker'; import type { Env } from './env'; import { readBoundedResponse, responseMediaType } from './http/bounded-response.ts'; +import { retryAt, UpstreamError } from './reliability/retry.ts'; const MAX_DURATION_SECONDS = 15 * 60; // cost/scope guard for class use const MAX_AUDIO_BYTES = 100 * 1024 * 1024; @@ -14,9 +15,8 @@ const MODEL_NAME_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; const MODEL_VERSION_PATTERN = /^[0-9a-f]{64}$/; const SAFE_TOKEN_PATTERN = /^[^\s\u0000-\u001f\u007f]+$/; -// YouTube bot-checks the default WEB client from datacenter IPs; the app -// clients are usually exempt. Try them in order until one is playable. -const CLIENTS: Types.InnerTubeClient[] = ['IOS', 'ANDROID', 'TV', 'TV_EMBEDDED', 'WEB_EMBEDDED', 'MWEB', 'WEB']; +// One supported client. A source restriction is not a reason to rotate clients. +const CLIENTS: Types.InnerTubeClient[] = ['WEB']; const REPLICATE_API = 'https://api.replicate.com/v1'; const REPLICATE_POLL_MS = 2000; @@ -78,7 +78,7 @@ function replicateConfiguration(env: Env) { } export function parseYouTubeVideoId(url: string): string | null { - if (url.length > 2048) return null; + if (typeof url !== 'string' || url.length > 2048) return null; let u: URL; try { u = new URL(url.trim()); @@ -134,7 +134,7 @@ export async function fetchYouTubeAudio(url: string, env: Env): Promise { (error instanceof YouTubeError && error.code === 'youtube_fetch_timeout') || Date.now() >= deadline ) throw youtubeTimedOut(); - continue; // client-specific parse/availability failure — try the next one + if (/private|sign.?in|age.?restrict|region|country|unavailable|removed|drm|bot|captcha|challenge/i.test(error instanceof Error ? error.message : '')) throw sourceDenied(); + throw providerUnavailable(); } const basic = info.basic_info; @@ -362,7 +363,7 @@ async function fetchViaInnertube(videoId: string): Promise { ); } if (info.playability_status && info.playability_status.status !== 'OK') { - continue; + throw sourceDenied(); } // Audio-only M4A/AAC — Demucs ingests M4A directly, no transcoding needed. @@ -377,7 +378,8 @@ async function fetchViaInnertube(videoId: string): Promise { (error instanceof YouTubeError && error.code === 'youtube_fetch_timeout') || Date.now() >= deadline ) throw youtubeTimedOut(); - continue; // no matching format via this client — try the next one + if (/private|sign.?in|age.?restrict|region|country|unavailable|removed|drm|bot|captcha|challenge/i.test(error instanceof Error ? error.message : '')) throw sourceDenied(); + throw invalidAudioResponse(); } const data = await readBoundedResponse(new Response(stream), { maximumBytes: MAX_AUDIO_BYTES, @@ -398,7 +400,7 @@ async function fetchViaInnertube(videoId: string): Promise { throw new YouTubeError( 'Video is not playable with the available import clients.', 'youtube_fetch_blocked', - true + false ); } @@ -769,11 +771,46 @@ function normalizePredictionError(raw: string): YouTubeError { true ); } - return new YouTubeError( - 'YouTube blocked this import. Try a different video, upload your own audio, or browse the Crate for openly licensed music.', - 'youtube_fetch_blocked', - true - ); + return sourceDenied(); +} + +function sourceDenied(): YouTubeError { + return new YouTubeError('This source cannot be imported. Upload an original or licensed audio file, use your own Studio/Takeout download, or choose an openly licensed Archive track.','youtube_source_denied',false); +} + +/** Durable coordinator API. Exactly one POST; no polling or client fallbacks + * are hidden inside it. A lost response cannot be classified as a failed start. + */ +export async function startYouTubeImport(url: string, env: Env, webhook: string): Promise { + const config=replicateConfiguration(env), videoId=parseYouTubeVideoId(url); + if (!config || !videoId) throw new UpstreamError('import_configuration','rejected'); + let res: Response; + try { res=await fetch(`${REPLICATE_API}/predictions`,{ + method:'POST',headers:{Authorization:`Bearer ${config.apiToken}`,'Content-Type':'application/json','Cancel-After':'4m'}, + body:JSON.stringify({version:config.modelVersion,input:{url:`https://www.youtube.com/watch?v=${videoId}`,max_duration:MAX_DURATION_SECONDS},webhook,webhook_events_filter:['completed']}), + redirect:'manual',signal:AbortSignal.timeout(20000), + }); } catch { throw new UpstreamError('import_start_uncertain','uncertain'); } + if (!res.ok) { + await res.body?.cancel().catch(()=>undefined); + throw new UpstreamError(res.status===429?'provider_capacity':res.status===402?'provider_credit':'import_start_rejected',res.status>=500?'uncertain':'rejected',res.status, + res.status===429?retryAt(res.headers.get('retry-after')):0); + } + try { return (await readPrediction(res,15000)).id; } + catch { throw new UpstreamError('import_start_response_unreadable','uncertain'); } +} + +export async function pollYouTubeImport(id: string, env: Env): Promise { + if (!PREDICTION_ID_PATTERN.test(id)) throw providerUnavailable(); + const config=replicateConfiguration(env); + if (!config) throw providerUnavailable(); + const deadline=Date.now()+AUDIO_DOWNLOAD_TIMEOUT_MS; + const res=await replicateFetch(`${REPLICATE_API}/predictions/${id}`,{headers:{Authorization:`Bearer ${config.apiToken}`}},15000); + if (!res.ok) { await res.body?.cancel(); throw new UpstreamError('import_poll_unavailable','uncertain',res.status,res.status===429?retryAt(res.headers.get('retry-after')):0); } + const prediction=await readPrediction(res,15000,id); + if (prediction.status==='starting'||prediction.status==='processing') return null; + if (prediction.status!=='succeeded'||!prediction.output) throw normalizePredictionError(prediction.error ?? ''); + const data=await downloadPredictionFile(prediction.output.audio,config.apiToken,deadline); + return validateFetchedAudio({data,title:prediction.output.title,durationSec:prediction.output.duration}); } function invalidAudioResponse(): YouTubeError { From c7aa244cd37086803a5b466f8508733a152e2766 Mon Sep 17 00:00:00 2001 From: zmuhls Date: Tue, 6 Oct 2026 14:39:21 -0400 Subject: [PATCH 03/25] Exercise operation concurrency and extractor failure boundaries --- cloudflare/daily-allowance.test.ts | 123 +++++++++++++++++------------ cloudflare/gateway.test.ts | 34 ++++++++ replicate-yt-audio/test_predict.py | 89 +++++++++++++++++++++ 3 files changed, 195 insertions(+), 51 deletions(-) create mode 100644 replicate-yt-audio/test_predict.py diff --git a/cloudflare/daily-allowance.test.ts b/cloudflare/daily-allowance.test.ts index a207c7d..f456fc2 100644 --- a/cloudflare/daily-allowance.test.ts +++ b/cloudflare/daily-allowance.test.ts @@ -1,60 +1,81 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; -import { TEST_SUBJECTS } from '@cuny-ai-lab/cail-identity/testing'; +import { readFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { SqliteD1 } from '../server/d1.ts'; -import { dailyWindow, reserveDailyRequest, splitAllowance } from '../src/daily-allowance.ts'; - -async function setup() { - const db = new SqliteD1(':memory:'); - db.applySchema(readFileSync(new URL('../schema.sql', import.meta.url), 'utf8')); - for (const subject of Object.values(TEST_SUBJECTS)) { - await db.prepare('INSERT OR IGNORE INTO app_users (subject) VALUES (?)').bind(subject).run(); - } - return db as unknown as D1Database; +import { dailyWindow, operationAllowance, splitAllowance } from '../src/daily-allowance.ts'; +import { OperationError, readOperation, reserveOperation, settleOperation } from '../src/reliability/ledger.ts'; +import { claimNext, recoverExpired, transition, QUEUE_CAPS } from '../src/reliability/queue.ts'; +import { retryAt } from '../src/reliability/retry.ts'; +const alice='cail-'+'a'.repeat(32),bob='cail-'+'b'.repeat(32); +const now=Date.parse('2026-12-31T23:59:59.999Z'); +function setup(t:{after:(f:()=>void)=>void}) { + const dir=mkdtempSync(join(tmpdir(),'stem-ledger-')),left=new SqliteD1(join(dir,'db.sqlite')); + left.applySchema(readFileSync(new URL('../schema.sql',import.meta.url),'utf8')); + const right=new SqliteD1(join(dir,'db.sqlite'));t.after(()=>rmSync(dir,{recursive:true,force:true})); + return [left,right] as unknown as D1Database[]; } +async function user(db:D1Database,subject:string){await db.prepare('INSERT OR IGNORE INTO app_users(subject) VALUES(?)').bind(subject).run();} +const reserve=(db:D1Database,i:number,subject=alice,kind:'split'|'chat'='split',at=now)=>reserveOperation(db,{subject,courseId:i%2?'course-a':'course-b',kind,key:`request-number-${String(i).padStart(4,'0')}`,fingerprint:`fingerprint-${i}`,phase:kind==='split'?'split':'chat',now:at}); -test('ten signed-in requests each, with no shared split cap and no client-selected reset', async () => { - const db = await setup(); - const today = new Date('2026-09-29T20:00:00Z'); - for (const subject of [TEST_SUBJECTS.alice, TEST_SUBJECTS.bob, TEST_SUBJECTS.carol]) { - const reservations = await Promise.all(Array.from({ length: 30 }, () => reserveDailyRequest(db, subject, 'split', today))); - assert.equal(reservations.filter(result => result.allowed).length, 10); - assert.deepEqual(await splitAllowance(db, subject, today), { limit: 10, used: 10, remaining: 0, resetsAt: '2026-09-30T00:00:00.000Z' }); - } - assert.equal((await db.prepare("SELECT COUNT(*) AS n FROM app_request_reservations WHERE scope = 'split'").first<{ n: number }>())?.n, 30); +test('two connections atomically admit fifteen across courses; failures release exactly once',async t=>{ + const db=setup(t);for(const u of [alice,bob])await user(db[0],u); + const results=await Promise.allSettled(Array.from({length:30},(_,i)=>reserve(db[i%2],i))); + const admitted=results.flatMap(r=>r.status==='fulfilled'?[r.value.operation]:[]);assert.equal(admitted.length,15); + await Promise.all(admitted.map((op,i)=>settleOperation(db[i%2],op.id,i<7?'succeeded':'failed'))); + await Promise.all(admitted.slice(7).map(op=>settleOperation(db[1],op.id,'failed'))); + assert.deepEqual(await splitAllowance(db[0],alice,new Date(now)),{limit:15,completed:7,inProgress:0,remaining:8,resetsAt:'2027-01-01T00:00:00.000Z'}); + const replacements=await Promise.allSettled(Array.from({length:20},(_,i)=>reserve(db[i%2],100+i)));assert.equal(replacements.filter(r=>r.status==='fulfilled').length,8); + await reserve(db[1],400,bob);assert.equal((await splitAllowance(db[1],bob,new Date(now))).remaining,14); + assert.equal(await settleOperation(db[0],admitted[8].id,'succeeded'),false); }); - -test('resets at UTC midnight, independently of DST, month/year boundaries and old reservations', async () => { - const db = await setup(); - const before = new Date('2026-12-31T23:59:59.999Z'); - assert.deepEqual(dailyWindow(before), { day: '2026-12-31', resetsAt: '2027-01-01T00:00:00.000Z', retryAfter: 1 }); - assert.equal(dailyWindow(new Date('2026-11-01T00:00:00Z')).retryAfter, 86400); - await Promise.all(Array.from({ length: 10 }, () => reserveDailyRequest(db, TEST_SUBJECTS.alice, 'split', before))); - assert.equal((await reserveDailyRequest(db, TEST_SUBJECTS.alice, 'split', before)).allowed, false); - const after = new Date('2027-01-01T00:00:00Z'); - assert.equal((await splitAllowance(db, TEST_SUBJECTS.alice, after)).remaining, 10); - assert.equal((await reserveDailyRequest(db, TEST_SUBJECTS.alice, 'split', after)).allowed, true); - assert.equal((await splitAllowance(db, TEST_SUBJECTS.alice, after)).remaining, 9); +test('sixty human messages admit fifty; one replay/partial/failure uses one reservation',async t=>{ + const db=setup(t);await user(db[0],alice); + const results=await Promise.allSettled(Array.from({length:60},(_,i)=>reserve(db[i%2],i,alice,'chat')));assert.equal(results.filter(r=>r.status==='fulfilled').length,50); + const first=await reserve(db[1],0,alice,'chat');assert.equal(first.created,false); + await assert.rejects(reserveOperation(db[1],{subject:alice,courseId:'course-b',kind:'chat',key:'request-number-0000',fingerprint:'different',phase:'chat',now}),OperationError); + await settleOperation(db[0],first.operation.id,'partial',null,{text:'Partial answer'}); + const second=await reserve(db[0],1,alice,'chat');await settleOperation(db[0],second.operation.id,'failed'); + const count=await operationAllowance(db[0],alice,'chat',new Date(now));assert.equal(count.completed,1);assert.equal(count.inProgress,48);assert.equal(count.remaining,1); + await reserve(db[1],70,alice,'chat');await assert.rejects(reserve(db[0],71,alice,'chat'),OperationError); }); - -test('legacy guide guards remain independent of splits, including its shared ceiling', async () => { - const db = await setup(); - const now = new Date('2026-09-29T20:00:00Z'); - const day = dailyWindow(now).day; - for (let i = 0; i < 499; i++) { - await db.prepare('INSERT INTO app_request_reservations (id, subject, scope, day) VALUES (?, ?, ?, ?)') - .bind(`fixture-${i}`, TEST_SUBJECTS.bob, 'guide', day).run(); - } - assert.equal((await reserveDailyRequest(db, TEST_SUBJECTS.bob, 'guide', now)).allowed, false); - assert.equal((await reserveDailyRequest(db, TEST_SUBJECTS.alice, 'guide', now)).allowed, true); - assert.equal((await reserveDailyRequest(db, TEST_SUBJECTS.alice, 'guide', now)).allowed, false); - assert.equal((await reserveDailyRequest(db, TEST_SUBJECTS.alice, 'split', now)).allowed, true); - assert.equal((await splitAllowance(db, TEST_SUBJECTS.alice, now)).remaining, 9); +test('submission-day accounting survives UTC midnight/year rollover and DST',async t=>{ + const db=setup(t);await user(db[0],alice); + assert.deepEqual(dailyWindow(new Date(now)),{day:'2026-12-31',resetsAt:'2027-01-01T00:00:00.000Z',retryAfter:1});assert.equal(dailyWindow(new Date('2026-11-01T00:00:00Z')).retryAfter,86400); + const op=await reserve(db[0],0);await settleOperation(db[1],op.operation.id,'succeeded'); + assert.equal((await splitAllowance(db[0],alice,new Date(now))).completed,1);assert.equal((await splitAllowance(db[0],alice,new Date(now+1))).remaining,15); + await assert.rejects(db[0].prepare("UPDATE app_operations SET day='2027-01-01' WHERE id=?").bind(op.operation.id).run()); }); - -test('counter/storage failures never grant or report unused requests', async () => { - const db = { prepare: () => { throw new Error('fixture unavailable'); } } as unknown as D1Database; - await assert.rejects(reserveDailyRequest(db, TEST_SUBJECTS.alice, 'split')); - await assert.rejects(splitAllowance(db, TEST_SUBJECTS.alice)); +test('twenty students with two coordinators make fair progress within global/per-user caps',async t=>{ + const db=setup(t),seen=new Set(); + for(let i=0;i<20;i++){const subject='cail-'+String(i).padStart(32,'0');await user(db[0],subject);await reserve(db[i%2],i,subject);await reserve(db[(i+1)%2],i+30,subject);} + for(let i=0;i<20;i++){ + const claims=(await Promise.all([claimNext(db[0],'split',now+1),claimNext(db[1],'split',now+1)])).filter(Boolean);assert.equal(claims.length,QUEUE_CAPS.split); + const op=claims[0]!;assert.equal(seen.has(op.subject),false);seen.add(op.subject);await settleOperation(db[i%2],op.id,'succeeded'); + } + assert.equal(seen.size,20); +}); +test('lost starts retain capacity; expiry fences old Worker completions without another POST',async t=>{ + const db=setup(t);await user(db[0],alice);await reserve(db[0],1); + const op=(await claimNext(db[0],'split',now+1))!;await transition(db[0],op,'starting'); + await db[0].prepare("INSERT INTO operation_attempts(id,operation_id,phase,provider,outcome,created_at,updated_at) VALUES('attempt',?,'split','replicate','starting',?,?)").bind(op.id,now,now).run(); + await recoverExpired(db[1],now+600001);assert.equal((await readOperation(db[1],op.id))?.state,'reconciling');assert.equal(await transition(db[0],op,'succeeded'),false); + assert.equal(await claimNext(db[0],'split',now+600002),null);assert.equal((await splitAllowance(db[0],alice,new Date(now))).inProgress,1); +}); +test('Retry-After 90/date never shortens server floor; malformed headers use bounded jitter',()=>{ + assert.equal(retryAt('90',now),now+90000);const date=new Date(now+120001).toUTCString();assert.equal(retryAt(date,now),Date.parse(date)); + assert.equal(retryAt(null,now,1,()=>0),now+2000);assert.equal(retryAt('garbage',now,1,()=>0.5),now+3000);assert.equal(retryAt('-1',now,0,()=>0),now+1000); +}); +test('quota rejection/duplicate key create no orphan jobs; late success cannot revive failure',async t=>{ + const db=setup(t);await user(db[0],alice); + const submit=(id:string,key:string)=>reserveOperation(db[0],{subject:alice,courseId:null,kind:'split',key,fingerprint:key,phase:'split',jobId:id,now,statements:operationId=>[db[0].prepare("INSERT INTO jobs(id,filename,source_key) SELECT id,'fixture','uploads/fixture/a.mp3' FROM app_operations WHERE id=?").bind(operationId)]}); + await submit('job-a','request-duplicate-0001');await submit('job-b','request-duplicate-0001');for(let i=0;i<14;i++)await reserve(db[i%2],i+100); + await assert.rejects(submit('job-full','request-quota-full-01'),OperationError);assert.equal((await db[0].prepare('SELECT COUNT(*) AS n FROM jobs').first<{n:number}>())?.n,1); + await db[0].prepare("UPDATE jobs SET status='failed' WHERE id='job-a'").run();await assert.rejects(db[0].prepare("UPDATE jobs SET status='done' WHERE id='job-a'").run()); +}); +test('migration keeps old attempt evidence separate and never invents successful charges',async t=>{ + const db=setup(t);await user(db[0],alice);await db[0].prepare("INSERT INTO app_request_reservations(id,subject,scope,day) VALUES('old',?,'split','2026-12-31')").bind(alice).run(); + (db[0] as unknown as SqliteD1).applySchema(readFileSync(new URL('../migrations/0022-reliable-operations.sql',import.meta.url),'utf8')); + assert.equal((await splitAllowance(db[0],alice,new Date(now))).completed,0);assert.equal((await db[0].prepare('SELECT COUNT(*) AS n FROM app_request_reservations').first<{n:number}>())?.n,1); }); diff --git a/cloudflare/gateway.test.ts b/cloudflare/gateway.test.ts index 37fbd6f..518c484 100644 --- a/cloudflare/gateway.test.ts +++ b/cloudflare/gateway.test.ts @@ -61,3 +61,37 @@ test('unavailable aggregate quota stays unavailable and is never a local permiss const transport = gatewayForRequest({ fetch: async () => { calls++; return Response.json({ error: { code: 'quota_unavailable', message: 'Unavailable' } }, { status: 503 }); } } as unknown as Fetcher, 'gateway-leg', request(), 'test'); await assert.rejects(transport.quota()); assert.equal(calls, 1); }); + +test('ordered fallback uses only documented capacity rejections; policy/quota/auth/unknown do not fall back',async()=>{ + for(const [code,status,shouldRetry,fallback] of [ + ['upstream_rate_limited',429,true,true],['upstream_unavailable',503,true,true],['model_unavailable',503,true,true], + ['quota_exceeded',429,false,false],['rate_limit_unknown',429,false,false],['upstream_auth_error',401,false,false], + ['upstream_payment_required',402,false,false],['upstream_timeout',504,false,false],['upstream_unavailable',503,false,false], + ] as const) { + const models:string[]=[]; + const transport=gatewayForRequest({fetch:async(outbound:Request)=>{ + assert.equal(outbound.headers.get('x-cail-identity-jwt'),'same-verified-subject'); + const body=await outbound.json() as {model:string;messages:unknown[]};models.push(body.model);assert.deepEqual(body.messages,params.messages); + if(models.length===1)return Response.json({error:{code,type:code,param:null,message:'DO NOT DISPLAY',cail:{should_retry:shouldRetry}}},{status,headers:{'x-should-retry':String(shouldRetry)}}); + return sse([{choices:[{delta:{content:'A calm backup answer.'},finish_reason:null}]},finished]); + }} as unknown as Fetcher,'same-verified-subject',request(),'test'); + const pending=transport.stream({...env,ASSISTANT_FALLBACK_MODELS:'deepseek-v4-flash-0731'},params,()=>{}); + if(fallback){assert.equal((await pending).model,'deepseek-v4-flash-0731');assert.deepEqual(models,['glm-5.2','deepseek-v4-flash-0731']);} + else {await assert.rejects(pending,AssistantError);assert.deepEqual(models,['glm-5.2']);} + } +}); +test('both failures stop at two, stream/tool output forbids fallback, and mis-mapped backup fails closed',async()=>{ + for(const mode of ['both','partial','tools','trailing']) { + let calls=0; + const error={error:{code:'upstream_unavailable',type:'upstream_unavailable',param:null,message:'secret',cail:{should_retry:true}}}; + const transport=gatewayForRequest({fetch:async()=>{calls++; + if(mode==='both')return Response.json(error,{status:503,headers:{'x-should-retry':'true'}}); + const delta=mode==='tools'?{tool_calls:[{index:0,id:'a',function:{name:'solo',arguments:'{"stem":"vocals"}'}}]}:{content:'Partial.'}; + return sse([{choices:[{delta,finish_reason:null}]},...(mode==='trailing'?[finished]:[]),error]); + }} as unknown as Fetcher,'identity',request(),'test'); + await assert.rejects(transport.stream({...env,ASSISTANT_FALLBACK_MODELS:'deepseek-v4-flash-0731'},params,()=>{}),AssistantError); + assert.equal(calls,mode==='both'?2:1); + } + let calls=0;const transport=gatewayForRequest({fetch:async()=>{calls++;return sse([finished]);}} as unknown as Fetcher,'identity',request(),'test'); + await assert.rejects(transport.stream({...env,ASSISTANT_FALLBACK_MODELS:'deepseek-flash'},params,()=>{}),AssistantError);assert.equal(calls,0); +}); diff --git a/replicate-yt-audio/test_predict.py b/replicate-yt-audio/test_predict.py new file mode 100644 index 0000000..d55d0fa --- /dev/null +++ b/replicate-yt-audio/test_predict.py @@ -0,0 +1,89 @@ +"""Free extractor fault fixtures; no YouTube/Replicate requests.""" +import importlib.util +import json +import os +import sys +import tempfile +import threading +import time +import types +import unittest +from pathlib import Path +from unittest.mock import patch + +# Cog is an image dependency; these tests exercise our code, not a Cog upload. +class Model: + def __init__(self, **kwargs): self.__dict__.update(kwargs) +cog = types.ModuleType('cog') +cog.BaseModel = Model +cog.BasePredictor = object +cog.Input = lambda **kwargs: kwargs.get('default') +cog.Path = Path +sys.modules['cog'] = cog +spec = importlib.util.spec_from_file_location('predict', Path(__file__).with_name('predict.py')) +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) + +class ExtractorTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root_patch = patch.object(m, 'ROOT', Path(self.tmp.name)) + self.root_patch.start() + self.addCleanup(self.root_patch.stop) + self.addCleanup(self.tmp.cleanup) + + def fixture(self, failure=None, barrier=None): + calls=[] + def run(cmd, deadline=None): + calls.append(cmd) + if '-J' in cmd: + if failure == 'denial': raise ValueError('source_denied') + return json.dumps({'title':'Fixture','duration':12,'is_live':failure=='live'}) + if cmd[0]=='yt-dlp': + target=cmd[cmd.index('-o')+1].replace('%(ext)s','m4a') + if failure=='disk': raise OSError('disk full') + Path(target).write_bytes(b'fixture' * 300) + if barrier: barrier.wait(timeout=3) + return '' + if cmd[0]=='ffprobe': return json.dumps({'format':{'duration':0 if failure=='zero' else 12},'streams':[{'codec_type':'audio'}]}) + if failure=='decode': raise ValueError('invalid_audio_response') + return '' + return run,calls + + def test_urls_and_option_injection(self): + for url in ['https://youtu.be/abcdefghijk','https://youtube.com/watch?v=abcdefghijk','https://music.youtube.com/watch?v=abcdefghijk','https://youtube.com/shorts/abcdefghijk','https://youtube.com/embed/abcdefghijk','https://youtube.com/live/abcdefghijk']: + self.assertEqual(m.canonical_url(url),'https://www.youtube.com/watch?v=abcdefghijk') + for url in ['--exec=evil','file:///etc/passwd','https://youtube.com.evil/watch?v=abcdefghijk','https://u:p@youtube.com/watch?v=abcdefghijk','https://youtube.com:88/watch?v=abcdefghijk','http://127.0.0.1/','https://youtube.com/playlist?list=abc']: + with self.assertRaises(ValueError): m.canonical_url(url) + + def test_denial_live_disk_and_bad_media_leave_no_partial_outputs(self): + for failure in ['denial','live','disk','zero','decode']: + predictor=m.Predictor();run,calls=self.fixture(failure) + with patch.object(predictor,'_run',side_effect=run): + with self.assertRaises((ValueError,OSError)):predictor.predict('https://youtu.be/abcdefghijk',900) + self.assertEqual(list(m.ROOT.iterdir()),[]) + if failure=='denial':self.assertEqual(len(calls),1) + + def test_concurrent_outputs_have_unique_paths_and_stale_restart_cleanup(self): + barrier=threading.Barrier(2);outputs=[];errors=[] + def invoke(): + predictor=m.Predictor();run,calls=self.fixture(barrier=barrier) + try: + with patch.object(predictor,'_run',side_effect=run):outputs.append(predictor.predict('https://youtu.be/abcdefghijk',900)) + for cmd in calls: + if cmd[0]=='yt-dlp': + self.assertEqual(cmd[-2],'--');self.assertIn('--retries',cmd);self.assertNotIn('--extractor-args',cmd) + except BaseException as error:errors.append(error) + threads=[threading.Thread(target=invoke) for _ in range(2)] + for thread in threads:thread.start() + for thread in threads:thread.join(5) + self.assertEqual(errors,[]);self.assertEqual(len(outputs),2);self.assertNotEqual(outputs[0].audio,outputs[1].audio) + m.cleanup_stale(time.time()+3700);self.assertEqual(list(m.ROOT.iterdir()),[]) + + def test_real_timeout_kills_child_process_and_missing_dependency_fails_closed(self): + with self.assertRaisesRegex(ValueError,'timeout'): + m.Predictor()._run([sys.executable,'-c','import time;time.sleep(20)'],time.monotonic()+.03) + with patch.object(m.shutil,'which',return_value=None): + with self.assertRaisesRegex(RuntimeError,'dependency'):m.Predictor().setup() + +if __name__=='__main__':unittest.main() From 22f0a97e75e4626956e32845eb009231af9c02de Mon Sep 17 00:00:00 2001 From: zmuhls Date: Tue, 6 Oct 2026 14:39:21 -0400 Subject: [PATCH 04/25] Show successful and pending allowances with saved request recovery --- public/account.html | 1 + public/account.js | 8 ++++++-- public/app.js | 20 ++++++++++++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/public/account.html b/public/account.html index 4f49543..90133e3 100644 --- a/public/account.html +++ b/public/account.html @@ -16,6 +16,7 @@

My account

Having trouble?
- + + diff --git a/public/index.html b/public/index.html index 4babc30..8c3e988 100644 --- a/public/index.html +++ b/public/index.html @@ -187,7 +187,8 @@

ENTER CLASS CODE

- + + diff --git a/public/session-guard.js b/public/session-guard.js index d7f1684..34cfe70 100644 --- a/public/session-guard.js +++ b/public/session-guard.js @@ -33,7 +33,7 @@ } function clear() { if (clearing) return; - clearing = true; suspend(); cleanup(); + clearing = true; window.StemUsage?.clear(); suspend(); cleanup(); const main = document.querySelector('main'); if (main) main.replaceChildren(); message.textContent = 'Your account changed. Reloading your workspace…'; retry.hidden = true; @@ -71,7 +71,7 @@ return checking; } window.StemSessionGuard = { - start(value, onClear, onPause) { subject = value; cleanup = onClear || cleanup; pause = onPause || pause; }, + start(value, onClear, onPause) { subject = value; cleanup = onClear || cleanup; pause = onPause || pause; window.StemUsage?.start(); }, observe, check, clear, }; window.addEventListener('focus', () => void check()); diff --git a/public/teacher.html b/public/teacher.html index 96bb631..8ee771f 100644 --- a/public/teacher.html +++ b/public/teacher.html @@ -132,5 +132,6 @@

More

Having trouble?
+ diff --git a/public/usage.js b/public/usage.js new file mode 100644 index 0000000..6b946d6 --- /dev/null +++ b/public/usage.js @@ -0,0 +1,36 @@ +/* Discrete, content-free product diagnostics. Playback never waits for this. + * In-memory queue only: at most 50 events, one network retry, no offline archive. + * Stable IDs make that retry idempotent; page close/offline/rate limits can lose + * events. These are observations, not billing or guaranteed delivery receipts. */ +(() => { + let active=false,course='personal',queue=[],timer=null,sending=false,generation=0,controller=null; + const allowed=new Set(['session_start','page_view','playback_start','playback_stop','seek','download_intent']); + function record(type,detail={}) { + if(!active||!allowed.has(type)||queue.length>=50)return; + const event={id:crypto.randomUUID(),type}; + if(typeof detail.jobId==='string'&&/^[a-zA-Z0-9_-]{1,80}$/.test(detail.jobId))event.jobId=detail.jobId; + if(Number.isFinite(detail.durationMs))event.durationMs=Math.min(86400000,Math.max(0,Math.round(detail.durationMs))); + if(Number.isFinite(detail.position))event.positionBucket=Math.min(90,Math.max(0,Math.floor(detail.position/10))); + queue.push(event);if(!timer)timer=setTimeout(flush,1200); + } + async function flush() { + clearTimeout(timer);timer=null; + if(sending||!active||!queue.length)return; + sending=true;const current=generation;controller=new AbortController();const batch=queue.splice(0,10); + try { + for(let attempt=0;attempt<2;attempt++) { + if(!active||generation!==current)break; + try { + const response=await fetch('/api/usage-events',{method:'POST',credentials:'same-origin',keepalive:true,signal:AbortSignal.any([AbortSignal.timeout(5000),controller.signal]), + headers:{'Content-Type':'application/json','X-Stem-Course':course},body:JSON.stringify({events:batch})}); + if(response.ok||response.status<500)break; + }catch{} + if(attempt===0)await new Promise(resolve=>setTimeout(resolve,2000)); + } + }finally{sending=false;if(queue.length&&active)timer=setTimeout(flush,1200);} + } + window.StemUsage={record,setCourse(value){course=typeof value==='string'?value:'personal';},start(){ + if(active)return;generation++;active=true;record('session_start');record('page_view'); + },clear(){generation++;controller?.abort();active=false;queue=[];clearTimeout(timer);timer=null;}}; + window.addEventListener('pagehide',()=>void flush()); +})(); diff --git a/schema.sql b/schema.sql index c249792..4c1c32f 100644 --- a/schema.sql +++ b/schema.sql @@ -716,16 +716,26 @@ CREATE TABLE IF NOT EXISTS operation_events ( phase TEXT NOT NULL, fence INTEGER NOT NULL, code TEXT, - at INTEGER NOT NULL + at INTEGER NOT NULL, + event_type TEXT NOT NULL DEFAULT 'lifecycle', + attempt_id TEXT, + kind TEXT NOT NULL, + duration_ms INTEGER NOT NULL DEFAULT 0, + model TEXT, + fallback INTEGER NOT NULL DEFAULT 0, + quota_effect TEXT NOT NULL DEFAULT 'none' ); CREATE TRIGGER IF NOT EXISTS operation_created AFTER INSERT ON app_operations BEGIN - INSERT INTO operation_events(operation_id,state,phase,fence,code,at) - VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at); + INSERT INTO operation_events(operation_id,state,phase,fence,code,at,kind,duration_ms,quota_effect) + VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at,NEW.kind, + MAX(0,NEW.updated_at-NEW.created_at),'reserved'); END; CREATE TRIGGER IF NOT EXISTS operation_changed AFTER UPDATE ON app_operations WHEN OLD.state<>NEW.state OR OLD.phase<>NEW.phase OR OLD.fence<>NEW.fence BEGIN - INSERT INTO operation_events(operation_id,state,phase,fence,code,at) - VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at); + INSERT INTO operation_events(operation_id,state,phase,fence,code,at,kind,duration_ms,quota_effect) + VALUES(NEW.id,NEW.state,NEW.phase,NEW.fence,NEW.error_code,NEW.updated_at,NEW.kind, + MAX(0,NEW.updated_at-NEW.created_at),CASE WHEN NEW.state IN ('succeeded','partial') THEN 'charged' + WHEN NEW.state IN ('failed','cancelled') THEN 'released' ELSE 'none' END); END; CREATE TRIGGER IF NOT EXISTS operation_terminal_fence BEFORE UPDATE ON app_operations WHEN OLD.state IN ('succeeded','partial','failed','cancelled') AND NEW.state<>OLD.state @@ -786,3 +796,37 @@ WHEN NEW.status IN ('done','failed') BEGIN lease_owner=NULL,lease_until=0,updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE job_id=NEW.id AND kind='split' AND state NOT IN ('succeeded','partial','failed','cancelled'); END; + +-- Attempts and lifecycle events are in the same transaction as their evidence. +-- No body, transcript, filename, source URL, name, email or credential is copied. +CREATE TRIGGER IF NOT EXISTS operation_attempt_created AFTER INSERT ON operation_attempts BEGIN + INSERT INTO operation_events(operation_id,state,phase,fence,code,at,event_type,attempt_id,kind,duration_ms,model,fallback) + SELECT NEW.operation_id,NEW.outcome,NEW.phase,o.fence,NEW.code,NEW.created_at,'attempt_start',NEW.id,o.kind,0,NEW.model, + CASE WHEN NEW.provider='cail-gateway' AND EXISTS(SELECT 1 FROM operation_attempts a WHERE a.operation_id=NEW.operation_id AND a.id<>NEW.id) THEN 1 ELSE 0 END + FROM app_operations o WHERE o.id=NEW.operation_id; +END; +CREATE TRIGGER IF NOT EXISTS operation_attempt_changed AFTER UPDATE ON operation_attempts +WHEN NEW.outcome<>OLD.outcome BEGIN + INSERT INTO operation_events(operation_id,state,phase,fence,code,at,event_type,attempt_id,kind,duration_ms,model,fallback) + SELECT NEW.operation_id,NEW.outcome,NEW.phase,o.fence,NEW.code,NEW.updated_at,'attempt_end',NEW.id,o.kind, + MAX(0,NEW.updated_at-NEW.created_at),NEW.model, + CASE WHEN NEW.provider='cail-gateway' AND NEW.model='deepseek-v4-flash-0731' THEN 1 ELSE 0 END + FROM app_operations o WHERE o.id=NEW.operation_id; +END; +CREATE INDEX IF NOT EXISTS operation_events_retention ON operation_events(at); +CREATE TABLE IF NOT EXISTS usage_events ( + event_id TEXT PRIMARY KEY, + actor_key TEXT NOT NULL, + actor_class TEXT NOT NULL CHECK(actor_class IN ('member','guest','anonymous')), + event_type TEXT NOT NULL, + source TEXT NOT NULL CHECK(source IN ('client','server')), + job_id TEXT, + outcome TEXT NOT NULL, + code TEXT, + http_status INTEGER, + duration_ms INTEGER, + position_bucket INTEGER, + at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS usage_events_retention ON usage_events(at); +CREATE INDEX IF NOT EXISTS usage_events_actor ON usage_events(actor_key,at); diff --git a/src/analysis/source.ts b/src/analysis/source.ts index 534f5c9..be72965 100644 --- a/src/analysis/source.ts +++ b/src/analysis/source.ts @@ -214,10 +214,14 @@ export async function prepareAuthoritativeAutoSource( sourceKey: string; /** Test seam for the local 30-day retention boundary. */ nowMs?: number; + /** Durable callers fence cleanup and may reuse their own completed snapshot. */ + isCurrent?: () => Promise; + allowExisting?: boolean; } ): Promise { validateInput(input); const snapshotKey = `auto-inputs/v1/${input.jobId}`; + const rollback=async()=>{try {if(!input.isCurrent||await input.isCurrent())await rollbackSnapshot(env,snapshotKey);}catch {/* Uncertain ownership never licenses deletion. */}}; let source: R2ObjectBody | null; try { @@ -242,7 +246,12 @@ export async function prepareAuthoritativeAutoSource( } try { - if (await env.AUDIO.head(snapshotKey)) { + const existing=await env.AUDIO.head(snapshotKey); + if(existing&&input.allowExisting&&existing.size===source.size&&existing.customMetadata?.purpose==='authoritative-auto-input-v1'&&existing.customMetadata?.sourceBytes===String(source.size)) { + await source.body.cancel().catch(()=>undefined); + return {snapshotKey,bytes:existing.size}; + } + if (existing) { throw new AuthoritativeAutoSourceError( 'snapshot_conflict', 'The immutable Auto source snapshot already exists' @@ -256,6 +265,7 @@ export async function prepareAuthoritativeAutoSource( ); } + if(input.isCurrent&&!await input.isCurrent()) {await source.body.cancel().catch(()=>undefined);throw new AuthoritativeAutoSourceError('snapshot_conflict','This source request is being recovered.');} const bounded = boundedCopyStream(source.body, source.size); const upload = fixedLengthUpload(bounded.stream, source.size); let stored: R2Object | null = null; @@ -275,7 +285,7 @@ export async function prepareAuthoritativeAutoSource( } catch { await bounded.cancel(); if (upload.pump) await Promise.allSettled([upload.pump]); - await rollbackSnapshot(env, snapshotKey); + await rollback(); throw bounded.failure() ?? new AuthoritativeAutoSourceError( 'snapshot_failed', 'The immutable Auto source snapshot could not be stored' @@ -284,7 +294,7 @@ export async function prepareAuthoritativeAutoSource( if (!stored || stored.size !== source.size || !bounded.completed()) { await bounded.cancel(); - await rollbackSnapshot(env, snapshotKey); + await rollback(); throw bounded.failure() ?? new AuthoritativeAutoSourceError( 'snapshot_failed', 'The immutable Auto source snapshot did not match the uploaded source size' @@ -295,14 +305,14 @@ export async function prepareAuthoritativeAutoSource( try { committed = await env.AUDIO.head(snapshotKey); } catch { - await rollbackSnapshot(env, snapshotKey); + await rollback(); throw new AuthoritativeAutoSourceError( 'snapshot_failed', 'The immutable Auto source snapshot could not be verified' ); } if (!committed || committed.size !== source.size) { - await rollbackSnapshot(env, snapshotKey); + await rollback(); throw new AuthoritativeAutoSourceError( 'snapshot_failed', 'The immutable Auto source snapshot could not be verified' diff --git a/src/index.ts b/src/index.ts index c6828f2..d7015ab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,5 @@ +import { retryAt } from './reliability/retry.ts'; +import { parseClientUses, recordClientUses, recordServerUse, usageSummary } from './reliability/observability.ts'; import classroomRoutes from './classroom/routes.ts'; import { courseAssignmentStatement, jobCourse, jobPermission, CourseError, courseErrorResponse } from './classroom/access.ts'; import { beginCourseTurn, finishCourseTurn, conversationPage, resetCourseConversation } from './classroom/conversations.ts'; @@ -8,7 +10,7 @@ import { createMiddleware } from 'hono/factory'; import type { Env } from './env'; import { splitAllowance, operationAllowance, dailyWindow } from './daily-allowance.ts'; import { submitSplit, drainSplitQueue, reconcileSplit, recoverCallback } from './reliability/splits.ts'; -import { readOperation, OperationError, fingerprint } from './reliability/ledger.ts'; +import { type Operation, readOperation, OperationError, fingerprint } from './reliability/ledger.ts'; import { cancelOperation } from './reliability/queue.ts'; import { reserveAssistant, assistantReceipt } from './reliability/assistant.ts'; import { validMp3Frames } from './reliability/media.ts'; @@ -182,6 +184,12 @@ async function sha256Text(value: string): Promise { type AppContext = { Bindings: Env; Variables: { principal?: AppPrincipal } }; const app = new Hono(); +app.use('/api/*', async (c,next)=>{ + const started=Date.now(); + await next(); + await recordServerUse(c.env,c.req.raw,c.res,c.get('principal'),started); +}); + app.use('/api/*', async (c, next) => { if (c.env.AUTH_MODE === 'cail') { const denied = await authorizeCailRequest(c.req.raw, c.env, (principal) => c.set('principal', principal)); @@ -197,6 +205,25 @@ app.use('/api/*', async (c, next) => { app.route('/api/classroom', classroomRoutes); +app.post('/api/usage-events',async c=>{ + const principal=c.get('principal'); + if(!principal||c.env.AUTH_MODE!=='cail')return c.json({error:'Sign in to continue.'},401); + const parsed=await boundedJson(c,8192);if('response'in parsed)return parsed.response; + const events=parseClientUses(parsed.value);if(!events)return c.json({error:'Invalid usage event batch.'},400); + for(const jobId of new Set(events.map(e=>e.jobId).filter((id):id is string=>!!id))) { + if(!await jobPermission(c.env,principal,jobId))return c.json({error:'Job not found'},404); + } + await recordClientUses(c.env.DB,principal,events); + return c.json({accepted:true,capture:'best_effort'}); +}); +app.get('/api/admin/usage-events',async c=>{ + if(c.get('principal')?.role!=='admin')return c.json({error:'Not allowed'},403); + const days=Number(c.req.query('days')??7); + if(!Number.isInteger(days)||days<1||days>30)return c.json({error:'days must be 1–30.'},400); + return c.json(await usageSummary(c.env.DB,days)); +}); + + app.get('/api/runtime', (c) => c.json({ authMode: c.env.AUTH_MODE === 'cail' ? 'cail' : 'class-code', loginUrl: c.env.CAIL_LOGIN_URL || null, @@ -1970,7 +1997,7 @@ app.post('/api/webhooks/separation', async (c) => { operation=await readOperation(c.env.DB,jobId); } else await c.req.raw.body?.cancel().catch(()=>undefined); if(operation?.provider_id) { - await c.env.DB.prepare('UPDATE app_operations SET not_before=0 WHERE id=?').bind(operation.id).run(); + await c.env.DB.prepare(`UPDATE app_operations SET not_before=MAX(not_before,COALESCE((SELECT until_ms FROM provider_cooldowns WHERE provider='replicate'),0)) WHERE id=?`).bind(operation.id).run(); await reconcileSplit(c.env,operation,ingestResult); } try {c.executionCtx.waitUntil(runReliableJobs(c.env));} catch {} @@ -2048,7 +2075,7 @@ export async function runReliableJobs(env: Env) { return drainSplitQueue(env,ing // --- helpers ------------------------------------------------------------ /** Download finished stems from the provider and store them in R2. */ -async function ingestResult(env: Env, jobId: string, result: SeparationResult): Promise { +async function ingestResult(env: Env, jobId: string, result: SeparationResult, operation?: Operation): Promise { if (result.status !== 'failed' && result.status !== 'succeeded') return; const job = await env.DB.prepare('SELECT model FROM jobs WHERE id = ?') @@ -2063,9 +2090,10 @@ async function ingestResult(env: Env, jobId: string, result: SeparationResult): // so a later poll can recover if this Worker dies before releasing the claim. const lease = `${INGEST_LEASE_PREFIX}${Date.now()}:${crypto.randomUUID()}`; const claim = await env.DB.prepare( - "UPDATE jobs SET status = 'ingesting', error = ? WHERE id = ? AND status = 'processing'" + `UPDATE jobs SET status = 'ingesting', error = ? WHERE id = ? AND status = 'processing' + AND (? IS NULL OR EXISTS(SELECT 1 FROM app_operations WHERE id=? AND fence=? AND lease_owner IS ? AND lease_until>?))` ) - .bind(lease, jobId) + .bind(lease, jobId,operation?.id??null,operation?.id??null,operation?.fence??null,operation?.lease_owner??null,Date.now()) .run(); if (!claim.meta.changes) return; @@ -2113,34 +2141,32 @@ async function ingestResult(env: Env, jobId: string, result: SeparationResult): } const committed = await env.DB.prepare( - "UPDATE jobs SET status = ?, stems = ?, error = NULL WHERE id = ? AND status = 'ingesting' AND error = ?" + `UPDATE jobs SET status = ?, stems = ?, error = NULL WHERE id = ? AND status = 'ingesting' AND error = ? + AND (? IS NULL OR EXISTS(SELECT 1 FROM app_operations WHERE id=? AND fence=? AND lease_owner IS ?))` ) - .bind('done', JSON.stringify(stored), jobId, lease) + .bind('done', JSON.stringify(stored), jobId, lease,operation?.id??null,operation?.id??null,operation?.fence??null,operation?.lease_owner??null) .run(); if (!committed.meta.changes) await Promise.allSettled(stored.map(({key})=>env.AUDIO.delete(key))); } catch (error) { - const cleanup = await Promise.allSettled(stored.map(({ key }) => env.AUDIO.delete(key))); - if (cleanup.some((result) => result.status === 'rejected')) { - console.error('failed to remove partial stem files', { jobId }); - } - if (error instanceof InvalidStemAudioError) { - await env.DB.prepare( - "UPDATE jobs SET status = 'failed', error = ? WHERE id = ? AND status = 'ingesting' AND error = ?" - ) - .bind(error.message, jobId, lease) - .run(); - return; - } - if(env.AUTH_MODE==='cail' && await readOperation(env.DB,jobId)) { - await env.DB.prepare("UPDATE jobs SET status='failed',error='The finished audio could not be saved. No successful-split allowance was used.' WHERE id=? AND status='ingesting' AND error=?").bind(jobId,lease).run(); - return; - } - // Let a provider retry or the next browser poll make another attempt. - await env.DB.prepare( - "UPDATE jobs SET status = 'processing', error = NULL WHERE id = ? AND status = 'ingesting' AND error = ?" - ) - .bind(jobId, lease) - .run(); + // A failed acknowledgement does not prove a failed commit. First fence the + // old ingestion lease, then inspect durable references before any deletion. + // If either database read/write is uncertain, leave objects for recovery. + try { + const snapshot=await env.DB.prepare('SELECT status,stems,error FROM jobs WHERE id=?').bind(jobId).first<{status:string;stems:string|null;error:string|null}>(); + const referenced=(row:typeof snapshot)=>new Set(row?.stems?JSON.parse(row.stems).map((stem:{key:string})=>stem.key):[]); + if(stored.some(({key})=>referenced(snapshot).has(key)))return; + const failure=error instanceof InvalidStemAudioError?error.message:'The finished audio could not be saved. No successful-split allowance was used.'; + const terminal=error instanceof InvalidStemAudioError||env.AUTH_MODE==='cail'; + await env.DB.prepare(`UPDATE jobs SET status=?,error=? WHERE id=? AND status='ingesting' AND error=? + AND (? IS NULL OR EXISTS(SELECT 1 FROM app_operations WHERE id=? AND fence=? AND lease_owner IS ?))`) + .bind(terminal?'failed':'processing',terminal?failure:null,jobId,lease,operation?.id??null,operation?.id??null,operation?.fence??null,operation?.lease_owner??null).run(); + const durable=await env.DB.prepare('SELECT status,stems,error FROM jobs WHERE id=?').bind(jobId).first<{status:string;stems:string|null;error:string|null}>(); + if(durable?.status==='ingesting'&&durable.error===lease)return; + const keep=referenced(durable); + const cleanup=await Promise.allSettled(stored.filter(({key})=>!keep.has(key)).map(({key})=>env.AUDIO.delete(key))); + if(cleanup.some(result=>result.status==='rejected'))console.error(JSON.stringify({event:'partial_stem_cleanup_failed',jobId})); + if(terminal)return; + }catch {return;} throw error; } } @@ -2149,11 +2175,13 @@ class InvalidStemAudioError extends Error {} async function downloadStem(name: string, url: string, strict = false): Promise { let lastError: Error | null = null; + const deadline=Date.now()+60000; for (let attempt = 1; attempt <= 3; attempt += 1) { + if(strict&&Date.now()>=deadline)throw lastError??new Error('Audio download deadline expired'); let response: Response; try { - response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(20000) }); + response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(strict?Math.max(1,Math.min(20000,deadline-Date.now())):20000) }); } catch (error) { lastError = error instanceof Error @@ -2169,7 +2197,7 @@ async function downloadStem(name: string, url: string, strict = false): Promise< try { const audio = await readBoundedResponse(response, { maximumBytes: (strict?24:32) * 1024 * 1024, - timeoutMs: 30000, + timeoutMs: strict?Math.max(1,Math.min(30000,deadline-Date.now())):30000, errors: { tooLarge: () => new InvalidStemAudioError('The separator returned an oversized track'), timedOut: () => new Error('Audio download timed out'), @@ -2207,7 +2235,10 @@ async function downloadStem(name: string, url: string, strict = false): Promise< if (response.status !== 429 && response.status < 500) throw lastError; if (attempt < 3) { - await new Promise((resolve) => setTimeout(resolve, attempt * 100)); + const wait=strict&&response.status===429?Math.max(0,retryAt(response.headers.get('retry-after'))-Date.now()):attempt*100; + // A safe GET may stop retrying; it may never retry before the full floor. + if(strict&&Date.now()+wait>=deadline)throw lastError; + await new Promise((resolve) => setTimeout(resolve,wait)); } } diff --git a/src/reliability/ledger.ts b/src/reliability/ledger.ts index cd57f64..eb1f770 100644 --- a/src/reliability/ledger.ts +++ b/src/reliability/ledger.ts @@ -84,10 +84,20 @@ export async function beginAttempt(db: D1Database, operation: Operation, provide const inserted=await db.prepare(`INSERT INTO operation_attempts(id,operation_id,phase,provider,model,outcome,created_at,updated_at) SELECT ?,id,phase,?,?,'starting',?,? FROM app_operations WHERE id=? AND fence=? AND state NOT IN (${TERMINAL})`) .bind(id,provider,model ?? null,now,now,operation.id,operation.fence).run(); - if(inserted.meta.changes!==1)throw new OperationError('operation_fenced',409,'This request is being recovered.'); + if(!inserted.meta.changes)throw new OperationError('operation_fenced',409,'This request is being recovered.'); return id; } export async function finishAttempt(db: D1Database, id: string, outcome: string, detail: { externalId?: string; status?: number; code?: string; usage?: unknown } = {}) { await db.prepare('UPDATE operation_attempts SET outcome=?,external_id=COALESCE(?,external_id),http_status=?,code=?,usage_json=?,updated_at=? WHERE id=?') - .bind(outcome,detail.externalId ?? null,detail.status ?? null,detail.code ?? null,detail.usage === undefined ? null : JSON.stringify(detail.usage),Date.now(),id).run(); + .bind(outcome,detail.externalId ?? null,detail.status ?? null,safeEventCode(detail.code),numericUsage(detail.usage),Date.now(),id).run(); } + +// Gateway usage is untrusted structured data. Preserve only numeric accounting +// fields, never arbitrary provider strings or nested response content. +function numericUsage(value:unknown) { + if(!value||typeof value!=='object'||Array.isArray(value))return null; + const fields=['prompt_tokens','completion_tokens','total_tokens','input_tokens','output_tokens','cost','cost_usd']; + const result=Object.fromEntries(fields.flatMap(key=>{const n=(value as Record)[key];return typeof n==='number'&&Number.isFinite(n)&&n>=0?[[key,n]]:[];})); + return Object.keys(result).length?JSON.stringify(result):null; +} +export const safeEventCode=(code:unknown):string|null=>typeof code==='string'&&/^[a-z][a-z0-9_]{0,63}$/.test(code)?code:null; diff --git a/src/reliability/observability.ts b/src/reliability/observability.ts new file mode 100644 index 0000000..a2836b4 --- /dev/null +++ b/src/reliability/observability.ts @@ -0,0 +1,87 @@ +import type { Env } from '../env.ts'; +import type { AppPrincipal } from '../identity.ts'; +import { fingerprint, safeEventCode } from './ledger.ts'; + +export const CLIENT_USAGE_TYPES=['session_start','page_view','playback_start','playback_stop','seek','download_intent'] as const; +type ClientType=typeof CLIENT_USAGE_TYPES[number]; +export interface ClientUse { id:string; type:ClientType; jobId?:string; durationMs?:number; positionBucket?:number; } +const identifier=(v:unknown):v is string=>typeof v==='string'&&/^[a-zA-Z0-9_-]{16,80}$/.test(v); +export function parseClientUses(value:unknown):ClientUse[]|null { + if(!value||typeof value!=='object'||Array.isArray(value)||Object.keys(value).some(k=>k!=='events'))return null; + const events=(value as {events?:unknown}).events; + if(!Array.isArray(events)||events.length<1||events.length>10)return null; + for(const event of events) { + if(!event||typeof event!=='object'||Array.isArray(event)||Object.keys(event).some(k=>!['id','type','jobId','durationMs','positionBucket'].includes(k)))return null; + if(!identifier(event.id)||!CLIENT_USAGE_TYPES.includes(event.type))return null; + if(event.jobId!==undefined&&(typeof event.jobId!=='string'||!/^[a-zA-Z0-9_-]{1,80}$/.test(event.jobId)))return null; + if(['playback_start','playback_stop','seek','download_intent'].includes(event.type)&&!event.jobId)return null; + if(event.durationMs!==undefined&&(!Number.isSafeInteger(event.durationMs)||event.durationMs<0||event.durationMs>86400000))return null; + if(event.positionBucket!==undefined&&(!Number.isSafeInteger(event.positionBucket)||event.positionBucket<0||event.positionBucket>90))return null; + } + return events; +} +function actorClass(principal?:AppPrincipal) { + return !principal?'anonymous':(principal as AppPrincipal&{quotaClass?:string}).quotaClass==='guest'?'guest':'member'; +} +async function actorKey(principal:AppPrincipal|undefined,now:number) { + // Rotating daily pseudonym bounds abusive logging without recording identity. + return fingerprint(['usage-actor-v1',new Date(now).toISOString().slice(0,10),principal?.subject??'anonymous']); +} +interface Use {id:string;type:string;jobId?:string;source:'client'|'server';outcome:string;code?:string;status?:number;durationMs?:number;positionBucket?:number;} +async function persist(db:D1Database,principal:AppPrincipal|undefined,events:Use[],now=Date.now()) { + const actor=await actorKey(principal,now),kind=actorClass(principal),day=Math.floor(now/86400000)*86400000; + const statements=await Promise.all(events.map(async e=>db.prepare(`INSERT OR IGNORE INTO usage_events + (event_id,actor_key,actor_class,event_type,source,job_id,outcome,code,http_status,duration_ms,position_bucket,at) + SELECT ?,?,?,?,?,?,?,?,?,?,?,? WHERE + (?='server' OR (SELECT COUNT(*) FROM usage_events WHERE actor_key=? AND source='client' AND at>=?)=?)<25000`) + .bind(await fingerprint(['usage-v1',principal?.subject??'anonymous',e.id,e.type]),actor,kind,e.type,e.source,e.jobId??null,e.outcome, + safeEventCode(e.code),e.status??null,e.durationMs??null,e.positionBucket??null,now,e.source,actor,day,kind==='guest'?250:500,day))); + await db.batch(statements); +} +export async function recordClientUses(db:D1Database,principal:AppPrincipal,events:ClientUse[]) { + await persist(db,principal,events.map(e=>({...e,source:'client',outcome:'observed'}))); +} +/** A diagnostic failure must not alter the response or expose private errors. + * Critical lifecycle/attempt receipts instead use transactional SQL triggers. */ +export async function recordServerUse(env:Env,request:Request,response:Response,principal:AppPrincipal|undefined,started:number) { + if(env.AUTH_MODE!=='cail')return; + const url=new URL(request.url),path=url.pathname; + const split=path==='/api/jobs'&&request.method==='POST'; + const chat=/^\/api\/jobs\/[^/]+\/chat$/.test(path)&&request.method==='POST'; + const guide=/^\/api\/jobs\/[^/]+\/guide$/.test(path)&&request.method==='POST'; + const annotation=/^\/api\/jobs\/[^/]+\/annotations(?:\/[^/]+)?$/.test(path)&&!['GET','HEAD'].includes(request.method); + const download=/^\/api\/(?:files\/stems\/|shared-jobs\/)/.test(path)&&url.searchParams.has('download')&&request.method==='GET'; + if(!(split||chat||guide||annotation||download))return; + // Accepted operations already have a durable created event. Only request + // rejection is added here; retries with the same operation key coalesce. + if((split||chat||guide)&&response.status<400)return; + const type=split?'split_request':chat?'chat_request':guide?'guide_request':annotation?(request.method==='DELETE'?'annotation_delete':'annotation_save'):'download'; + const supplied=request.headers.get('Idempotency-Key'); + const key=(identifier(supplied)?supplied:crypto.randomUUID())+':'+response.status; + const job=/^\/api\/(?:jobs\/|files\/stems\/|shared-jobs\/)([a-zA-Z0-9_-]{1,80})(?:\/|$)/.exec(path)?.[1]; + try { + let code:unknown; + if(response.status>=400&&response.headers.get('content-type')?.includes('application/json')) { + const data=await response.clone().json() as {code?:unknown;error?:{code?:unknown}}; + code=data.code??(typeof data.error==='object'?data.error?.code:undefined); + } + await persist(env.DB,principal,[{id:key,type,source:'server',jobId:principal?job:undefined,outcome:response.status<400?'accepted':'failed', + code:safeEventCode(code)??(response.status>=400?'request_rejected':undefined),status:response.status,durationMs:Math.max(0,Math.min(86400000,Date.now()-started))}]); + } catch {console.warn(JSON.stringify({event:'usage_record_failed',type}));} +} +export async function usageSummary(db:D1Database,days:number) { + const since=Date.now()-days*86400000; + const [uses,operations]=await Promise.all([ + db.prepare(`SELECT event_type,source,actor_class,outcome,code,http_status,COUNT(*) AS count,ROUND(AVG(duration_ms)) AS mean_duration_ms + FROM usage_events WHERE at>=? GROUP BY event_type,source,actor_class,outcome,code,http_status ORDER BY count DESC LIMIT 250`).bind(since).all(), + db.prepare(`SELECT event_type,kind,phase,state,code,model,fallback,quota_effect,COUNT(*) AS count,ROUND(AVG(duration_ms)) AS mean_duration_ms + FROM operation_events WHERE at>=? GROUP BY event_type,kind,phase,state,code,model,fallback,quota_effect ORDER BY count DESC LIMIT 250`).bind(since).all(), + ]); + return {days,retentionDays:30,clientCapture:'best_effort',uses:uses.results??[],operations:operations.results??[]}; +} +export async function recordPageUse(env:Env,response:Response) { + if(env.AUTH_MODE!=='cail')return; + try {await persist(env.DB,undefined,[{id:crypto.randomUUID(),type:'page_response',source:'server',outcome:response.ok?'accepted':'failed',status:response.status}]);} + catch {console.warn(JSON.stringify({event:'usage_record_failed',type:'page_response'}));} +} diff --git a/src/reliability/prediction-json.ts b/src/reliability/prediction-json.ts new file mode 100644 index 0000000..8167990 --- /dev/null +++ b/src/reliability/prediction-json.ts @@ -0,0 +1,51 @@ +/** Strip a single inline audio value before JSON text allocation. JSON.parse + * still validates the complete metadata document and its exact output.audio + * location. The audio bytes are decoded in small base64 chunks, never as a + * whole UTF-16 string. Provider URLs follow the normal bounded download path. */ +export function parsePredictionJson(data:ArrayBuffer,maximumInlineBytes:number):{value:unknown;inlineAudio?:ArrayBuffer;inlineTooLarge?:boolean} { + const bytes=new Uint8Array(data),metadataLimit=256*1024; + const decoder=new TextDecoder('utf-8',{fatal:true,ignoreBOM:false}); + const white=(n:number)=>n===32||n===10||n===13||n===9; + let start=-1,end=-1,payload=-1; + for(let i=0;i=bytes.length)throw new Error('invalid_prediction_json'); + if(escaped||i-token!==6||decoder.decode(bytes.subarray(token+1,i))!=='audio')continue; + let next=i+1;while(white(bytes[next]))next++; + if(bytes[next++]!==58)continue;while(white(bytes[next]))next++; + if(bytes[next]!==34)continue; + let comma=next+1;while(comma=bytes.length)throw new Error('invalid_prediction_json'); + break; + } + if(start<0) { + if(bytes.length>metadataLimit)throw new Error('prediction_metadata_too_large'); + return {value:JSON.parse(decoder.decode(bytes))}; + } + if(bytes.length-(end-start+1)>metadataLimit)throw new Error('prediction_metadata_too_large'); + const length=end-payload; + if(length%4)throw new Error('invalid_prediction_base64'); + let padding=0;if(bytes[end-1]===61)padding++;if(bytes[end-2]===61)padding++; + const decodedLength=length/4*3-padding; + const marker='inline:'+crypto.randomUUID(); + const text=decoder.decode(bytes.subarray(0,start))+JSON.stringify(marker)+decoder.decode(bytes.subarray(end+1)); + const value=JSON.parse(text); + if(!value||typeof value!=='object'||value.output?.audio!==marker)throw new Error('invalid_prediction_audio_location'); + if(decodedLength>maximumInlineBytes)return {value,inlineTooLarge:true}; + const audio=new Uint8Array(decodedLength);let offset=0; + for(let at=payload;at(); + for(const job of ingestions.results) { + const timestamp=/^ingesting:(\d{10,16}):[a-zA-Z0-9-]+$/.exec(job.error??'')?.[1]; + if(timestamp&&Number(timestamp)+WORK_LEASE_MS<=now)await db.prepare(`UPDATE jobs SET status='processing',error=NULL + WHERE id=? AND status='ingesting' AND error=? AND EXISTS(SELECT 1 FROM app_operations WHERE job_id=? + AND state IN ('processing','reconciling') AND lease_until<=?)`).bind(job.id,job.error,job.id,now).run(); + } + // Never blindly repeat an accepted or ambiguously accepted POST. Work that // died before an attempt record is safe to enqueue; starts reconcile. await db.prepare(`UPDATE app_operations SET state=CASE diff --git a/src/reliability/retention.ts b/src/reliability/retention.ts index cec16b2..7cd5dac 100644 --- a/src/reliability/retention.ts +++ b/src/reliability/retention.ts @@ -4,6 +4,8 @@ export async function purgeOperationContent(db: D1Database, now=Date.now()) { const cutoff=now-90*86400000; await db.batch([ + db.prepare('DELETE FROM usage_events WHERE at'{}' OR result_json IS NOT NULL)`).bind(now,cutoff), diff --git a/src/reliability/splits.ts b/src/reliability/splits.ts index 73fa7c0..e1a90a3 100644 --- a/src/reliability/splits.ts +++ b/src/reliability/splits.ts @@ -18,7 +18,7 @@ interface SplitInput { sourceType: 'upload'|'youtube'|'archive'; key?: string; filename: string; youtubeUrl?: string; archiveId?: string; archiveFile?: string; model: string; auto: boolean; browserAnalysis?: unknown; } -type Complete = (env: Env, jobId: string, result: SeparationResult) => Promise; +type Complete = (env: Env, jobId: string, result: SeparationResult,operation?:Operation) => Promise; const title=(value: unknown) => typeof value==='string' ? value.replace(/[\x00-\x1f\x7f/\\]/g,' ').trim().slice(0,200) : ''; export const operationWebhook=(env: Env,id: string,phase: 'fetch'|'split') => `${env.PUBLIC_BASE_URL}/api/webhooks/separation?job=${id}&phase=${phase}&token=${env.WEBHOOK_SECRET}`; @@ -81,7 +81,9 @@ async function imported(env: Env,op: Operation,input: SplitInput,audio: {data:Ar await env.AUDIO.put(key,audio.data,{httpMetadata:{contentType:audio.fileName?archiveContentType(audio.fileName):'audio/mp4'}}); const current=await readOperation(env.DB,op.id); if (!current||current.fence!==op.fence||current.cancel_requested||['failed','cancelled'].includes(current.state)) { - await env.AUDIO.delete(key);if(current)await fail(env,current,'cancelled');return; + await env.AUDIO.delete(key); + if(current?.fence===op.fence&¤t.cancel_requested)await fail(env,op,'cancelled'); + return; } input.key=key;input.filename=title(audio.title)||input.filename; await env.DB.batch([ @@ -105,7 +107,7 @@ async function storeArchive(env: Env,op: Operation,input: SplitInput) { const audio=await fetchArchiveAudio(input.archiveId!,input.archiveFile,env,{maximumBytes:CAIL_IMPORT_BYTES}); await imported(env,op,input,audio); const current=await readOperation(env.DB,op.id); - if (current?.phase==='split'&&input.key) await env.DB.prepare(`INSERT INTO import_cache(scope,source,operation_id,object_key,metadata_json,expires_at) + if (current?.fence===op.fence&¤t.phase==='split'&&input.key) await env.DB.prepare(`INSERT INTO import_cache(scope,source,operation_id,object_key,metadata_json,expires_at) VALUES(?,?,?,?,?,?) ON CONFLICT(scope,source) DO UPDATE SET operation_id=excluded.operation_id,object_key=excluded.object_key,metadata_json=excluded.metadata_json,expires_at=excluded.expires_at`) .bind(scope,source,op.id,input.key,JSON.stringify({title:audio.title,fileName:audio.fileName,attribution:audio.attribution}),op.created_at+24*60*60*1000).run(); } @@ -128,7 +130,7 @@ async function start(env: Env,op: Operation) { const options=getSeparationOptions(env.SEPARATION_BACKEND); let expectedSourceBytes: number | undefined; if (input.sourceType==='upload'&&capability.mode==='authoritative') { - const snapshot=await prepareAuthoritativeAutoSource(env,{jobId:op.id,sourceKey});sourceKey=snapshot.snapshotKey;expectedSourceBytes=snapshot.bytes; + const snapshot=await prepareAuthoritativeAutoSource(env,{jobId:op.id,sourceKey,allowExisting:true,isCurrent:async()=>{const current=await readOperation(env.DB,op.id);return current?.fence===op.fence&¤t.lease_owner===op.lease_owner&¤t.state==='running'&&!current.cancel_requested;}});sourceKey=snapshot.snapshotKey;expectedSourceBytes=snapshot.bytes; } const stored=await env.DB.prepare('SELECT source_hash FROM jobs WHERE id=?').bind(op.job_id).first<{source_hash:string|null}>(); const head=stored?.source_hash?await env.AUDIO.head(sourceKey):null; @@ -138,14 +140,17 @@ async function start(env: Env,op: Operation) { ...(expectedSourceBytes!==undefined?{expectedSourceBytes}:{}), provider:configuredAudioAnalysisProvider(env),timeoutMs:audioAnalysisTimeoutMs(env),instrumentDiscovery:processingFeatureFlags(env).instrumentDiscovery}); model=resolution.decision.resolvedCoreModel; - await env.DB.prepare('UPDATE jobs SET model=?,routing_request=?,analysis=?,source_key=? WHERE id=?').bind(model,AUTO_ROUTING_REQUEST,JSON.stringify(resolution.decision),sourceKey,op.job_id).run(); + const changed=await env.DB.prepare(`UPDATE jobs SET model=?,routing_request=?,analysis=?,source_key=? WHERE id=? + AND EXISTS(SELECT 1 FROM app_operations WHERE id=? AND fence=? AND lease_owner IS ? AND cancel_requested=0 AND state='running')`).bind(model,AUTO_ROUTING_REQUEST,JSON.stringify(resolution.decision),sourceKey,op.job_id,op.id,op.fence,op.lease_owner).run(); + if(!changed.meta.changes)return; } sourceUrl=await presignDownload(env,sourceKey); } - // This write precedes every paid POST. A crash from here reconciles, never retries. - if (!await transition(env.DB,op,'starting')) return; const runner=getReplicateRunner(model); const version=op.phase==='fetch'?env.REPLICATE_YT_MODEL_VERSION:runner?replicateVersion(env,runner):undefined; + if(!version||!env.REPLICATE_API_TOKEN) {await fail(env,op,'provider_configuration','Audio separation is temporarily unavailable. No successful-split allowance was used.');return;} + // This write precedes every paid POST. A crash from here reconciles, never retries. + if (!await transition(env.DB,op,'starting')) return; attempt=await beginAttempt(env.DB,op,'replicate',version); const providerId=op.phase==='fetch' ? await startYouTubeImport(input.youtubeUrl!,env,operationWebhook(env,op.id,op.phase as 'fetch'|'split')) @@ -153,8 +158,8 @@ async function start(env: Env,op: Operation) { await finishAttempt(env.DB,attempt,'accepted',{externalId:providerId}); if (await transition(env.DB,op,'processing',{providerId})) { await env.DB.prepare('UPDATE app_operations SET lease_owner=NULL,lease_until=0 WHERE id=? AND fence=?').bind(op.id,op.fence).run(); - await env.DB.prepare('UPDATE jobs SET status=?,external_id=? WHERE id=? AND status<>\'failed\'') - .bind(op.phase==='fetch'?'importing':'processing',op.phase==='fetch'?null:providerId,op.job_id).run(); + await env.DB.prepare(`UPDATE jobs SET status=?,external_id=? WHERE id=? AND status NOT IN ('failed','done') AND EXISTS(SELECT 1 FROM app_operations WHERE id=? AND fence=? AND state='processing')`) + .bind(op.phase==='fetch'?'importing':'processing',op.phase==='fetch'?null:providerId,op.job_id,op.id,op.fence).run(); } } catch(error) { if (attempt) await finishAttempt(env.DB,attempt,error instanceof UpstreamError?error.outcome:'uncertain',{ @@ -170,7 +175,7 @@ async function start(env: Env,op: Operation) { await fail(env,op,error.code);return; } await transition(env.DB,op,'reconciling',{code:'start_uncertain'}); - await env.DB.prepare("UPDATE jobs SET status='reconciling' WHERE id=? AND status<>'failed'").bind(op.job_id).run(); + await env.DB.prepare("UPDATE jobs SET status='reconciling' WHERE id=? AND status NOT IN ('failed','done') AND EXISTS(SELECT 1 FROM app_operations WHERE id=? AND fence=? AND state='reconciling')").bind(op.job_id,op.id,op.fence).run(); } } @@ -187,9 +192,11 @@ export async function reconcileSplit(env: Env,op: Operation,complete: Complete) if(audio)await imported(env,claimed,JSON.parse(claimed.request_json),audio); } else { const result=await getBackend(env).fetchStatus(claimed.provider_id!); + const current=await readOperation(env.DB,claimed.id); + if(current?.fence!==claimed.fence||current.lease_owner!==claimed.lease_owner)return; if(result.status!=='processing') { if(claimed.cancel_requested)await fail(env,claimed,'cancelled','Split cancelled. No successful-split allowance was used.'); - else await complete(env,claimed.job_id!,result); + else await complete(env,claimed.job_id!,result,claimed); } } } catch(error) { @@ -219,7 +226,9 @@ export async function recoverCallback(env: Env,operation: Operation,externalId: AND provider='replicate' AND outcome IN ('starting','uncertain') ORDER BY created_at DESC LIMIT 1`) .bind(operation.id,operation.phase).first<{id:string;model:string|null}>(); if(!attempt?.model)return; - if(!await getBackend(env).confirmStart?.(externalId,operationWebhook(env,operation.id,operation.phase as 'fetch'|'split'),attempt.model))return; + if(await cooldown(env.DB,'replicate')>Date.now())return; + try {if(!await getBackend(env).confirmStart?.(externalId,operationWebhook(env,operation.id,operation.phase as 'fetch'|'split'),attempt.model))return;} + catch(error) {if(error instanceof UpstreamError&&error.notBefore) {await setCooldown(env.DB,'replicate',error.notBefore);await env.DB.prepare('UPDATE app_operations SET not_before=MAX(not_before,?) WHERE id=?').bind(error.notBefore,operation.id).run();}return;} await finishAttempt(env.DB,attempt.id,'accepted',{externalId}); const accepted=await env.DB.prepare(`UPDATE app_operations SET provider_id=?,state='processing',lease_until=0,lease_owner=NULL,updated_at=? WHERE id=? AND fence=? AND phase=? AND provider_id IS NULL AND state IN ('starting','reconciling')`) diff --git a/src/separation/replicate.ts b/src/separation/replicate.ts index 460ab75..b368763 100644 --- a/src/separation/replicate.ts +++ b/src/separation/replicate.ts @@ -92,15 +92,17 @@ export function replicateBackend(env: Env): SeparationBackend { const res = await fetch(`${API}/predictions/${externalId}`, { headers, redirect: 'manual', signal: AbortSignal.timeout(20000) }); if (!res.ok) { await res.body?.cancel().catch(() => undefined); - throw new Error(`The separator status is unavailable (${res.status}).`); + throw new UpstreamError('provider_poll_unavailable','uncertain',res.status,res.status===429?retryAt(res.headers.get('retry-after')):0); } - return backend.parseResult(await predictionJson(res)); + const prediction=await predictionJson(res); + if(prediction.id!==externalId)throw new UpstreamError('provider_poll_identity','uncertain'); + return backend.parseResult(prediction); }, async confirmStart(externalId: string, webhookUrl: string, version: string): Promise { if (!/^[a-zA-Z0-9_-]{1,128}$/.test(externalId)) return false; const res=await fetch(`${API}/predictions/${externalId}`,{headers,redirect:'manual',signal:AbortSignal.timeout(20000)}); - if (!res.ok) { await res.body?.cancel(); return false; } + if (!res.ok) { await res.body?.cancel();if(res.status===429)throw new UpstreamError('provider_poll_unavailable','uncertain',429,retryAt(res.headers.get('retry-after'))); return false; } const prediction=await predictionJson(res); return prediction.id===externalId && prediction.webhook===webhookUrl && prediction.version===version; }, diff --git a/src/youtube.ts b/src/youtube.ts index 03ce6db..e7eeafb 100644 --- a/src/youtube.ts +++ b/src/youtube.ts @@ -1,3 +1,4 @@ +import { parsePredictionJson } from './reliability/prediction-json.ts'; // YouTube audio fetch behind the fetchYouTubeAudio() seam. // // Strategy: use the configured provider order, with a Replicate-hosted yt-dlp @@ -11,7 +12,7 @@ const MAX_DURATION_SECONDS = 15 * 60; // cost/scope guard for class use const MAX_AUDIO_BYTES = 100 * 1024 * 1024; // Cloudflare's 128 MiB isolate must also hold JSON/base64 and a split decoder. export const CAIL_IMPORT_BYTES=12*1024*1024; -export const CAIL_INLINE_BYTES=4*1024*1024; +export const CAIL_INLINE_BYTES=12*1024*1024; export const CAIL_PREDICTION_BYTES=Math.ceil(CAIL_INLINE_BYTES*4/3)+256*1024; const MIN_AUDIO_BYTES = 1024; const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; @@ -428,6 +429,8 @@ interface YtPrediction { status: 'starting' | 'processing' | 'succeeded' | 'failed' | 'canceled'; output?: { audio: string; title: string; duration: number }; error?: string; + inlineAudio?: ArrayBuffer; + inlineTooLarge?: boolean; } async function replicateFetch( @@ -454,7 +457,8 @@ async function readPrediction( response: Response, timeoutMs: number, expectedId?: string, - maximumBytes=MAX_PREDICTION_BYTES + maximumBytes=MAX_PREDICTION_BYTES, + boundedInline=false ): Promise { if (responseMediaType(response) !== 'application/json') { await response.body?.cancel().catch(() => undefined); @@ -470,9 +474,10 @@ async function readPrediction( }, }); - let value: unknown; + let value: unknown,inlineAudio:ArrayBuffer|undefined,inlineTooLarge:boolean|undefined; try { - value = JSON.parse( + if(boundedInline)({value,inlineAudio,inlineTooLarge}=parsePredictionJson(bytes,CAIL_INLINE_BYTES)); + else value = JSON.parse( new TextDecoder('utf-8', { fatal: true, ignoreBOM: false }).decode(bytes) ); } catch { @@ -522,7 +527,7 @@ async function readPrediction( return { id: candidate.id, status: candidate.status as YtPrediction['status'], - output, + output,inlineAudio,inlineTooLarge, error: typeof candidate.error === 'string' ? candidate.error.slice(0, 300) : undefined, }; } @@ -820,7 +825,7 @@ export async function startYouTubeImport(url: string, env: Env, webhook: string) throw new UpstreamError(res.status===429?'provider_capacity':res.status===402?'provider_credit':'import_start_rejected',res.status>=500?'uncertain':'rejected',res.status, res.status===429?retryAt(res.headers.get('retry-after')):0); } - try { return (await readPrediction(res,15000,undefined,CAIL_PREDICTION_BYTES)).id; } + try { return (await readPrediction(res,15000,undefined,CAIL_PREDICTION_BYTES,true)).id; } catch { throw new UpstreamError('import_start_response_unreadable','uncertain'); } } @@ -833,11 +838,12 @@ export async function pollYouTubeImport(id: string, env: Env): Promise { ); assert.deepEqual(deleted, ['auto-inputs/v1/auto_job_rollback']); }); + +test('durable Auto recovery reuses its own verified snapshot and stale cleanup cannot delete it',async()=>{ + await withBucket(async(bucket,env)=>{ + // The lightweight Node fixture shim omits R2 custom metadata. Preserve the + // actual put metadata in this fixture to exercise the native R2 contract. + const metadata=new Map>(),nativePut=bucket.put.bind(bucket),nativeHead=bucket.head.bind(bucket); + bucket.put=async(key,body,options)=>{if(options?.customMetadata)metadata.set(key,options.customMetadata);return nativePut(key,body,options);}; + bucket.head=async(key)=>{const object=await nativeHead(key);if(object)Object.assign(object,{customMetadata:metadata.get(key)});return object;}; + await bucket.put(SOURCE_KEY,ORIGINAL); + const initial=await prepareAuthoritativeAutoSource(env,{jobId:'durable_auto',sourceKey:SOURCE_KEY}); + const replay=await prepareAuthoritativeAutoSource(env,{jobId:'durable_auto',sourceKey:SOURCE_KEY,allowExisting:true,isCurrent:async()=>true}); + assert.deepEqual(replay,initial); + await assert.rejects(prepareAuthoritativeAutoSource(env,{jobId:'durable_auto',sourceKey:SOURCE_KEY}),/already exists/); + const put=bucket.put.bind(bucket);let current=true; + bucket.put=async(...args)=>{await put(...args);current=false;throw new Error('old Worker lost acknowledgement');}; + await assert.rejects(prepareAuthoritativeAutoSource(env,{jobId:'stale_auto',sourceKey:SOURCE_KEY,isCurrent:async()=>current})); + assert.deepEqual(await storedBytes(bucket,'auto-inputs/v1/stale_auto'),ORIGINAL); + }); +}); diff --git a/tests/e2e/local-hosting.spec.mjs b/tests/e2e/local-hosting.spec.mjs index ac42f93..da2302d 100644 --- a/tests/e2e/local-hosting.spec.mjs +++ b/tests/e2e/local-hosting.spec.mjs @@ -595,8 +595,10 @@ test('renames Demucs no_vocals to instrumental for the two-track split', async ( const storedKeysResponse = await e2eFetch(server, '/__e2e/audio'); const storedKeys = (await storedKeysResponse.json()).keys; // Stored under the contract name, so /api/files and the mixer agree. - expect(storedKeys).toContain(`stems/${created.id}/instrumental.mp3`); - expect(storedKeys).not.toContain(`stems/${created.id}/no_vocals.mp3`); + const instrumental=result.stems.find(stem=>stem.name==='instrumental'); + expect(instrumental.url).toMatch(new RegExp(`^/api/files/stems/${created.id}/[a-f0-9-]+/instrumental\\.mp3$`)); + expect(storedKeys).toContain(instrumental.url.replace('/api/files/','')); + expect(storedKeys.some(key=>key.startsWith(`stems/${created.id}/`)&&key.endsWith('/no_vocals.mp3'))).toBe(false); expect(browserErrors).toEqual([]); }); @@ -941,7 +943,9 @@ test('completes a six-track split whose guitar and piano tracks are near-silent' // The quiet tracks were stored verbatim rather than dropped or substituted. for (const name of quietNames) { - const stored = await server.fetch(`/api/files/stems/${created.id}/${name}.mp3`); + const track=result.stems.find(stem=>stem.name===name); + expect(track.url).toMatch(new RegExp(`^/api/files/stems/${created.id}/[a-f0-9-]+/${name}\\.mp3$`)); + const stored = await server.fetch(track.url); expect(stored.status).toBe(200); expect(Buffer.from(await stored.arrayBuffer()).equals(quietAudio)).toBe(true); } @@ -1045,8 +1049,11 @@ test('imports a YouTube link and renames no_vocals for the two-track split', asy const storedKeysResponse = await e2eFetch(server, '/__e2e/audio'); const { keys } = await storedKeysResponse.json(); - expect(keys).toContain(`stems/${created.id}/instrumental.mp3`); - expect(keys).not.toContain(`stems/${created.id}/no_vocals.mp3`); + const result=await (await server.fetch(`/api/jobs/${created.id}`)).json(); + const instrumental=result.stems.find(stem=>stem.name==='instrumental'); + expect(instrumental.url).toMatch(new RegExp(`^/api/files/stems/${created.id}/[a-f0-9-]+/instrumental\\.mp3$`)); + expect(keys).toContain(instrumental.url.replace('/api/files/','')); + expect(keys.some(key=>key.startsWith(`stems/${created.id}/`)&&key.endsWith('/no_vocals.mp3'))).toBe(false); expect(keys.some((key) => /^uploads\/[0-9a-f-]+\/source\.m4a$/.test(key))).toBe(true); expect(browserErrors).toEqual([]); }); diff --git a/tests/youtube.test.mts b/tests/youtube.test.mts index deed6fd..2e09e9c 100644 --- a/tests/youtube.test.mts +++ b/tests/youtube.test.mts @@ -1,3 +1,4 @@ +import { parsePredictionJson } from '../src/reliability/prediction-json.ts'; import assert from 'node:assert/strict'; import test from 'node:test'; @@ -631,10 +632,10 @@ test('durable importer bounds prediction and audio copies before reading oversiz globalThis.fetch=async()=>new Response('{}',{headers:{'Content-Type':'application/json','Content-Length':String(CAIL_PREDICTION_BYTES+1)}}); await assert.rejects(()=>startYouTubeImport('https://youtu.be/jNQXAC9IVRw',TEST_ENV as never,'https://app.test/callback'),(error:any)=>error.code==='import_start_response_unreadable'&&error.outcome==='uncertain'); // The inline limit is independently enforced after a bounded JSON parse. - for(const size of [2048,CAIL_INLINE_BYTES+1]) { + for(const size of [2048,6*1024*1024,CAIL_INLINE_BYTES,CAIL_INLINE_BYTES+1]) { const audio=Buffer.from(makeM4a(size)).toString('base64'); globalThis.fetch=async()=>Response.json({id:'bounded',status:'succeeded',output:{audio:`data:audio/mp4;base64,${audio}`,title:'Fixture',duration:1}}); - if(size===2048)assert.equal((await pollYouTubeImport('bounded',TEST_ENV as never))?.data.byteLength,size); + if(size<=CAIL_INLINE_BYTES)assert.equal((await pollYouTubeImport('bounded',TEST_ENV as never))?.data.byteLength,size); else await assert.rejects(()=>pollYouTubeImport('bounded',TEST_ENV as never),(error:any)=>error.code==='audio_too_large'); } } finally {globalThis.fetch=saved;} @@ -655,3 +656,17 @@ test('durable extractor errors distinguish format, configuration and explicit so } }finally{globalThis.fetch=saved;} }); + +test('bounded inline parsing validates metadata location, escapes, base64 and duplicate audio fields',()=>{ + const parse=(text:string)=>parsePredictionJson(new TextEncoder().encode(text).buffer,32); + const payload='data:audio/mp4;base64,AQIDBA=='; + assert.deepEqual(new Uint8Array(parse(JSON.stringify({output:{audio:payload,title:'a "title"',duration:1}})).inlineAudio!),new Uint8Array([1,2,3,4])); + for(const text of [ + JSON.stringify({error:{audio:payload},output:{audio:'https://safe.test'}}), + '{"output":{"audio":"'+payload+'","audio":"https://safe.test"}}', + '{"output":{"audio":"'+payload+'"},"invalid":}', + JSON.stringify({output:{audio:'data:audio/mp4;base64,AAAA=A=='}}), + JSON.stringify({output:{audio:payload},logs:'x'.repeat(256*1024)}), + ])assert.throws(()=>parse(text)); + assert.deepEqual(parse('{"output":{"audio":"https://safe.test"}}').value,{output:{audio:'https://safe.test'}}); +}); From 8c8d53b6bcceec58e30b3f85cfb5df971bc22a3c Mon Sep 17 00:00:00 2001 From: zmuhls Date: Tue, 6 Oct 2026 15:43:59 -0400 Subject: [PATCH 08/25] Enforce shared recovery cooldown and verify candidate dependency aliases --- cloudflare/account-rack.spec.mjs | 2 +- cloudflare/gateway.ts | 2 +- cloudflare/reliability.test.ts | 17 +++++++++++++++++ .../bundle-dependencies.json | 10 ++++++++++ .../reliability-20261006/verification.md | 11 ++++++++--- public/classroom.js | 1 + public/teacher.js | 2 +- src/reliability/queue.ts | 3 ++- src/reliability/splits.ts | 5 +++-- tests/e2e/local-hosting.spec.mjs | 13 +++++++------ 10 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 docs/evidence/reliability-20261006/bundle-dependencies.json diff --git a/cloudflare/account-rack.spec.mjs b/cloudflare/account-rack.spec.mjs index 36c9810..75c9465 100644 --- a/cloudflare/account-rack.spec.mjs +++ b/cloudflare/account-rack.spec.mjs @@ -70,7 +70,7 @@ test('account rack recovers on a new browser; isolates people; handles paging, i }); await page.locator('#yt-url').fill('https://www.youtube.com/watch?v=fixture1234'); await page.locator('#yt-form button').click(); - await expect(page.locator('#upload-message')).toContainText("You've used today's runs."); + await expect(page.locator('#upload-message')).toContainText("Today's 15 places are completed or in progress."); await expect(page.locator('#upload-message')).not.toContainText('Saved to your account'); await page.unroute('**/api/jobs'); await page.getByRole('link', { name: 'My account', exact: true }).click(); diff --git a/cloudflare/gateway.ts b/cloudflare/gateway.ts index 83aa692..24211f2 100644 --- a/cloudflare/gateway.ts +++ b/cloudflare/gateway.ts @@ -130,7 +130,7 @@ export function gatewayForRequest(binding: Fetcher | undefined, token: string | if (attemptId) await finishAttempt(env.DB,attemptId,'succeeded',{usage}); return { content: content.trim(), model, toolCalls: [...calls.entries()].sort(([a], [b]) => a - b).map(([, call]) => call), finishReason }; } catch (error) { - if (attemptId) await finishAttempt(env.DB,attemptId,outputStarted?'partial':accepted||!(error instanceof CailError)?'uncertain':'rejected',{code:error instanceof CailError?error.code:'gateway_failure',usage}); + if (attemptId) await finishAttempt(env.DB,attemptId,outputStarted?'partial':accepted||!(error instanceof CailError)?'uncertain':'rejected',{code:error instanceof CailError?error.code:'gateway_failure',status:error instanceof CailError?error.status:undefined,usage}); if (!signal.aborted && !accepted && !outputStarted && modelIndex===0 && models.length===2 && eligibleGatewayFallback(error)) { if (operation) await setCooldown(env.DB,`gateway:${model}`,retryAt(error instanceof CailError ? String(error.extras.retry_after ?? error.extras.retry_after_seconds ?? '30') : '30')); continue; diff --git a/cloudflare/reliability.test.ts b/cloudflare/reliability.test.ts index 25351f7..3c21318 100644 --- a/cloudflare/reliability.test.ts +++ b/cloudflare/reliability.test.ts @@ -208,3 +208,20 @@ test('lost final database acknowledgement preserves committed stem objects and c mock.mock.restore();dbMock.mock.restore(); } }); + +test('a shared provider cooldown fences reconciliation of other operations in both phases',async t=>{ + const {db,env,split}=await setup(t),fetch=await split('shared-floor-fetch-01','fetch'),separate=await split('shared-floor-split-01'); + for(const op of [fetch,separate])await db.prepare("UPDATE app_operations SET state='processing',provider_id='known-provider-id',not_before=0 WHERE id=?").bind(op.id).run(); + await db.prepare("INSERT INTO provider_cooldowns(provider,until_ms) VALUES('replicate',?)").bind(Date.now()+120000).run(); + let calls=0;t.mock.method(globalThis,'fetch',async()=>{calls++;throw new Error('shared floor must prevent any network');}); + await runReliableJobs(env);assert.equal(calls,0); + for(const op of [fetch,separate])assert.equal((await readOperation(db,op.id))!.fence,0); +}); + +test('a crash after chat reservation but before its running transition releases without usable output',async t=>{ + const {db}=await setup(t);const now=Date.now(); + const {operation}=await reserveOperation(db,{subject,courseId:null,kind:'chat',phase:'chat',key:'queued-chat-crash-01',fingerprint:'input',now}); + await recoverExpired(db,now+89999);assert.equal((await readOperation(db,operation.id))!.state,'queued'); + await recoverExpired(db,now+90001);assert.equal((await readOperation(db,operation.id))!.state,'failed'); + assert.equal((await operationAllowance(db,subject,'chat')).remaining,50); +}); diff --git a/docs/evidence/reliability-20261006/bundle-dependencies.json b/docs/evidence/reliability-20261006/bundle-dependencies.json new file mode 100644 index 0000000..408ffdb --- /dev/null +++ b/docs/evidence/reliability-20261006/bundle-dependencies.json @@ -0,0 +1,10 @@ +{ + "checkedAt": "2026-10-06", + "command": "wrangler deploy --config wrangler.jsonc --dry-run --metafile", + "honoInputCount": 27, + "allHonoInputsUsePatchedAlias": true, + "legacyRootHonoInputCount": 0, + "bundledHonoVersion": "4.13.11", + "metafileSha256": "a74175fa5fdfc5a573ba0a102d20b896412f22186df9b269b93fef2e53e0fecb", + "rootAnalysisDependencyInputsChanged": false +} diff --git a/docs/evidence/reliability-20261006/verification.md b/docs/evidence/reliability-20261006/verification.md index 5286fa6..278c48a 100644 --- a/docs/evidence/reliability-20261006/verification.md +++ b/docs/evidence/reliability-20261006/verification.md @@ -28,12 +28,17 @@ The follow-up implementation tree passed: | Chrome usage events, one retry, deduplication and account-clear case | 1/1 | | Candidate Wrangler dry-run | Pass; no deploy | | Adapter production dependency audit | No vulnerabilities in 10 packages | -| Root production dependency audit | **Blocked: four moderate Hono advisories** | +| Root legacy dependency audit | Four moderate Hono advisories; absent from candidate bundle | The root dependency inputs remain frozen. Audit reported `hono@4.12.34` with GHSA-gqvv-2mrq-wpjv, GHSA-g6gw-c38x-mqfc, GHSA-crvj-82cr-hjcx and -GHSA-hxh3-vqpv-xpqv. Shared Worker imports still use the root application package; -the adapter-only audit does not establish a clean deployed dependency tree. +GHSA-hxh3-vqpv-xpqv. The candidate and test Wrangler configurations explicitly alias both `hono` +and `hono/factory` to adapter `hono-cloudflare@4.13.11`. The dry-run build +metafile confirms all 27 Hono inputs use that patched alias and zero use the +legacy root package. The [bundle receipt](bundle-dependencies.json) records +that check. The four root advisories remain a legacy Node/analyzer dependency +finding; they are not a demonstrated candidate runtime exposure. Frozen root +analysis-image inputs were not changed. Five direct independent-review regressions now cover abandoned-ingestion cron recovery, stale import fencing, no-network missing-pin failure, complete diff --git a/public/classroom.js b/public/classroom.js index 4baac6c..a7c0e50 100644 --- a/public/classroom.js +++ b/public/classroom.js @@ -111,6 +111,7 @@ async function openFolder(id, scope = context()) { async function selectCourse() { generation++; ready = false; course = courses.find(entry => entry.classId === el('classroom-course').value); clearCourseView(); if (!course) return null; + window.StemUsage?.setCourse(course.classId); const scope = context(); el('classroom-refresh').disabled = true; status('Loading…'); try { await Promise.all([folders(scope), course.owner ? roster(false, scope) : Promise.resolve()]); diff --git a/public/teacher.js b/public/teacher.js index 730e23b..65176a3 100644 --- a/public/teacher.js +++ b/public/teacher.js @@ -451,7 +451,7 @@ historyMoreBtn.addEventListener('click', async () => { for(const course of courses)select.add(new Option(`${course.className} · ${course.term} · ${course.section}`,course.classId)); const requested=new URLSearchParams(location.search).get('course');selectedCourse=courses.some(c=>c.classId===requested)?requested:courses[0].classId;select.value=selectedCourse; document.getElementById('teacher-courses').hidden=false; - const setRoster=()=>{document.getElementById('teacher-roster').href=`/classroom.html?course=${encodeURIComponent(selectedCourse)}`;};setRoster(); + const setRoster=()=>{window.StemUsage?.setCourse(selectedCourse);document.getElementById('teacher-roster').href=`/classroom.html?course=${encodeURIComponent(selectedCourse)}`;};setRoster(); select.addEventListener('change',async()=>{ if(amendment.value.trim()!==loadedAmendment&&!confirm('Discard unsaved course instructions?')){select.value=selectedCourse;return;} selectedCourse=select.value;setRoster();try{await loadPrompt();}catch{/* The current load owns its error state. */} diff --git a/src/reliability/queue.ts b/src/reliability/queue.ts index 6132168..d89adbd 100644 --- a/src/reliability/queue.ts +++ b/src/reliability/queue.ts @@ -71,7 +71,8 @@ export async function recoverExpired(db: D1Database, now=Date.now()) { // Both outcomes fence the old key/runner and never regenerate its tools. await db.prepare(`UPDATE app_operations SET state=CASE WHEN result_json IS NULL THEN 'failed' ELSE 'partial' END, error_code=CASE WHEN result_json IS NULL THEN 'no_usable_response' ELSE 'delivery_uncertain' END,updated_at=? - WHERE kind IN ('chat','guide') AND state IN ('running','starting','reconciling') AND lease_until(); + WHERE id=? AND fence=? AND lease_until<=? AND not_before<=? AND state IN ('processing','reconciling') + AND COALESCE((SELECT until_ms FROM provider_cooldowns WHERE provider='replicate'),0)<=? RETURNING *`) + .bind(owner,now+WORK_LEASE_MS,now,op.id,op.fence,now,now,now).first(); if(!claimed)return; try { if (claimed.phase==='fetch') { diff --git a/tests/e2e/local-hosting.spec.mjs b/tests/e2e/local-hosting.spec.mjs index da2302d..15e9e2a 100644 --- a/tests/e2e/local-hosting.spec.mjs +++ b/tests/e2e/local-hosting.spec.mjs @@ -429,7 +429,11 @@ test('uploads and processes a real WAV through local R2 in a browser', async ({ expect(await page.evaluate((id) => localStorage.getItem(`coachChat:${id}`), jobId)).toBeNull(); await expect(page.locator('.coach-guide-text')).toBeVisible(); - const storedStemResponse = await server.fetch(`/api/files/stems/${jobId}/vocals.mp3`); + const savedJob=await(await server.fetch(`/api/jobs/${jobId}`)).json(); + expect(savedJob.stems.map(stem=>stem.name)).toEqual(['vocals','drums','bass','other']); + for(const stem of savedJob.stems)expect(stem.url).toMatch(new RegExp(`^/api/files/stems/${jobId}/[a-f0-9-]+/${stem.name}\\.mp3$`)); + const savedVocals=savedJob.stems.find(stem=>stem.name==='vocals'); + const storedStemResponse = await server.fetch(savedVocals.url); expect(storedStemResponse.status).toBe(200); expect(storedStemResponse.headers.get('content-length')).toBe( String(stemAudio.get('vocals').length) @@ -440,17 +444,14 @@ test('uploads and processes a real WAV through local R2 in a browser', async ({ expect(storedKeysResponse.status).toBe(200); const { keys: storedKeys } = await storedKeysResponse.json(); expect(storedKeys).toEqual([ - `stems/${jobId}/bass.mp3`, - `stems/${jobId}/drums.mp3`, - `stems/${jobId}/other.mp3`, - `stems/${jobId}/vocals.mp3`, + ...savedJob.stems.map(stem=>stem.url.replace('/api/files/','')).sort(), expect.stringMatching(/^uploads\/[0-9a-f-]+\/source\.wav$/), ]); expect(browserErrors).toEqual([]); await page.screenshot({ path: testInfo.outputPath('local-hosting-ready.png'), fullPage: false }); - await page.route(`**/api/files/stems/${jobId}/vocals.mp3`, (route) => + await page.route(`**${savedVocals.url}`, (route) => route.fulfill({ status: 404, body: 'Not found' }) ); await page.reload(); From af1fe9576a2aeb76cca2ffae07c4f8c752832c10 Mon Sep 17 00:00:00 2001 From: zmuhls Date: Tue, 6 Oct 2026 15:50:51 -0400 Subject: [PATCH 09/25] Fence final assistant delivery and course effects against recovered operations --- cloudflare/classroom.test.ts | 29 +++++++++++++++++++++++++++++ cloudflare/reliability.test.ts | 6 +++++- docs/reliability-operations.md | 7 +++++++ src/classroom/conversations.ts | 22 ++++++++++++++-------- src/index.ts | 10 +++++----- src/reliability/assistant.ts | 28 +++++++++++++++++++++++----- 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/cloudflare/classroom.test.ts b/cloudflare/classroom.test.ts index fa068d0..4acfed9 100644 --- a/cloudflare/classroom.test.ts +++ b/cloudflare/classroom.test.ts @@ -14,6 +14,8 @@ import { hashSystemPromptFingerprint } from '../src/assistant/prompt.ts'; import { authorizeStoredCourseWork } from '../src/classroom/access.ts'; import { saveCoursePrompt } from '../src/classroom/prompts.ts'; import { purgeExpiredListeningConversations } from './retention.ts'; +import { assistantReceipt, reserveAssistant } from '../src/reliability/assistant.ts'; +import { recoverExpired } from '../src/reliability/queue.ts'; const B='another-reviewed-course'; const ids={alice:`cail-${'1'.repeat(32)}`,bob:`cail-${'2'.repeat(32)}`,ownerA:`cail-${'3'.repeat(32)}`,ownerB:`cail-${'4'.repeat(32)}`,outsider:`cail-${'5'.repeat(32)}`,admin:`cail-${'6'.repeat(32)}`}; type Person=keyof typeof ids; @@ -112,6 +114,33 @@ test('reset during an in-flight chat fences late deltas, receipts and notes with assert.equal(operations[0].state,'failed');assert.doesNotMatch(JSON.stringify(operations),/Late private|Private input/); assert.equal((await request('/api/jobs/reset-stream/chat','alice','POST',body)).status,409);assert.equal(calls,1); }); +test('the operation deadline fences course effects before the longer conversation lease expires',async()=>{ + for(const recovered of [false,true]) { + const {env,job,db,principal}=await setup();await job('deadline');const p=await principal(); + const {operation}=await reserveAssistant(env,p.subject,A,'chat','operation-fenced-turn-01','deadline',[]); + const receipt=assistantReceipt(env,operation),turn=await beginCourseTurn(env,'deadline',p,'Listen','operation-fenced-turn-01',0); + receipt.bindCourseClaim(turn.claimId);await receipt.effect(); + if(recovered)await recoverExpired(db as unknown as D1Database,Date.now()+90001); + else await db.prepare('UPDATE app_operations SET lease_until=0 WHERE id=?').bind(operation.id).run(); + const result={reply:'Late private result',toolCalls:[{name:'add_note' as const,args:{seconds:1,text:'Late note'}}],finishReason:'stop'}; + assert.equal(await finishCourseTurn(env,'deadline',p.subject,turn.turnId,turn.claimId,result,false,p.displayName,operation.id),null); + assert.equal((await db.prepare('SELECT COUNT(*) AS n FROM annotations').first())!.n,0); + assert.deepEqual((await conversationPage(env,'deadline',p.subject)).entries.map(e=>e.text),['Listen']); + await assert.rejects(receipt.complete(result.reply),/input ended/); + await assert.rejects(receipt.delta('Late delta'),/input ended/); + } +}); +test('reset between durable course completion and final delivery invalidates the completion receipt',async()=>{ + const {env,job,principal}=await setup();await job('reset-finish');const p=await principal(); + const {operation}=await reserveAssistant(env,p.subject,A,'chat','reset-finish-turn-0001','reset-finish',[]); + const receipt=assistantReceipt(env,operation),turn=await beginCourseTurn(env,'reset-finish',p,'Listen','reset-finish-turn-0001',0); + receipt.bindCourseClaim(turn.claimId);await receipt.effect(); + const revision=await finishCourseTurn(env,'reset-finish',p.subject,turn.turnId,turn.claimId,{reply:'Private answer',toolCalls:[],finishReason:'stop'},false,p.displayName,operation.id); + assert.equal(revision,2);await resetCourseConversation(env,'reset-finish',p.subject,revision); + await assert.rejects(receipt.complete('Private answer'),/input ended/); + await assert.rejects(receipt.delta('Late delta'),/input ended/); + assert.deepEqual((await conversationPage(env,'reset-finish',p.subject)).entries,[]); +}); test('course prompt writes conflict, preserve immutable history and invalidate only matching caches',async()=>{ const {env,job,db}=await setup();await job('a');await job('b','alice',B);await job('personal','alice',null);const empty=await hashSystemPromptFingerprint(); for(const [id,scope] of [['a',A],['b',B],['personal',null]] as const)assert.equal(await cacheGuideIfPromptCurrent({...env,ASSISTANT_COURSE_ID:scope},{jobId:id,text:'Old guide',model:'fixture',createdAt:new Date().toISOString()},0,empty),true); diff --git a/cloudflare/reliability.test.ts b/cloudflare/reliability.test.ts index 3c21318..e283f80 100644 --- a/cloudflare/reliability.test.ts +++ b/cloudflare/reliability.test.ts @@ -100,7 +100,7 @@ test('ninety-day purge redacts private request/cache content but preserves an un test('chat receipts never copy content; failure releases, partial/unknown count once, settled replay cannot call a model',async t=>{ const {db,env}=await setup(t); const first=await reserveAssistant(env,subject,null,'chat','human-message-0001','job',[]);const receipt=assistantReceipt(env,first.operation); - await receipt.delta('Secret transcript');await receipt.failed();await receipt.complete('Late secret'); + await receipt.delta('Secret transcript');await receipt.failed();await assert.rejects(receipt.complete('Late secret'),/input ended/); assert.equal((await readOperation(db,first.operation.id))!.state,'partial');assert.doesNotMatch((await readOperation(db,first.operation.id))!.result_json!,/Secret|Late/); await assert.rejects(reserveAssistant(env,subject,null,'chat','human-message-0001','job',[]),/already accepted/); const second=await reserveAssistant(env,subject,null,'chat','human-message-0002','job',[]);await assistantReceipt(env,second.operation).failed(); @@ -109,6 +109,10 @@ test('chat receipts never copy content; failure releases, partial/unknown count await assert.rejects(assistantReceipt(env,third.operation).delta('Late output')); const fourth=await reserveAssistant(env,subject,null,'chat','human-message-0004','job',[]);await assistantReceipt(env,fourth.operation).effect();await recoverExpired(db,Date.now()+90001); assert.equal((await readOperation(db,fourth.operation.id))!.error_code,'delivery_uncertain');assert.equal((await operationAllowance(db,subject,'chat')).completed,2); + await assert.rejects(assistantReceipt(env,fourth.operation).complete('Late usable result'),/input ended/); + const fifth=await reserveAssistant(env,subject,null,'chat','human-message-0005','job',[]); + await assert.rejects(assistantReceipt(env,fifth.operation).complete(' '),/could not finish/); + assert.equal((await readOperation(db,fifth.operation.id))!.state,'failed'); }); test('mocked lifecycle rejects empty/corrupt audio, failed storage and cancelled/failed predictions without a successful charge',async t=>{ diff --git a/docs/reliability-operations.md b/docs/reliability-operations.md index acc9092..9d892fb 100644 --- a/docs/reliability-operations.md +++ b/docs/reliability-operations.md @@ -129,6 +129,13 @@ prompt remain unchanged. Partial/trailing stream failure never calls another model. Tools-only replies get local narration, and duplicate tool IDs fail. Missing or different backup configuration fails closed. +Course deltas and effect intents atomically check the active conversation claim. +Notes and durable replies also require the live 90-second operation lease, +within the conversation's longer 120-second recovery lease. Final delivery +requires a winning operation settlement and the unchanged completed course +revision. Expiry, recovery or reset cannot authorize a stale tool event or +restore a cleared transcript. + ## Private operational and usage evidence Lifecycle and attempt events are transactionally recorded alongside the durable diff --git a/src/classroom/conversations.ts b/src/classroom/conversations.ts index fefd043..9745980 100644 --- a/src/classroom/conversations.ts +++ b/src/classroom/conversations.ts @@ -62,22 +62,28 @@ export async function beginCourseTurn(env: Env, jobId: string, principal: AppPri const turns = results.reverse().map(row => ({ role: row.kind === 'you' ? 'user' as const : 'assistant' as const, content: row.text.slice(0,2000) })); return { turnId, claimId, replay: null, revision: Number(revision)+1, turns }; } -export async function finishCourseTurn(env: Env, jobId: string, subject: string, turnId: string, claimId: string, result: ChatResult, failed = false, displayName = 'Student'): Promise { +export async function finishCourseTurn(env: Env, jobId: string, subject: string, turnId: string, claimId: string, result: ChatResult, failed = false, displayName = 'Student', operationId?: string): Promise { + // PR3's reservation may expire before the longer conversation recovery lease. + // Each effect checks it in the same transaction, so an old runner cannot + // restore output or notes after allowance recovery has fenced the operation. + const operationGuard=operationId?` AND EXISTS(SELECT 1 FROM app_operations o + WHERE o.id=? AND o.subject=? AND o.job_id=? AND o.kind='chat' AND o.state='running' AND o.lease_until>?)`:''; + const operationArgs=operationId?[operationId,subject,jobId,Date.now()]:[]; const statements = []; for (const call of result.toolCalls) if (!failed && call.name === 'add_note') { statements.push(env.DB.prepare(`INSERT INTO annotations(id,job_id,at_seconds,text,author_subject,author_name,provenance) - SELECT ?,job_id,?,?,subject,?,'server-assistant' FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now')`) - .bind(crypto.randomUUID(),Number(call.args.seconds),String(call.args.text).slice(0,200),displayName,jobId,subject,claimId)); + SELECT ?,job_id,?,?,subject,?,'server-assistant' FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now')${operationGuard}`) + .bind(crypto.randomUUID(),Number(call.args.seconds),String(call.args.text).slice(0,200),displayName,jobId,subject,claimId,...operationArgs)); } if (result.reply) statements.push(env.DB.prepare(`INSERT INTO course_messages(id,job_id,subject,turn_id,kind,provenance,text) - SELECT ?,job_id,subject,?,'coach','server-assistant',? FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now')`).bind(crypto.randomUUID(),turnId,result.reply,jobId,subject,claimId)); + SELECT ?,job_id,subject,?,'coach','server-assistant',? FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now')${operationGuard}`).bind(crypto.randomUUID(),turnId,result.reply,jobId,subject,claimId,...operationArgs)); if (result.toolCalls.length) statements.push(env.DB.prepare(`INSERT INTO course_messages(id,job_id,subject,turn_id,kind,provenance,text) - SELECT ?,job_id,subject,?,'action','server-tool',? FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now')`).bind(crypto.randomUUID(),turnId,JSON.stringify(result.toolCalls),jobId,subject,claimId)); + SELECT ?,job_id,subject,?,'action','server-tool',? FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now')${operationGuard}`).bind(crypto.randomUUID(),turnId,JSON.stringify(result.toolCalls),jobId,subject,claimId,...operationArgs)); statements.push(env.DB.prepare(`UPDATE course_turns SET state=?,reply=?,tools=?,finish_reason=?,revision=revision+1 - WHERE job_id=? AND subject=? AND turn_id=? AND state='pending' AND EXISTS(SELECT 1 FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now'))`) - .bind(failed?'failed':'complete',result.reply,JSON.stringify(result.toolCalls),result.finishReason,jobId,subject,turnId,jobId,subject,claimId)); - statements.push(env.DB.prepare(`UPDATE course_conversations SET revision=revision+1,pending_turn=NULL,pending_expires_at=NULL WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now') RETURNING revision`).bind(jobId,subject,claimId)); + WHERE job_id=? AND subject=? AND turn_id=? AND state='pending' AND EXISTS(SELECT 1 FROM course_conversations WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now'))${operationGuard}`) + .bind(failed?'failed':'complete',result.reply,JSON.stringify(result.toolCalls),result.finishReason,jobId,subject,turnId,jobId,subject,claimId,...operationArgs)); + statements.push(env.DB.prepare(`UPDATE course_conversations SET revision=revision+1,pending_turn=NULL,pending_expires_at=NULL WHERE job_id=? AND subject=? AND pending_turn=? AND pending_expires_at>datetime('now') AND expires_at>datetime('now')${operationGuard} RETURNING revision`).bind(jobId,subject,claimId,...operationArgs)); const saved = await env.DB.batch<{ revision: number }>(statements); if(!saved.at(-1)?.meta.changes)return null; const completed=await env.DB.prepare('SELECT revision FROM course_turns WHERE job_id=? AND subject=? AND turn_id=?').bind(jobId,subject,turnId).first<{revision:number}>(); diff --git a/src/index.ts b/src/index.ts index d7015ab..2a9d464 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1939,7 +1939,7 @@ app.post('/api/jobs/:id/chat', requireClassCode, async (c) => { const receipt=reservation?assistantReceipt(c.env,reservation.operation):null; let durable: Awaited> | null = null; if (course) { - try { durable = await beginCourseTurn(c.env,id,c.get('principal')!,turns.at(-1)!.content,body?.messageId,body?.revision); } + try { durable = await beginCourseTurn(c.env,id,c.get('principal')!,turns.at(-1)!.content,body?.messageId,body?.revision);receipt?.bindCourseClaim(durable.claimId); } catch(error) {await receipt?.failed();if(error instanceof CourseError)return courseErrorResponse(error);throw error;} } return sseResponse(c, async (emit, signal) => { @@ -1955,15 +1955,15 @@ app.post('/api/jobs/:id/chat', requireClassCode, async (c) => { } ); await receipt?.effect(); - const revision = durable ? await finishCourseTurn(c.env,id,principal!.subject,durable.turnId,durable.claimId,result,false,principal!.displayName) : undefined; + const revision = durable ? await finishCourseTurn(c.env,id,principal!.subject,durable.turnId,durable.claimId,result,false,principal!.displayName,reservation?.operation.id) : undefined; if(durable && revision===null) throw new AssistantError(503,'This conversation changed. Refresh before continuing.'); // A personal note also has a durable operation/call identity. Completion // replay cannot add it twice. Course notes are fenced in finishCourseTurn. if(principal && reservation && !durable) { for(const [index,call] of result.toolCalls.entries()) if(call.name==='add_note') { await c.env.DB.prepare(`INSERT OR IGNORE INTO annotations(id,job_id,at_seconds,text,author_subject,author_name,provenance) - SELECT ?,?,?,?,?,?,'server-assistant' WHERE EXISTS(SELECT 1 FROM app_operations WHERE id=? AND state='running')`) - .bind(`${reservation.operation.id}-${index}`,id,Number(call.args.seconds),String(call.args.text),principal.subject,principal.displayName,reservation.operation.id).run(); + SELECT ?,?,?,?,?,?,'server-assistant' WHERE EXISTS(SELECT 1 FROM app_operations WHERE id=? AND state='running' AND lease_until>?)`) + .bind(`${reservation.operation.id}-${index}`,id,Number(call.args.seconds),String(call.args.text),principal.subject,principal.displayName,reservation.operation.id,Date.now()).run(); } } const calls=principal?result.toolCalls.filter(call=>call.name!=='add_note'):result.toolCalls; @@ -1972,7 +1972,7 @@ app.post('/api/jobs/:id/chat', requireClassCode, async (c) => { await emit({type:'done',text:result.reply,finishReason:result.finishReason,operationId:reservation?.operation.id, ...(revision===undefined?{}:{revision}),notesChanged:!!principal&&result.toolCalls.some(call=>call.name==='add_note')}); } catch(error) { - if(durable)await finishCourseTurn(c.env,id,principal!.subject,durable.turnId,durable.claimId,{reply:streamed,toolCalls:[],finishReason:'error'},true); + if(durable)await finishCourseTurn(c.env,id,principal!.subject,durable.turnId,durable.claimId,{reply:streamed,toolCalls:[],finishReason:'error'},true,principal!.displayName,reservation?.operation.id); await receipt?.failed();throw error; } }); diff --git a/src/reliability/assistant.ts b/src/reliability/assistant.ts index 0f46408..71d2830 100644 --- a/src/reliability/assistant.ts +++ b/src/reliability/assistant.ts @@ -14,14 +14,18 @@ export async function reserveAssistant(env: Env, subject: string, courseId: stri return {operation:op,replay:false}; } export function assistantReceipt(env: Env,operation: Operation) { - let delivered=false; + let delivered=false,courseClaim:string|null=null; const mark=async(delivery:'partial'|'effect_intent')=>{ - const result=await env.DB.prepare("UPDATE app_operations SET result_json=?,updated_at=? WHERE id=? AND state='running' AND lease_until>?") - .bind(JSON.stringify({delivery}),Date.now(),operation.id,Date.now()).run(); + const result=await env.DB.prepare(`UPDATE app_operations SET result_json=?,updated_at=? WHERE id=? AND state='running' AND lease_until>? + AND (course_id IS NULL OR kind<>'chat' OR EXISTS(SELECT 1 FROM course_conversations c + WHERE c.job_id=app_operations.job_id AND c.subject=app_operations.subject AND c.pending_turn=? + AND c.pending_expires_at>datetime('now') AND c.expires_at>datetime('now')))`) + .bind(JSON.stringify({delivery}),Date.now(),operation.id,Date.now(),courseClaim).run(); if(!result.meta.changes)throw new OperationError('input_no_longer_active',409,'This input ended. Reload the conversation before continuing.'); delivered=true; }; return { + bindCourseClaim(claimId:string) {courseClaim=claimId;}, /** Persist BEFORE handing output to the response writer. A process crash * after this point must not turn a delivered partial into a free replay. */ async delta(value: string) { @@ -31,8 +35,22 @@ export function assistantReceipt(env: Env,operation: Operation) { * unusual provider response that supplied final content without deltas. */ async effect() {await mark('effect_intent');}, async complete(_reply: string) { - if(!_reply.trim()) {await settleOperation(env.DB,operation.id,'failed','no_usable_response');return;} - await settleOperation(env.DB,operation.id,'succeeded',null,{delivery:'completed'}); + if(!_reply.trim()) { + await settleOperation(env.DB,operation.id,'failed','no_usable_response'); + throw new OperationError('no_usable_response',503,'The Listening Guide could not finish this reply. Please try a new message.'); + } + // Final delivery is conditional on the same live reservation. A stale + // runner must not emit tools/done after recovery has already settled it. + // For course work, reset or a new turn also invalidates the saved receipt. + const result=await env.DB.prepare(`UPDATE app_operations SET state='succeeded',error_code=NULL,result_json=?, + lease_owner=NULL,lease_until=0,updated_at=? WHERE id=? AND state='running' AND lease_until>? AND cancel_requested=0 + AND (course_id IS NULL OR kind<>'chat' OR EXISTS(SELECT 1 FROM course_turns t + JOIN course_conversations c ON c.job_id=t.job_id AND c.subject=t.subject + WHERE t.job_id=app_operations.job_id AND t.subject=app_operations.subject + AND t.turn_id=app_operations.idempotency_key AND t.state='complete' AND c.pending_turn IS NULL + AND c.revision=t.revision AND c.expires_at>datetime('now')))`) + .bind(JSON.stringify({delivery:'completed'}),Date.now(),operation.id,Date.now()).run(); + if(!result.meta.changes)throw new OperationError('input_no_longer_active',409,'This input ended. Reload the conversation before continuing.'); }, async failed() { await settleOperation(env.DB,operation.id,delivered?'partial':'failed',delivered?'partial_response':'no_usable_response',{delivery:delivered?'partial':'none'}); From 64aaa287ed665e03ecb72780541e805c1c38cce2 Mon Sep 17 00:00:00 2001 From: zmuhls Date: Tue, 6 Oct 2026 15:28:36 -0400 Subject: [PATCH 10/25] Add isolated guest sessions and successful-use allowances --- cloudflare/gateway.ts | 6 +- cloudflare/guest.spec.mjs | 97 ++++++++++++ cloudflare/guest.test.ts | 194 ++++++++++++++++++++++++ cloudflare/guest.ts | 96 ++++++++++++ cloudflare/playwright.config.mjs | 2 +- cloudflare/retention.ts | 1 + cloudflare/sso.test.ts | 4 +- cloudflare/sso.ts | 16 +- cloudflare/test-worker.ts | 6 +- cloudflare/worker.ts | 31 +++- docs/guest-access.md | 54 +++++++ migrations/0022-reliable-operations.sql | 80 +++++++++- public/account.html | 7 +- public/account.js | 19 ++- public/app.js | 16 +- public/guest.js | 66 ++++++++ public/index.html | 5 +- schema.sql | 80 +++++++++- src/classroom/access.ts | 3 + src/daily-allowance.ts | 16 +- src/env.ts | 2 +- src/guest/access.ts | 74 +++++++++ src/identity.ts | 11 +- src/index.ts | 59 ++++--- src/reliability/assistant.ts | 5 +- src/reliability/ledger.ts | 29 ++-- src/reliability/splits.ts | 13 +- 27 files changed, 904 insertions(+), 88 deletions(-) create mode 100644 cloudflare/guest.spec.mjs create mode 100644 cloudflare/guest.test.ts create mode 100644 cloudflare/guest.ts create mode 100644 docs/guest-access.md create mode 100644 public/guest.js create mode 100644 src/guest/access.ts diff --git a/cloudflare/gateway.ts b/cloudflare/gateway.ts index 24211f2..797078c 100644 --- a/cloudflare/gateway.ts +++ b/cloudflare/gateway.ts @@ -31,7 +31,7 @@ function safeFailure(error: unknown, fallback: string): AssistantError { return new AssistantError(error instanceof CailError && error.status === 429 ? 503 : 502, message, { requestId, code, shouldRetry }); } -export function gatewayForRequest(binding: Fetcher | undefined, token: string | null, request: Request, release: string) { +export function gatewayForRequest(binding: Fetcher | undefined, token: string | null, request: Request, release: string, credentialKind: 'jwt' | 'key' = 'jwt') { const headers = new Headers(request.headers); const incoming = headers.get('x-cail-request-id') ?? headers.get('x-request-id'); if (incoming && REQUEST_ID.test(incoming)) headers.set('x-cail-request-id', incoming); @@ -73,7 +73,7 @@ export function gatewayForRequest(binding: Fetcher | undefined, token: string | model, messages: params.messages.map(({ role, content }) => ({ role, content })), ...(params.tools?.length ? { tools: params.tools.map(tool => ({ type: tool.type, function: { ...tool.function, parameters: JSON.parse(JSON.stringify(tool.function.parameters)) } })), tool_choice: 'auto' } : {}), stream: true, stream_options: { include_usage: true }, max_tokens: params.maxTokens, temperature: params.temperature, - }, { kind: 'jwt', token }, { signal, correlation }); + }, { kind: credentialKind, token }, { signal, correlation }); accepted = true; const responseId = response.headers.get('x-request-id') ?? response.headers.get('x-cail-request-id'); if (responseId && REQUEST_ID.test(responseId)) requestId = responseId; @@ -152,6 +152,8 @@ export function gatewayForRequest(binding: Fetcher | undefined, token: string | throw new AssistantError(503,COACH_DOWN); }, env.ASSISTANT_ABORT_SIGNAL ?? request.signal, 60000); return { stream, quota: () => deadline(async signal => { + // Anonymous guests must never inspect the sponsor application's balance. + if (credentialKind === 'key') throw new AssistantError(503,COACH_UNCONFIGURED); if (!binding || !token) throw new AssistantError(503, COACH_UNCONFIGURED); // cail-client validates the bounded quota envelope; it never grants access. const quotaClient = createCailClient({ app: 'stem-splitter', fetchImpl: async (input, init) => { diff --git a/cloudflare/guest.spec.mjs b/cloudflare/guest.spec.mjs new file mode 100644 index 0000000..bc43dc7 --- /dev/null +++ b/cloudflare/guest.spec.mjs @@ -0,0 +1,97 @@ +import { test,expect } from '@playwright/test'; +import { createTestHarness } from 'wrangler'; +import { createTestIdentityIssuer } from '@cuny-ai-lab/cail-identity/testing'; +import { readFile,mkdir } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { setupServer } from 'msw/node'; +import { http,HttpResponse } from 'msw'; +import { schemaStatements } from '../tests/e2e/schema-statements.mjs'; +test.use({ignoreHTTPSErrors:true}); + +test('guest browser enters through a verified check, resumes own work, conceals pending authority and switches cleanly',async({page,context})=>{ + let origin='';const issuer=await createTestIdentityIssuer(); + const mp3=await readFile(new URL('../tests/fixtures/audio/vocals.mp3',import.meta.url)); + let providerStarts=0,courseRequests=0,holdAccount=false,releaseAccount; + const network=setupServer( + http.post('https://challenges.cloudflare.com/turnstile/v0/siteverify',()=>HttpResponse.json({success:true,hostname:new URL(origin).hostname,action:'stem_guest',challenge_ts:new Date().toISOString()})), + http.post('https://api.replicate.com/v1/predictions',()=>{providerStarts++;return HttpResponse.json({id:'browser-guest-prediction',status:'starting'});}), + http.get('https://api.replicate.com/v1/predictions/browser-guest-prediction',()=>HttpResponse.json({id:'browser-guest-prediction',status:'succeeded',output:Object.fromEntries(['vocals','drums','bass','other'].map(name=>[name,`https://fixtures.replicate.delivery/${name}.mp3`]))})), + http.get('https://fixtures.replicate.delivery/:name.mp3',()=>new HttpResponse(mp3,{headers:{'Content-Type':'audio/mpeg'}})), + ); + network.listen({onUnhandledRequest:'error'}); + const config=JSON.parse(await readFile(new URL('./test-wrangler.jsonc',import.meta.url),'utf8')); + const harness=createTestHarness({root:fileURLToPath(new URL('./',import.meta.url)),workers:[{config:{...config,dev:{local_protocol:'https'}, + vars:{...config.vars,TEST_BROWSER:'true',TEST_JWKS:issuer.jwksJson,GUEST_ENABLED:'true',GUEST_COOKIE_SECRET:'browser-fixture-cookie-secret-0000000000',GUEST_TURNSTILE_SITE_KEY:'fixture-site-key', + GUEST_TURNSTILE_SECRET:'fixture-turnstile-secret-only',GUEST_GATEWAY_API_KEY:'sk-cail-fixture-sponsor-key-only-000000', + REPLICATE_API_TOKEN:'guest-fixture-token',REPLICATE_MODEL_VERSION:'guest-fixture-pin',WEBHOOK_SECRET:'guest-fixture-webhook'}, + }}]}); + const errors=[];page.on('pageerror',error=>errors.push(error.message)); + page.on('request',request=>{if(new URL(request.url()).pathname.startsWith('/api/classroom/'))courseRequests++;}); + try { + const {url}=await harness.listen();origin=url.origin;expect(url.protocol).toBe('https:');const worker=harness.getWorker('stem-preview-contract-test'); + await worker.fetch('/__fixture/schema',{method:'POST',headers:{'x-fixture':'local-only'},body:JSON.stringify(schemaStatements(await readFile(new URL('../schema.sql',import.meta.url),'utf8')))}); + await page.route('https://fonts.googleapis.com/**',route=>route.fulfill({body:'',contentType:'text/css'})); + await page.route('https://fonts.gstatic.com/**',route=>route.abort()); + await page.route('https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit',route=>route.fulfill({contentType:'text/javascript',body:`window.turnstile={render(root,options){const button=document.createElement('button');button.textContent='Complete fixture guest check';button.onclick=()=>options.callback('browser-fixture-'+crypto.randomUUID());root.append(button);return 'fixture';},reset(){}};`})); + await page.route('**/api/account',async route=>{ + if(holdAccount){await new Promise(resolve=>{releaseAccount=resolve;});holdAccount=false;await route.fulfill({status:503,contentType:'application/json',body:'{"error":"fixture unavailable"}'});return;} + await route.continue(); + }); + await page.goto(origin); + await expect(page.getByRole('button',{name:'Continue as guest',exact:true})).toBeVisible(); + await page.getByRole('button',{name:'Continue as guest',exact:true}).click(); + await page.getByRole('button',{name:'Complete fixture guest check'}).click(); + await expect(page.getByRole('link',{name:'My guest session',exact:true})).toBeVisible(); + expect(courseRequests).toBe(0); + const uploaded=page.waitForResponse(response=>response.url().includes('/api/local-uploads/')); + await page.locator('#file-input').setInputFiles(fileURLToPath(new URL('../tests/fixtures/audio/source.wav',import.meta.url))); + const upload=await uploaded;expect(upload.status()).toBe(204); + await expect(page.locator('.console-title')).toHaveText('source.wav',{timeout:20000}); + await expect(page.locator('.badge.ready')).toBeVisible({timeout:20000}); + expect(providerStarts).toBe(1); + await expect(page.locator('.share-btn:not(.to-remix-btn):not(.refresh-btn)')).toBeHidden(); + await page.reload(); + await expect(page.locator('.console-title')).toHaveText('source.wav'); + await page.getByRole('link',{name:'My guest session',exact:true}).click(); + await expect(page.locator('#account-splits')).toContainText('1 completed'); + await expect(page.locator('#account-splits')).toContainText('of 5'); + await expect(page.locator('#account-chat')).toContainText('of 25'); + await expect(page.locator('#account-admin')).toBeHidden(); + await expect(page.locator('#account-reference')).toBeHidden(); + for(const width of [320,390,768,1280]){await page.setViewportSize({width,height:900});expect(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth)).toBe(true);} + await page.goto(origin); + await expect(page.locator('.console-title')).toHaveText('source.wav'); + holdAccount=true; + await page.evaluate(()=>window.dispatchEvent(new Event('focus'))); + await expect(page.locator('main')).toBeHidden(); + await expect(page.locator('#session-check')).toContainText('Checking'); + releaseAccount(); + await expect(page.locator('#session-check')).toContainText('hidden until access is checked'); + await expect(page.locator('main')).toBeHidden(); + await page.getByRole('button',{name:'Retry',exact:true}).click(); + await expect(page.locator('main')).toBeVisible(); + await expect(page.locator('.console-title')).toHaveText('source.wav'); + for(const width of [320,390,768,1280]){await page.setViewportSize({width,height:900});expect(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth)).toBe(true);} + await expect(page.locator('main')).toHaveCount(1);await expect(page.locator('h1')).toHaveCount(1); + if(process.env.STEM_SCREENSHOT_DIR){ + await mkdir(process.env.STEM_SCREENSHOT_DIR,{recursive:true}); + await page.evaluate(()=>{window.scrollTo(0,0);return new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)));}); + await page.screenshot({path:process.env.STEM_SCREENSHOT_DIR+'/guest-resumed-work-'+(process.env.STEM_BROWSER||'chrome')+'.png',animations:'disabled'}); + await page.locator('.console').screenshot({path:process.env.STEM_SCREENSHOT_DIR+'/guest-owned-split-'+(process.env.STEM_BROWSER||'chrome')+'.png',animations:'disabled'}); + } + const oldCookie=(await context.cookies()).find(cookie=>cookie.name==='__Host-stem-guest'); + await context.clearCookies(); + const second=await worker.fetch(origin+'/auth/guest',{method:'POST',headers:{Origin:origin,'Content-Type':'application/json'},body:JSON.stringify({token:'second-browser-guest'})}); + expect(second.status).toBe(200);const cookie=second.headers.get('set-cookie').split(';')[0].split('='); + await context.addCookies([{name:cookie[0],value:cookie[1],url:origin,secure:true,httpOnly:true,sameSite:'Lax'}]); + await page.evaluate(()=>window.dispatchEvent(new Event('focus'))); + await expect(page.locator('.console-title')).toHaveCount(0); + await expect(page.getByRole('link',{name:'My guest session',exact:true})).toBeVisible(); + await page.getByRole('link',{name:'My guest session',exact:true}).click(); + await expect(page.locator('#account-splits')).toContainText('0 completed'); + await page.getByRole('button',{name:'End guest session',exact:true}).click(); + await expect(page.getByRole('button',{name:'Continue as guest',exact:true})).toBeVisible(); + expect((await context.cookies()).some(cookie=>cookie.name==='__Host-stem-guest')).toBe(false); + expect(oldCookie.value).not.toBe(cookie[1]);expect(errors).toEqual([]); + }finally{releaseAccount?.();await page.unrouteAll({behavior:'wait'});await page.close();await harness.close();network.close();} +}); diff --git a/cloudflare/guest.test.ts b/cloudflare/guest.test.ts new file mode 100644 index 0000000..ac00fe1 --- /dev/null +++ b/cloudflare/guest.test.ts @@ -0,0 +1,194 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import { createTestHarness } from 'wrangler'; +import { createTestIdentityIssuer, TEST_SUBJECTS } from '@cuny-ai-lab/cail-identity/testing'; +import { setupServer } from 'msw/node'; +import { http, HttpResponse } from 'msw'; +import { schemaStatements } from '../tests/e2e/schema-statements.mjs'; +import { GUEST_COOKIE, guestConfigured, startGuestSession } from './guest.ts'; +import { readGuestSession } from './guest.ts'; +import { mkdtempSync,readFileSync,rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { SqliteD1 } from '../server/d1.ts'; +import { reserveOperation,settleOperation } from '../src/reliability/ledger.ts'; +import { dailyWindow,splitAllowance } from '../src/daily-allowance.ts'; +import type { Env } from '../src/env.ts'; + +const origin='https://split.test'; +const settings={GUEST_ENABLED:'true',GUEST_COOKIE_SECRET:'fixture-cookie-secret-'.repeat(3),GUEST_TURNSTILE_SITE_KEY:'fixture-site-key', + GUEST_TURNSTILE_SECRET:'fixture-turnstile-secret-only',GUEST_GATEWAY_API_KEY:'sk-cail-fixture-sponsor-key-only-000000'}; + +test('guest activation requires every reviewed capability; legacy provider keys never enable it',()=>{ + const env={...settings,GATEWAY:{},GATEWAY_MODEL:'glm-5.2',GATEWAY_FALLBACK_MODEL:'deepseek-v4-flash-0731',REQUEST_LIMIT:{}} as never; + assert.equal(guestConfigured(env),true); + for(const field of Object.keys(settings))assert.equal(guestConfigured({...env,[field]:undefined,OPENROUTER_API_KEY:'old-provider-key'}),false,field); + assert.equal(guestConfigured({...env,GUEST_ENABLED:'false'}),false); + assert.equal(guestConfigured({...env,GATEWAY:undefined}),false); +}); + +test('guest-start rate rejection performs no database, verification or provider work',async t=>{ + let databaseCalls=0,externalCalls=0;const ingressKeys:string[]=[]; + t.mock.method(globalThis,'fetch',async()=>{externalCalls++;throw new Error('No external call expected');}); + const env={...settings,GATEWAY:{},GATEWAY_MODEL:'glm-5.2',GATEWAY_FALLBACK_MODEL:'deepseek-v4-flash-0731',PUBLIC_BASE_URL:origin, + REQUEST_LIMIT:{limit:async({key}:{key:string})=>{ingressKeys.push(key);return {success:false};}}, + DB:{prepare:()=>{databaseCalls++;throw new Error('No database call expected');}}, + } as unknown as Parameters[1]; + const response=await startGuestSession(new Request(origin+'/auth/guest',{method:'POST',headers:{Origin:origin,'Content-Type':'application/json','cf-connecting-ip':'192.0.2.1'},body:'{"token":"unused-fixture-proof"}'}),env); + assert.equal(response.status,429);assert.equal(response.headers.get('Retry-After'),'60'); + assert.equal(response.headers.get('Set-Cookie'),null);assert.equal((await response.json() as any).error.code,'guest_rate_limited'); + assert.deepEqual(ingressKeys,['stem-guest-start:192.0.2.1']);assert.equal(databaseCalls,0);assert.equal(externalCalls,0); +}); + +test('workerd guests: verified cookie, split/chat settlement, replay, isolation, proof refresh and logout',async()=>{ + const issuer=await createTestIdentityIssuer(); + const member=TEST_SUBJECTS.alice; + const jwt=await issuer.mintIdentityJwt({audience:'cail:stem-splitter',subject:member}); + const mp3=await readFile(new URL('../tests/fixtures/audio/vocals.mp3',import.meta.url)); + const wav=await readFile(new URL('../tests/fixtures/audio/source.wav',import.meta.url)); + let challengeMode='ok',predictionMode='success',starts=0,verifications=0; + const predictions=new Map(); + const network=setupServer( + http.post('https://challenges.cloudflare.com/turnstile/v0/siteverify',async({request})=>{ + const body=await request.json() as {secret:string;response:string}; + assert.equal(body.secret,settings.GUEST_TURNSTILE_SECRET);verifications++; + return HttpResponse.json({success:challengeMode!=='failed',hostname:challengeMode==='wrong-host'?'other.test':'split.test', + action:challengeMode==='wrong-action'?'other':'stem_guest',challenge_ts:new Date(Date.now()-(challengeMode==='expired'?400000:0)).toISOString()}); + }), + http.post('https://api.replicate.com/v1/predictions',async({request})=>{ + const input=await request.json() as {version:string};assert.equal(input.version,'guest-fixture-pin'); + const id=`guest-prediction-${++starts}`;predictions.set(id,predictionMode);return HttpResponse.json({id,status:'starting'}); + }), + http.get('https://api.replicate.com/v1/predictions/:id',({params})=>HttpResponse.json({id:params.id, + status:predictions.get(String(params.id))==='failed'?'failed':'succeeded',output:Object.fromEntries(['vocals','drums','bass','other'].map(name=>[name,`https://fixtures.replicate.delivery/${name}.mp3`]))})), + http.get('https://fixtures.replicate.delivery/:name.mp3',()=>new HttpResponse(mp3,{headers:{'Content-Type':'audio/mpeg'}})), + ); + network.listen({onUnhandledRequest:'error'}); + const harness=createTestHarness({workers:[{configPath:fileURLToPath(new URL('./test-wrangler.jsonc',import.meta.url)), + vars:{TEST_JWKS:issuer.jwksJson,...settings},secrets:{REPLICATE_API_TOKEN:'guest-fixture-token',REPLICATE_MODEL_VERSION:'guest-fixture-pin',WEBHOOK_SECRET:'guest-fixture-webhook'}}]}); + try { + await harness.listen();const worker=harness.getWorker('stem-preview-contract-test'); + async function sql(statements:string[]){assert.equal((await worker.fetch('/__fixture/schema',{method:'POST',headers:{'x-fixture':'local-only'},body:JSON.stringify(statements)})).status,200);} + await sql(schemaStatements(await readFile(new URL('../schema.sql',import.meta.url),'utf8'))); + const call=(path:string,cookie='',init:RequestInit={})=>worker.fetch(origin+path,{redirect:'manual',...init,headers:{Origin:origin,'Content-Type':'application/json',...(cookie?{Cookie:cookie}:{}),...init.headers}}); + const begin=(token:string,cookie='',headers={})=>call('/auth/guest',cookie,{method:'POST',body:JSON.stringify({token}),headers}); + assert.equal((await call('/api/account')).status,401); + assert.equal((await begin('cross-site','',{Origin:'https://attacker.test'})).status,403); + assert.equal(verifications,0); + for(challengeMode of ['failed','wrong-host','wrong-action','expired']){const check=await begin(challengeMode);assert.equal(check.status,403,`${challengeMode}: ${await check.text()}`);} + challengeMode='ok'; + const first=await begin('unique-guest-token-a');assert.equal(first.status,200,await first.clone().text()); + assert.match(first.headers.get('set-cookie')!,/Secure; HttpOnly; SameSite=Lax/); + const cookieA=first.headers.get('set-cookie')!.split(';')[0]; + assert.equal((await begin('unique-guest-token-a')).status,403,'challenge replay'); + const cookieB=(await begin('unique-guest-token-b')).headers.get('set-cookie')!.split(';')[0]; + const account=await(await call('/api/account',cookieA)).json() as any; + assert.equal(account.account.role,'guest');assert.equal(account.account.quotaClass,'guest');assert.equal(account.account.course,null); + assert.equal(account.splitAllowance.limit,5);assert.equal(account.chatAllowance.limit,25); + assert.equal((await(await call('/api/account',cookieA)).json() as any).account.subject,account.account.subject,'resumed guest'); + const macOffset=cookieA.lastIndexOf('.')+1; + assert.equal((await call('/api/account',cookieA.slice(0,macOffset)+(cookieA[macOffset]==='a'?'b':'a')+cookieA.slice(macOffset+1))).status,401,'tampered MAC'); + assert.equal((await call('/api/account',cookieA+'; '+cookieB)).status,401,'duplicate cookie'); + const memberCookie='__Host-stem-session=invalid'; + assert.equal((await call('/api/account',cookieA+'; '+memberCookie)).status,401,'invalid member cannot become guest'); + assert.equal((await begin('member-downgrade',cookieA+'; '+memberCookie)).status,403); + for(const path of ['/api/classroom/courses','/api/teacher/prompt','/api/teacher/folders','/api/admin/users','/api/model-quota'])assert.equal((await call(path,cookieA)).status,403,path); + assert.equal((await call('/api/account',cookieA,{headers:{'x-stem-course':'msh-245-the-american-musical-experience-fall-2026-01'}})).status,403); + assert.deepEqual(await(await call('/api/teacher/me',cookieA)).json(),{teacher:null}); + + await sql([`INSERT INTO app_users(subject) VALUES('${member}')`, + `INSERT INTO jobs(id,filename,source_key,status,model,stems) VALUES('member-private','Private member lesson','uploads/private.wav','done','htdemucs_ft','[{"name":"vocals","key":"stems/member-private/vocals.mp3"}]')`, + `INSERT INTO job_owners(job_id,subject) VALUES('member-private','${member}')`, + `INSERT INTO listening_conversations(job_id,subject,entries,revision,expires_at) VALUES('member-private','${member}','[{"kind":"you","text":"Private member transcript"}]',1,datetime('now','+1 day'))`]); + for(const path of ['/api/jobs/member-private','/api/jobs/member-private/listening-conversation','/api/files/stems/member-private/vocals.mp3']){ + const response=await call(path,cookieA);assert.equal(response.status,404,path);assert.doesNotMatch(await response.text(),/Private member/); + } + assert.doesNotMatch(await(await call('/api/jobs',cookieA)).text(),/Private member/); + + const grant=await(await call('/api/uploads',cookieA,{method:'POST',body:'{"filename":"guest.wav"}'})).json() as any; + assert.equal((await call(grant.uploadUrl,cookieB,{method:'PUT',headers:{'Content-Type':'audio/wav','Content-Length':String(wav.length)},body:wav})).status,404); + assert.equal((await call(grant.uploadUrl,cookieA,{method:'PUT',headers:{'Content-Type':'audio/wav','Content-Length':String(wav.length)},body:wav})).status,204); + const body=JSON.stringify({key:grant.key,filename:'Guest owned fixture',model:'htdemucs_ft'}); + const submit=(key:string)=>call('/api/jobs',cookieA,{method:'POST',headers:{'Idempotency-Key':key},body}); + async function finish(id:string){for(let i=0;i<50;i++){const response=await call(`/api/jobs/${id}`,cookieA);assert.equal(response.status,200,await response.clone().text());const job=await response.json() as any;if(['done','failed'].includes(job.status))return job;await new Promise(resolve=>setTimeout(resolve,20));}assert.fail('fixture job did not settle');} + predictionMode='failed';let response=await submit('guest-split-failed-0001');assert.equal(response.status,202,await response.clone().text()); + const failed=await response.json() as any;assert.equal((await finish(failed.id)).status,'failed'); + assert.equal((await(await call('/api/account',cookieA)).json() as any).splitAllowance.remaining,5); + predictionMode='success';let job:any; + for(let i=0;i<5;i++){ + const key=`guest-split-success-${String(i).padStart(4,'0')}`;response=await submit(key);assert.equal(response.status,202,await response.clone().text()); + const created=await response.json() as any;const repeated=await submit(key);assert.equal(repeated.status,200);assert.equal((await repeated.json() as any).id,created.id); + job=await finish(created.id);assert.equal(job.status,'done'); + } + assert.equal(starts,6,'failed plus five successful predictions, no replay start'); + assert.equal((await submit('guest-split-sixth-0006')).status,429); + const count=await(await call('/api/account',cookieA)).json() as any;assert.equal(count.splitAllowance.completed,5);assert.equal(count.splitAllowance.inProgress,0); + for(const path of [`/api/jobs/${job.id}`,`/api/jobs/${job.id}/listening-conversation`,new URL(job.stems[0].url,origin).pathname])assert.equal((await call(path,cookieB)).status,404,path); + assert.equal((await call(`/api/jobs/${job.id}`,cookieA,{headers:{'x-fixture-identity':jwt}})).status,404,'member does not inherit guest work'); + assert.equal((await call(`/api/jobs/${job.id}/share`,cookieA,{method:'POST',body:'{}'})).status,403); + const usage=JSON.stringify({events:[{id:'guest-playback-event-0001',type:'playback_start',jobId:job.id}]}); + assert.equal((await call('/api/usage-events',cookieA,{method:'POST',body:usage})).status,200,'own playback telemetry'); + assert.equal((await call('/api/usage-events',cookieB,{method:'POST',body:usage})).status,404,'other guest cannot attach a job to telemetry'); + assert.equal((await call('/api/admin/usage-events',cookieA)).status,403); + const note=await call(`/api/jobs/${job.id}/annotations`,cookieA,{method:'POST',body:JSON.stringify({atSeconds:1,text:'Guest listening note'})}); + assert.equal(note.status,200);const noteId=(await note.json() as any).id; + assert.equal((await call(`/api/jobs/${job.id}/annotations/${noteId}`,cookieB,{method:'DELETE'})).status,404); + assert.equal((await call(`/api/jobs/${job.id}/annotations/${noteId}`,cookieA,{method:'DELETE'})).status,200); + assert.equal((await call(`/api/jobs/${job.id}/listening-conversation`,cookieA,{method:'PUT',body:JSON.stringify({revision:0,entries:[{kind:'you',text:'Only guest A'}]})})).status,200); + assert.match(await(await call(`/api/jobs/${job.id}/listening-conversation`,cookieA)).text(),/Only guest A/); + const chat=(key:string,mode='ok')=>call(`/api/jobs/${job.id}/chat`,cookieA,{method:'POST',headers:{'x-fixture-gateway-mode':mode},body:JSON.stringify({messageId:key,messages:[{role:'user',content:'What should I listen for?'}]})}); + response=await chat('guest-chat-empty-0001','empty');await response.text(); + assert.equal((await(await call('/api/account',cookieA)).json() as any).chatAllowance.completed,0); + assert.equal((await(await call('/api/account',cookieA)).json() as any).chatAllowance.inProgress,0); + for(let i=0;i<25;i++){response=await chat(`guest-chat-human-${String(i).padStart(4,'0')}`);assert.equal(response.status,200);assert.match(await response.text(),/Listen for the bass/);} + assert.equal((await chat('guest-chat-human-0000')).status,409,'human replay has no extra inference'); + assert.equal((await chat('guest-chat-human-0026')).status,429); + assert.equal((await(await call('/api/account',cookieA)).json() as any).chatAllowance.completed,25); + const stats=await worker.fetch('/__fixture/gateway-stats',{headers:{'x-fixture':'local-only'}});assert.equal((await stats.json() as any).gatewayCalls,26); + await sql([`UPDATE guest_sessions SET verified_day='2000-01-01' WHERE subject='${account.account.subject}'`]); + assert.equal((await call('/api/jobs',cookieA)).status,200,'old proof preserves own reads'); + assert.equal((await submit('guest-needs-proof-0001')).status,403); + assert.equal((await begin('guest-proof-refresh',cookieA)).status,200); + assert.equal((await(await call('/api/account',cookieA)).json() as any).splitAllowance.completed,5,'proof refresh does not reset budget'); + assert.equal((await call('/auth/logout',cookieA,{method:'POST'})).status,303); + assert.equal((await call(`/api/jobs/${job.id}`,cookieA)).status,401,'logout revokes copied cookie'); + assert.equal((await call('/api/jobs',cookieB)).status,200,'other guest remains isolated and active'); + } finally {await harness.close();network.close();} +}); + +test('guest expiry, signing scope, immutable quota class, atomic reservations and midnight reset',async t=>{ + const directory=mkdtempSync(join(tmpdir(),'stem-guest-'));t.after(()=>rmSync(directory,{recursive:true,force:true})); + const local=new SqliteD1(join(directory,'db'));local.applySchema(readFileSync(new URL('../schema.sql',import.meta.url),'utf8')); + const db=local as unknown as D1Database,now=Date.now(),id='a'.repeat(64),subject=`guest-${id}`,expires=now+60000; + await db.prepare('INSERT INTO guest_sessions(subject,created_at,expires_at,verified_day) VALUES(?,?,?,?)').bind(subject,now,expires,dailyWindow().day).run(); + const env={...settings,DB:db} as Env; + const key=await crypto.subtle.importKey('raw',new TextEncoder().encode(settings.GUEST_COOKIE_SECRET),{name:'HMAC',hash:'SHA-256'},false,['sign']); + const sign=async(exp:number,scope=origin)=>Buffer.from(await crypto.subtle.sign('HMAC',key,new TextEncoder().encode(`stem-splitter-guest-v1\n${scope}\n${id}.${exp}`))).toString('base64url'); + const cookie=`${GUEST_COOKIE}=${id}.${expires}.${await sign(expires)}`; + assert.equal((await readGuestSession(new Request(origin,{headers:{Cookie:cookie}}),env))?.subject,subject); + const alphabet='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_',signature=await sign(expires); + const noncanonical=signature.slice(0,-1)+alphabet[alphabet.indexOf(signature.at(-1)!)+1]; + assert.deepEqual(Buffer.from(noncanonical,'base64url'),Buffer.from(signature,'base64url'),'only ignored padding bits changed'); + assert.equal(await readGuestSession(new Request(origin,{headers:{Cookie:`${GUEST_COOKIE}=${id}.${expires}.${noncanonical}`}}),env),null,'noncanonical MAC encoding'); + await assert.rejects(readGuestSession(new Request(origin,{headers:{Cookie:cookie}}),{...env,DB:{prepare(){throw new Error('fixture database unavailable');}} as unknown as D1Database}),/fixture database unavailable/,'storage outage preserves the distinction from an invalid cookie'); + assert.equal(await readGuestSession(new Request('https://other.test',{headers:{Cookie:cookie}}),env),null); + const past=now-1000;assert.equal(await readGuestSession(new Request(origin,{headers:{Cookie:`${GUEST_COOKIE}=${id}.${past}.${await sign(past)}`}}),env),null,'correctly signed expired cookie'); + assert.equal(await readGuestSession(new Request(origin,{headers:{Cookie:`${GUEST_COOKIE}=${id}.${expires+1000}.${await sign(expires+1000)}`}}),env),null,'database expiry must match'); + const reserve=(n:number,at=now)=>reserveOperation(db,{subject,quotaClass:'guest',courseId:null,kind:'split',key:`guest-parallel-${String(n).padStart(4,'0')}`,fingerprint:`${n}`,phase:'split',now:at}); + const first=await reserve(0);await settleOperation(db,first.operation.id,'failed'); + for(let i=1;i<=3;i++){const value=await reserve(i);await settleOperation(db,value.operation.id,'succeeded');} + const concurrent=await Promise.allSettled([4,5,6,7].map(n=>reserve(n))); + assert.equal(concurrent.filter(result=>result.status==='fulfilled').length,2); + assert.equal((await splitAllowance(db,subject,new Date(now),'guest')).remaining,0); + await assert.rejects(db.prepare("UPDATE app_operations SET quota_class='member' WHERE id=?").bind(first.operation.id).run(),/immutable/); + await assert.rejects(db.prepare("INSERT INTO app_users(subject) VALUES(?)").bind(subject).run(),/CHECK/); + await assert.rejects(reserveOperation(db,{subject,quotaClass:'member',courseId:null,kind:'chat',key:'guest-member-forgery',fingerprint:'x',phase:'chat'}),/FOREIGN KEY/); + await assert.rejects(reserveOperation(db,{subject,quotaClass:'guest',courseId:'course-a',kind:'chat',key:'guest-course-forgery',fingerprint:'x',phase:'chat'}),/course/); + const tomorrow=now+86400000; + assert.equal((await splitAllowance(db,subject,new Date(tomorrow),'guest')).remaining,5,'UTC-day allowance is separate'); + await db.prepare('UPDATE guest_sessions SET revoked_at=? WHERE subject=?').bind(now,subject).run(); + assert.equal(await readGuestSession(new Request(origin,{headers:{Cookie:cookie}}),env),null,'revoked copied cookie'); + await assert.rejects(db.prepare('UPDATE guest_sessions SET revoked_at=NULL WHERE subject=?').bind(subject).run(),/immutable/); +}); diff --git a/cloudflare/guest.ts b/cloudflare/guest.ts new file mode 100644 index 0000000..775dc58 --- /dev/null +++ b/cloudflare/guest.ts @@ -0,0 +1,96 @@ +import type { Env } from '../src/env.ts'; +import { activeGuest, guestFailure, GUEST_SESSION_MS, GUEST_SUBJECT, type GuestSession } from '../src/guest/access.ts'; +import { dailyWindow } from '../src/daily-allowance.ts'; +import { readBoundedJsonRequest } from '../src/http/bounded-request.ts'; +import { readBoundedResponse } from '../src/http/bounded-response.ts'; +import { uniqueCookie } from './session-contract/session.js'; +import { gatewayModelsConfigured } from './gateway.ts'; + +export const GUEST_COOKIE = '__Host-stem-guest'; +const MEMBER_COOKIE = '__Host-stem-session'; +const TOKEN = /^([0-9a-f]{64})\.(\d{13})\.([A-Za-z0-9_-]{43})$/; +const encode = (bytes: Uint8Array) => btoa(String.fromCharCode(...bytes)).replaceAll('+','-').replaceAll('/','_').replaceAll('=',''); +const cookie = (value: string, seconds: number) => `${GUEST_COOKIE}=${value}; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age=${seconds}`; +export const clearGuestCookie = () => cookie('',0); +export const hasMemberCookie = (request: Request) => (request.headers.get('cookie') ?? '').split(';').some(part=>part.trim().split('=',1)[0]===MEMBER_COOKIE); +export interface GuestAdapterEnv extends Env { + GATEWAY?: Fetcher; + GATEWAY_MODEL?: string; + GATEWAY_FALLBACK_MODEL?: string; + CANONICAL_BASE_URL?: string; +} +/** Presence is a local readiness gate, not proof that Registry granted spend. + * Every model request still receives live Registry authorization at Gateway. */ +export function guestConfigured(env: GuestAdapterEnv): boolean { + return env.GUEST_ENABLED === 'true' && typeof env.GUEST_COOKIE_SECRET === 'string' && env.GUEST_COOKIE_SECRET.length >= 32 && env.GUEST_COOKIE_SECRET.length <= 512 + && /^[A-Za-z0-9_-]{10,128}$/.test(env.GUEST_TURNSTILE_SITE_KEY ?? '') + && /^[A-Za-z0-9_-]{20,512}$/.test(env.GUEST_TURNSTILE_SECRET ?? '') + && /^sk-cail-[A-Za-z0-9_-]{20,256}$/.test(env.GUEST_GATEWAY_API_KEY ?? '') + && Boolean(env.GATEWAY && env.REQUEST_LIMIT && gatewayModelsConfigured(env.GATEWAY_MODEL,env.GATEWAY_FALLBACK_MODEL)); +} +async function signingKey(env: Env) { + if (!env.GUEST_COOKIE_SECRET || env.GUEST_COOKIE_SECRET.length < 32) throw new Error('Guest signing unavailable'); + return crypto.subtle.importKey('raw',new TextEncoder().encode(env.GUEST_COOKIE_SECRET),{name:'HMAC',hash:'SHA-256'},false,['sign','verify']); +} +const message = (origin: string,id: string,expires: number) => new TextEncoder().encode(`stem-splitter-guest-v1\n${origin}\n${id}.${expires}`); +export async function readGuestSession(request: Request,env: Env): Promise { + const match = TOKEN.exec(uniqueCookie(request,GUEST_COOKIE)); + if (!match || !Number.isSafeInteger(Number(match[2])) || Number(match[2]) <= Date.now() || Number(match[2]) > Date.now()+GUEST_SESSION_MS+1000) return null; + try { + const signature = Uint8Array.from(atob(match[3].replaceAll('-','+').replaceAll('_','/')+'='),c=>c.charCodeAt(0)); + if (encode(signature) !== match[3]) return null; + if (!await crypto.subtle.verify('HMAC',await signingKey(env),signature,message(new URL(request.url).origin,match[1],Number(match[2])))) return null; + } catch { return null; } + // An unavailable database is not an invalid cookie. Preserve the browser's + // session and let the adapter report 503 while keeping private content hidden. + const session=await activeGuest(env.DB,`guest-${match[1]}`); + return session?.expiresAt === Number(match[2]) ? session : null; +} +export async function revokeGuestSession(request: Request,env: Env): Promise { + const session=await readGuestSession(request,env); + if (session) await env.DB.prepare('UPDATE guest_sessions SET revoked_at=? WHERE subject=? AND revoked_at IS NULL').bind(Date.now(),session.subject).run(); +} +export async function startGuestSession(request: Request,env: GuestAdapterEnv): Promise { + const url=new URL(request.url); + if (request.method !== 'POST') return new Response(null,{status:405,headers:{Allow:'POST'}}); + if (url.protocol !== 'https:' || ![env.PUBLIC_BASE_URL,env.CANONICAL_BASE_URL].includes(url.origin) + || request.headers.get('origin') !== url.origin || request.headers.get('sec-fetch-site') === 'cross-site') return guestFailure('guest_origin_invalid',403,'Open Stem Splitter directly to start a guest session.'); + if (hasMemberCookie(request)) return guestFailure('guest_signout_required',403,'Sign out of your existing account before starting a separate guest session.'); + if (!guestConfigured(env)) return guestFailure('guest_unavailable',503,'Guest access is not available. You can use CUNY Login.'); + const limited=await env.REQUEST_LIMIT!.limit({key:`stem-guest-start:${request.headers.get('cf-connecting-ip') || 'unknown'}`}); + if (!limited.success) return new Response(JSON.stringify({error:{code:'guest_rate_limited',message:'Please wait a minute before trying again.'}}),{status:429,headers:{'Content-Type':'application/json','Retry-After':'60'}}); + let token: unknown; + try { token=(await readBoundedJsonRequest(request,4096) as {token?:unknown})?.token; } catch { return guestFailure('guest_verification_invalid',403,'Complete the guest check and try again.'); } + if (typeof token !== 'string' || token.length < 1 || token.length > 2048) return guestFailure('guest_verification_invalid',403,'Complete the guest check and try again.'); + let result: {success?:unknown;hostname?:unknown;action?:unknown;challenge_ts?:unknown}; + try { + const response=await fetch('https://challenges.cloudflare.com/turnstile/v0/siteverify',{method:'POST',headers:{'Content-Type':'application/json'}, + body:JSON.stringify({secret:env.GUEST_TURNSTILE_SECRET,response:token}),redirect:'manual',signal:AbortSignal.timeout(5000)}); + if (!response.ok) throw new Error('Verification unavailable'); + const failed=()=>new Error('Verification unavailable'); + result=JSON.parse(new TextDecoder().decode(await readBoundedResponse(response,{maximumBytes:8192,timeoutMs:5000,errors:{tooLarge:failed,timedOut:failed,unreadable:failed}}))); + } catch { return guestFailure('guest_verification_unavailable',503,'The guest check is temporarily unavailable. Try again shortly.'); } + const time=typeof result.challenge_ts==='string'?Date.parse(result.challenge_ts):NaN; + if (result.success !== true || result.hostname !== url.hostname || result.action !== 'stem_guest' || !Number.isFinite(time) || time>Date.now()+30000 || timeb.toString(16).padStart(2,'0')).join(''); + const now=Date.now(), day=dailyWindow().day; + const previous=await readGuestSession(request,env); + const id=previous?.subject.slice(6) ?? Array.from(crypto.getRandomValues(new Uint8Array(32)),b=>b.toString(16).padStart(2,'0')).join(''); + const subject=`guest-${id}`,expiresAt=previous?.expiresAt ?? now+GUEST_SESSION_MS; + if (!GUEST_SUBJECT.test(subject)) throw new Error('Guest subject invalid'); + const signature=encode(new Uint8Array(await crypto.subtle.sign('HMAC',await signingKey(env),message(url.origin,id,expiresAt)))); + try { + await env.DB.batch([ + env.DB.prepare('INSERT INTO guest_challenges(token_hash,expires_at) VALUES(?,?)').bind(tokenHash,now+600000), + previous ? env.DB.prepare('UPDATE guest_sessions SET verified_day=? WHERE subject=? AND revoked_at IS NULL AND expires_at>?').bind(day,subject,now) + : env.DB.prepare('INSERT INTO guest_sessions(subject,created_at,expires_at,verified_day) VALUES(?,?,?,?)').bind(subject,now,expiresAt,day), + ]); + } catch { + try { + if(await env.DB.prepare('SELECT 1 FROM guest_challenges WHERE token_hash=?').bind(tokenHash).first()) return guestFailure('guest_verification_replayed',403,'This guest check was already used. Complete a new check.'); + } catch { /* Keep storage failures distinct from rejected verification. */ } + return guestFailure('guest_session_unavailable',503,'The guest session could not be saved. Try again shortly.'); + } + return Response.json({ok:true,expiresAt,splitLimit:5,chatLimit:25},{headers:{'Cache-Control':'private, no-store','Set-Cookie':cookie(`${id}.${expiresAt}.${signature}`,Math.floor((expiresAt-now)/1000))}}); +} diff --git a/cloudflare/playwright.config.mjs b/cloudflare/playwright.config.mjs index 43fbf83..9b4cdc8 100644 --- a/cloudflare/playwright.config.mjs +++ b/cloudflare/playwright.config.mjs @@ -1,3 +1,3 @@ import { defineConfig } from '@playwright/test'; -export default defineConfig({ testDir: '.', testMatch: ['browser.spec.mjs', 'remix.spec.mjs', 'layout.spec.mjs', 'account.spec.mjs', 'account-rack.spec.mjs', 'gateway.spec.mjs', 'sharing.spec.mjs', 'playback.spec.mjs', 'classroom.spec.mjs'], workers: 1, timeout: 45000, +export default defineConfig({ testDir: '.', testMatch: ['guest.spec.mjs', 'browser.spec.mjs', 'remix.spec.mjs', 'layout.spec.mjs', 'account.spec.mjs', 'account-rack.spec.mjs', 'gateway.spec.mjs', 'sharing.spec.mjs', 'playback.spec.mjs', 'classroom.spec.mjs'], workers: 1, timeout: 45000, use: { browserName: process.env.STEM_BROWSER || 'chromium', channel: process.env.STEM_BROWSER ? undefined : 'chrome', headless: true, screenshot: 'only-on-failure' }, reporter: 'line' }); diff --git a/cloudflare/retention.ts b/cloudflare/retention.ts index ef1314e..5d198f0 100644 --- a/cloudflare/retention.ts +++ b/cloudflare/retention.ts @@ -1,6 +1,7 @@ export async function purgeExpiredListeningConversations(db: D1Database): Promise { const result = await db.prepare("DELETE FROM listening_conversations WHERE expires_at <= datetime('now')").run(); await db.batch([ + db.prepare("DELETE FROM guest_listening_conversations WHERE expires_at<=datetime('now') OR EXISTS(SELECT 1 FROM guest_sessions s WHERE s.subject=guest_listening_conversations.subject AND (s.revoked_at IS NOT NULL OR s.expires_at<=CAST(strftime('%s','now') AS INTEGER)*1000))"), db.prepare("DELETE FROM course_messages WHERE EXISTS(SELECT 1 FROM course_conversations c WHERE c.job_id=course_messages.job_id AND c.subject=course_messages.subject AND c.expires_at<=datetime('now')) OR NOT EXISTS(SELECT 1 FROM jobs WHERE id=course_messages.job_id AND created_at>datetime('now','-90 days'))"), db.prepare("DELETE FROM course_turns WHERE EXISTS(SELECT 1 FROM course_conversations c WHERE c.job_id=course_turns.job_id AND c.subject=course_turns.subject AND c.expires_at<=datetime('now')) OR NOT EXISTS(SELECT 1 FROM jobs WHERE id=course_turns.job_id AND created_at>datetime('now','-90 days'))"), db.prepare("DELETE FROM course_conversations WHERE expires_at<=datetime('now') OR NOT EXISTS(SELECT 1 FROM jobs WHERE id=course_conversations.job_id AND created_at>datetime('now','-90 days'))"), diff --git a/cloudflare/sso.test.ts b/cloudflare/sso.test.ts index f939f8e..66a678e 100644 --- a/cloudflare/sso.test.ts +++ b/cloudflare/sso.test.ts @@ -209,14 +209,14 @@ test('logout clears browser cookies during identity outages and without an activ for (const client of [undefined, { ...f.client, revoke: async () => { throw new Error('private'); } }]) { const response = await handleAuth(request, { ...f.env, IDENTITY: client }); assert.equal(response.status, 503); - assert.equal(response.headers.getSetCookie().length, 2); + assert.equal(response.headers.getSetCookie().length, 3); assert.ok(response.headers.getSetCookie().every(value => value.includes('Max-Age=0'))); assert.match(await response.text(), /Signed out of this browser/); } const anonymous = await handleAuth(f.request('/auth/logout', { method: 'POST', headers: { Origin: origin } }), { ...f.env, IDENTITY: undefined }); assert.equal(anonymous.status, 303); assert.equal(anonymous.headers.get('location'), '/'); - assert.equal(anonymous.headers.getSetCookie().length, 2); + assert.equal(anonymous.headers.getSetCookie().length, 3); const unsafe = await handleAuth(f.request('/auth/logout', { method: 'POST', headers: { Origin: 'null' } }), f.env); assert.equal(unsafe.status, 403); assert.equal(unsafe.headers.has('set-cookie'), false); diff --git a/cloudflare/sso.ts b/cloudflare/sso.ts index bff9574..b64ab3b 100644 --- a/cloudflare/sso.ts +++ b/cloudflare/sso.ts @@ -3,6 +3,7 @@ import { authFailure } from '../src/identity.ts'; import { CAIL_CANONICAL_ISSUER, loadIdentityVerifierConfig, verifyIdentityJwt } from '@cuny-ai-lab/cail-identity'; import { REQUEST_ID } from './gateway.ts'; import { validClassId } from '../src/classroom/contract.ts'; +import { clearGuestCookie } from './guest.ts'; // Doorway owns CUNY OIDC, one-use PKCE grants, session revocation and Admission. // These RPC capabilities have deployment-pinned audiences and callback hosts. @@ -19,6 +20,8 @@ export interface SsoEnv { PREVIEW_IDENTITY?: WorkerIdentity; REQUEST_LIMIT?: { limit(input: { key: string }): Promise<{ success: boolean }> }; CAIL_IDENTITY_JWKS?: string; + /** Private adapter callback; never sent to Doorway or supplied by a browser. */ + guestSignout?: () => Promise; } export const SESSION_COOKIE = '__Host-stem-session'; export const LOGIN_COOKIE = '__Host-stem-login'; @@ -122,9 +125,10 @@ export async function handleAuth(request: Request, env: SsoEnv): Promise { +export function sanitizedRequest(request: Request): Request { const headers = new Headers(request.headers); const requestId = headers.get('x-cail-request-id') ?? headers.get('x-request-id'); // Even a valid app JWT submitted by the browser is not authority here. for (const name of [...headers.keys()]) if (name === 'cookie' || name === 'authorization' || name.startsWith('x-cail-')) headers.delete(name); if (requestId && REQUEST_ID.test(requestId)) headers.set('x-cail-request-id', requestId); + return new Request(request.url,{method:request.method,headers,body:request.body,redirect:'manual',signal:request.signal}); +} + +export async function authenticatedRequest(request: Request, env: SsoEnv): Promise { + const headers = new Headers(sanitizedRequest(request).headers); if (!publicApi(request)) { if (!['GET', 'HEAD', 'OPTIONS'].includes(request.method) && (request.headers.get('origin') !== env.PUBLIC_BASE_URL || request.headers.get('sec-fetch-site') === 'cross-site')) return denied(403); diff --git a/cloudflare/test-worker.ts b/cloudflare/test-worker.ts index b975614..8820460 100644 --- a/cloudflare/test-worker.ts +++ b/cloudflare/test-worker.ts @@ -60,7 +60,9 @@ export default { const outbound = new Request(input, init); if (new URL(outbound.url).pathname.endsWith('/quota')) return Response.json({ object: 'quota', managed_by: 'cloudflare', state: 'estimated', unit: 'microdollar', currency: 'USD', limit: 1000000, estimated_used: 100000, estimated_remaining: 900000, used_percent: 10, remaining_percent: 90, window_seconds: 86400, window_technique: 'sliding', calculated_at: Math.floor(Date.now() / 1000) }); gatewayCalls++; - if (outbound.headers.get('x-cail-identity-jwt') !== gatewayJwt || outbound.headers.has('authorization')) throw new Error('fixture credential contract'); + if (env.GUEST_ENABLED === 'true' && outbound.headers.get('authorization') === `Bearer ${env.GUEST_GATEWAY_API_KEY}`) { + if(outbound.headers.has('x-cail-identity-jwt'))throw new Error('fixture mixed credential contract'); + } else if (outbound.headers.get('x-cail-identity-jwt') !== gatewayJwt || outbound.headers.has('authorization')) throw new Error('fixture credential contract'); const body = await outbound.json<{ model: string }>(); if (body.model !== 'glm-5.2') throw new Error('fixture model contract'); if ('messages' in body && Array.isArray((body as { messages?: unknown }).messages)) gatewayMessages = (body as { messages: Array<{ role: string; content: string }> }).messages; @@ -68,7 +70,7 @@ export default { const requestId = '01900000-0000-7000-8000-000000000001'; const error = { error: { code: 'quota_exceeded', type: 'quota_exceeded', param: null, message: 'private fixture', cail: { request_id: requestId, should_retry: false } } }; if (gatewayMode === 'quota') return Response.json(error, { status: 429, headers: { 'x-should-retry': 'false', 'x-request-id': requestId } }); - const chunks = [{ choices: [{ delta: { content: 'Listen for the bass against the drums.' }, finish_reason: null }] }, { choices: [{ delta: {}, finish_reason: 'stop' }] }, + const chunks = [{ choices: [{ delta: { content: gatewayMode==='empty'?'':'Listen for the bass against the drums.' }, finish_reason: null }] }, { choices: [{ delta: {}, finish_reason: 'stop' }] }, gatewayMode === 'trailing-error' ? error : { choices: [], usage: { total_tokens: 20 } }]; return new Response(chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join('') + 'data: [DONE]\n\n', { headers: { 'Content-Type': 'text/event-stream', 'x-request-id': requestId } }); } } as Fetcher, diff --git a/cloudflare/worker.ts b/cloudflare/worker.ts index 6503f74..60aee58 100644 --- a/cloudflare/worker.ts +++ b/cloudflare/worker.ts @@ -5,7 +5,9 @@ import app, { runReliableJobs } from '../src/index.ts'; import type { Env } from '../src/env.ts'; import { verifyCailIdentity } from './verify.ts'; import { gatewayForRequest, canonicalModel, gatewayModelsConfigured } from './gateway.ts'; -import { authenticatedRequest, handleAuth, type WorkerIdentity } from './sso.ts'; +import { authenticatedRequest, handleAuth, sanitizedRequest, publicApi, type WorkerIdentity } from './sso.ts'; +import { guestConfigured, hasMemberCookie, readGuestSession, revokeGuestSession, startGuestSession, clearGuestCookie, GUEST_COOKIE } from './guest.ts'; +import { guestFailure } from '../src/guest/access.ts'; import { purgeExpiredListeningConversations } from './retention.ts'; import { purgeOperationContent } from '../src/reliability/retention.ts'; export type WorkerEnv = Omit & @@ -34,15 +36,17 @@ async function serveApi(request: Request, env: WorkerEnv, ctx: ExecutionContext) ctx.waitUntil(recordServerUse(env,request,response,undefined,Date.now()));return response; } } - const gateway = gatewayForRequest(env.GATEWAY, request.headers.get('x-cail-gateway-identity-jwt'), request, env.RELEASE || 'candidate'); + const guest = Boolean(env.guestSession); + const gateway = gatewayForRequest(env.GATEWAY, guest ? env.GUEST_GATEWAY_API_KEY ?? null : request.headers.get('x-cail-gateway-identity-jwt'), request, env.RELEASE || 'candidate',guest?'key':'jwt'); return app.fetch(request, { ...env, ASSISTANT_MODEL: env.GATEWAY_MODEL, ASSISTANT_FALLBACK_MODELS: env.GATEWAY_FALLBACK_MODEL ?? '', - assistantTransport: gateway.stream, assistantQuota: gateway.quota, verifyCailIdentity, validateStemAudio }, ctx); + assistantTransport: gateway.stream, assistantQuota: guest ? undefined : gateway.quota, verifyCailIdentity, validateStemAudio }, ctx); } export default { async scheduled(_controller: ScheduledController, env: WorkerEnv, ctx: ExecutionContext): Promise { - ctx.waitUntil(runReliableJobs({...env,validateStemAudio})); + ctx.waitUntil(runReliableJobs({...env,guestRuntimeReady:guestConfigured(env),validateStemAudio})); + ctx.waitUntil(env.DB.prepare('DELETE FROM guest_challenges WHERE expires_at { @@ -53,6 +57,7 @@ export default { if (env.CANONICAL_BASE_URL && url.origin === env.CANONICAL_BASE_URL) { env = { ...env, PUBLIC_BASE_URL: env.CANONICAL_BASE_URL }; } + env = {...env,guestRuntimeReady:guestConfigured(env),guestStartAllowed:!hasMemberCookie(request),guestSession:undefined}; let response: Response; try { if (url.pathname === '/healthz') { @@ -64,12 +69,22 @@ export default { transport: 'cail-gateway', fallbackConfigured: gatewayModelsConfigured(env.GATEWAY_MODEL,env.GATEWAY_FALLBACK_MODEL) } }); } else if (env.AUTH_MODE !== 'cail') { response = Response.json({ error: 'Service configuration is incomplete.' }, { status: 503 }); + } else if (url.pathname === '/auth/guest') { + response = await startGuestSession(request,env); } else if (url.pathname.startsWith('/auth/')) { - response = await handleAuth(request, env); + response = await handleAuth(request, {...env,guestSignout:()=>revokeGuestSession(request,env)}); } else if (url.pathname.startsWith('/api/')) { // Admission and ownership run in the shared application. Limit expensive // ingress before provider work; signed provider callbacks are independent. - const internal = await authenticatedRequest(request, env); + let internal: Request | Response; + if (!publicApi(request) && !hasMemberCookie(request) && env.guestRuntimeReady) { + const session=await readGuestSession(request,env); + if (session) {env={...env,guestSession:session};internal=sanitizedRequest(request);} + else if ((request.headers.get('cookie') ?? '').split(';').some(part=>part.trim().split('=',1)[0]===GUEST_COOKIE)) { + internal=guestFailure('guest_session_expired',401,'This guest session has ended. Start a new guest session or sign in.'); + internal.headers.append('Set-Cookie',clearGuestCookie()); + } else internal=await authenticatedRequest(request,env); + } else internal=await authenticatedRequest(request,env); if(internal instanceof Response)ctx.waitUntil(recordServerUse(env,request,internal,undefined,Date.now())); response = internal instanceof Response ? internal : await serveApi(internal, { ...env, CAIL_LOGIN_URL: '/auth/login' }, ctx); } else { @@ -82,6 +97,10 @@ export default { } const headers = new Headers(response.headers); for (const [key, value] of Object.entries(HEADERS)) headers.set(key, value); + if (env.guestRuntimeReady) headers.set('Content-Security-Policy',HEADERS['Content-Security-Policy'] + .replace("script-src 'self';","script-src 'self' https://challenges.cloudflare.com;") + .replace("connect-src 'self';","connect-src 'self' https://challenges.cloudflare.com;") + + '; frame-src https://challenges.cloudflare.com'); if (url.pathname.startsWith('/auth/')) headers.set('Referrer-Policy', 'no-referrer'); if (url.pathname.startsWith('/api/') || url.pathname.startsWith('/auth/') || url.pathname === '/healthz') headers.set('Cache-Control', 'private, no-store'); return new Response(response.body, { status: response.status, headers }); diff --git a/docs/guest-access.md b/docs/guest-access.md new file mode 100644 index 0000000..e284e6e --- /dev/null +++ b/docs/guest-access.md @@ -0,0 +1,54 @@ +# Guest access candidate + +The October 6 policy is **5 successful splits and 25 human Listening Guy inputs per guest session per UTC day**. It supersedes the September 29 proposal for three visitor attempts. Pending operations hold a place; failed/cancelled splits and replies with no usable output release it. Usable partial replies count once. Guide generation and cache reads do not consume human-input allowance. Provider spend remains separate from these app counts. + +This implementation is **off by default and has not been activated**. No sponsor grant, API key, Turnstile widget, secret, production migration, or deployment was created. No old OpenRouter key is used. Production acceptance still requires the release gates below. + +## Authority and privacy + +`POST /auth/guest` requires an exact same-origin HTTPS request and a server-verified Turnstile token for the request hostname and `stem_guest` action. Token age is bounded to five minutes. A short-lived hash record rejects token replay in an atomic D1 batch even if a verification response is repeated. The browser receives an HMAC-signed `__Host-stem-guest` cookie with Secure, HttpOnly, SameSite=Lax, a fixed seven-day expiry, and an origin-bound signature. D1 expiry/revocation is rechecked on every protected request. A fresh human check is required on each UTC day before paid starts; renewal preserves the session and its counters. + +The guest subject is a server-generated random identifier. It is never a CAIL subject, email, IP address, or admission grant. `guest_sessions`, `guest_job_owners`, `guest_upload_owners`, and `guest_listening_conversations` are separate from member ownership. The operational ledger uses an immutable `quota_class` and foreign keys to the appropriate member or guest principal. That accounting provenance grants no access. Delayed paid starts recheck the guest session, or the existing member/course authority, according to the stored quota class. + +Guests can upload/import and split their own audio, play/download their own results, annotate them, and use Listening Guy for those results. A fixed route allowlist and live ownership checks deny member/private jobs, other guests' work, course folders, rosters, prompts, transcripts, instructor/admin functions, public-link creation, and sponsor quota reads. Existing explicitly shared audio-only links retain their existing public projection. App administrators do not gain guest content through member ownership APIs. + +A member session cookie always selects the existing CUNY authentication path, including malformed, expired, revoked, or unavailable sessions. Such a request never silently becomes a guest. A successful CUNY login revokes the browser's guest session and clears its cookie; no guest content is adopted into a member account or course. Logout revokes the guest server session before clearing the cookie. A copied cookie cannot reopen a revoked session. Session revalidation conceals private UI while authority is pending or unavailable; a changed identity clears and reloads it. + +Guest access lasts seven days in the same browser. Clearing the cookie or ending the session removes access to its saved work; it is not an account-recovery mechanism. Existing uploaded-audio/stem storage retains its 90-day deletion boundary. The daily retention job removes guest transcripts after session expiry/revocation. No member or course content is backfilled or reclassified. + +## Abuse boundaries + +Anonymous per-human limits cannot be exact. These counts identify the server-issued browser session; cookie resets can create another identity after a new valid human check. The existing Cloudflare ingress binding allows 30 requests/minute per key, with a separate `stem-guest-start` key derived from the trusted ingress IP for guest entry. This is best-effort reset-abuse control, not a unique-human or daily person guarantee. A rejected start performs no database, Turnstile or provider call and creates no cookie. No IP is used as content ownership or persisted as an account identifier. No global rate-limit setting changes are included. + +The atomic ledger additionally permits at most two active splits per guest, ten active guest splits across the app, fifteen split attempts per guest/day, fifty chat attempts per guest/day, and ten guide attempts per guest/day. Uploads allow at most three outstanding grants and fifteen grants per guest/day, with one-hour upload expiry. These are abuse and concurrency guards, not successful-use charges or a promised monetary ceiling. Existing provider backoff, uncertain-start reconciliation, idempotency, size/rights checks, and the global bounded queue remain in force. + +## Gateway and activation + +The installed `cail-client` 6.2.2 supports a registry-managed application key via `{kind:'key',token}`. A verified guest request can use only the explicit optional `GUEST_GATEWAY_API_KEY` on the existing private Gateway binding. Gateway reauthorizes that principal through its Registry on every model request. No guest JWT is minted, no member identity is reused, and no persistent provider credential reaches the browser. The guest cannot inspect the sponsor's quota. Signed-in requests retain their current exact app/Gateway identity pair. + +Guest readiness requires all of the following. Missing configuration leaves the feature unavailable, including when an unrelated legacy provider key exists. + +| Setting/capability | Purpose | +| --- | --- | +| `GUEST_ENABLED=true` | Explicit activation, absent by default | +| `GUEST_COOKIE_SECRET` | Dedicated app-local cookie signing secret, at least 32 characters | +| `GUEST_TURNSTILE_SITE_KEY` and `GUEST_TURNSTILE_SECRET` | Widget for the reviewed exact application hostnames; server verification | +| `GUEST_GATEWAY_API_KEY` | Approved Registry-managed sponsor application key; no fallback credential | +| Existing `GATEWAY`, exact primary/fallback models, and `REQUEST_LIMIT` | Private authorized inference and ingress control | + +Only the widget's site key is public. When configuration is complete, the CSP adds `https://challenges.cloudflare.com` to script/connect/frame sources for the widget. Turnstile follows its [server-side validation contract](https://developers.cloudflare.com/turnstile/get-started/server-side-validation/); the client callback alone grants nothing. Its documented [compact widget size](https://developers.cloudflare.com/turnstile/get-started/client-side-rendering/widget-configurations/#widget-sizes) fits narrow mobile layouts; real widget acceptance remains a release gate. + +Before activation: review and approve the sponsor's grant/budget/model entitlement, dedicated cookie key installation, exact-host Turnstile widget and secret, and CSP/abuse settings. Then apply the reviewed unshipped migration `0022-reliable-operations.sql` with the existing backup/drain/rollback plan and complete real browser/Turnstile/Registry acceptance. Source includes guest tables and accounting provenance in that same migration; no fourth PR or shared Gateway/Admission/SSO deployment is required. A database already initialized with an earlier *unreleased draft* of 0022 needs a reviewed rebuild/upgrade plan; `CREATE TABLE IF NOT EXISTS` does not upgrade that draft's old operation schema. + +Local tests use fixture credentials and mocked Turnstile, Replicate, and Gateway. They establish app behavior and privacy boundaries, not real bot resistance, sponsor entitlement, paid model output, production cost, or live deployment readiness. + +## Local verification + +Using Node 22.23.1 and the existing installed dependencies (no lockfile changes): + +- Root, server and Cloudflare TypeScript checks pass; the guest/app/account scripts pass syntax checks. +- Shared Worker regression suite: 325/325; server/SQLite compatibility suite: 42/42. +- Cloudflare suite: 78/78, including four guest tests. It covers empty-reply/failed-split release, 5/25 settlement, idempotency, atomic parallel reservation, daily reset/proof renewal, immutable quota provenance, rate-limit rejection with no side effects, cookie expiry/tamper/noncanonical encoding, unavailable storage, revocation, and cross-guest/member/course isolation. +- `cloudflare/guest.spec.mjs` passes in Chrome, Firefox and WebKit against the real local HTTPS Worker listener. The test uses browser file upload, secure-cookie resumption, account limits, pending/unavailable-authority concealment, second-guest clearing, logout, and 320/390/768/1280px overflow checks. Only external bot/provider responses are fixture-controlled. + +Run the Cloudflare suite from `cloudflare/` with `node --import ./test-wasm-register.mjs --import ../node_modules/tsx/dist/loader.mjs --test *.test.ts`. Run the guest browser spec from the repository root with `node ./node_modules/@playwright/test/cli.js test --config cloudflare/playwright.config.mjs guest.spec.mjs`; set `STEM_BROWSER=firefox` or `STEM_BROWSER=webkit` for the other engines. Local Workers/HTTPS need loopback permission. Final PR integration still needs its own exact-head audit and CI. diff --git a/migrations/0022-reliable-operations.sql b/migrations/0022-reliable-operations.sql index 8cf85c6..5395c95 100644 --- a/migrations/0022-reliable-operations.sql +++ b/migrations/0022-reliable-operations.sql @@ -1,9 +1,80 @@ -- Deliberately separate from historical app_request_reservations. No inferred -- successful charges/backfill. Apply only with the queue-drain rollback plan. +-- Guest authority is app-local. Never insert anonymous identities into app_users. +CREATE TABLE IF NOT EXISTS guest_sessions ( + subject TEXT PRIMARY KEY CHECK(length(subject)=70 AND subject GLOB 'guest-*'), + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL CHECK(expires_at>created_at AND expires_at<=created_at+604800000), + verified_day TEXT NOT NULL, + revoked_at INTEGER +); +CREATE TRIGGER IF NOT EXISTS guest_session_identity_fence BEFORE UPDATE ON guest_sessions +WHEN NEW.subject<>OLD.subject OR NEW.created_at<>OLD.created_at OR NEW.expires_at<>OLD.expires_at + OR (OLD.revoked_at IS NOT NULL AND NEW.revoked_at IS NOT OLD.revoked_at) +BEGIN SELECT RAISE(ABORT,'immutable guest session'); END; +CREATE TABLE IF NOT EXISTS guest_challenges ( + token_hash TEXT PRIMARY KEY, + expires_at INTEGER NOT NULL +); +CREATE TABLE IF NOT EXISTS guest_job_owners ( + job_id TEXT PRIMARY KEY REFERENCES jobs(id) ON DELETE CASCADE, + subject TEXT NOT NULL REFERENCES guest_sessions(subject) +); +CREATE INDEX IF NOT EXISTS guest_job_owners_subject ON guest_job_owners(subject); +CREATE TRIGGER IF NOT EXISTS guest_job_separation BEFORE INSERT ON guest_job_owners +WHEN EXISTS(SELECT 1 FROM job_owners WHERE job_id=NEW.job_id) OR EXISTS(SELECT 1 FROM job_courses WHERE job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'guest work cannot be course or member work'); END; +CREATE TRIGGER IF NOT EXISTS member_job_separation BEFORE INSERT ON job_owners +WHEN EXISTS(SELECT 1 FROM guest_job_owners WHERE job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'guest work cannot be member work'); END; +CREATE TRIGGER IF NOT EXISTS guest_course_separation BEFORE INSERT ON job_courses +WHEN EXISTS(SELECT 1 FROM guest_job_owners WHERE job_id=NEW.job_id) +BEGIN SELECT RAISE(ABORT,'guest work cannot be course work'); END; + +CREATE TABLE IF NOT EXISTS guest_upload_owners ( + object_key TEXT PRIMARY KEY, + subject TEXT NOT NULL REFERENCES guest_sessions(subject), + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + state TEXT NOT NULL DEFAULT 'issued' CHECK(state IN ('issued','uploading','ready')) +); +CREATE TABLE IF NOT EXISTS guest_listening_conversations ( + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + subject TEXT NOT NULL REFERENCES guest_sessions(subject), + entries TEXT NOT NULL CHECK(json_valid(entries) AND json_type(entries)='array'), + revision INTEGER NOT NULL CHECK(revision>=1), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + expires_at TEXT NOT NULL, + PRIMARY KEY(job_id,subject) +); +-- Accounting provenance is not an access grant. Live session/Admission checks +-- still run before requests and delayed paid starts. Keep member/guest FKs separate. +CREATE TABLE IF NOT EXISTS operation_principals ( + subject TEXT PRIMARY KEY, + quota_class TEXT NOT NULL CHECK(quota_class IN ('member','guest')), + member_subject TEXT UNIQUE REFERENCES app_users(subject), + guest_subject TEXT UNIQUE REFERENCES guest_sessions(subject), + UNIQUE(subject,quota_class), + CHECK((quota_class='member' AND member_subject IS NOT NULL AND member_subject=subject AND guest_subject IS NULL) + OR (quota_class='guest' AND guest_subject IS NOT NULL AND guest_subject=subject AND member_subject IS NULL)) +); +INSERT OR IGNORE INTO operation_principals(subject,quota_class,member_subject) + SELECT subject,'member',subject FROM app_users; +CREATE TRIGGER IF NOT EXISTS member_operation_principal AFTER INSERT ON app_users BEGIN + INSERT INTO operation_principals(subject,quota_class,member_subject) VALUES(NEW.subject,'member',NEW.subject); +END; +CREATE TRIGGER IF NOT EXISTS guest_operation_principal AFTER INSERT ON guest_sessions BEGIN + INSERT INTO operation_principals(subject,quota_class,guest_subject) VALUES(NEW.subject,'guest',NEW.subject); +END; +CREATE TRIGGER IF NOT EXISTS operation_principal_immutable BEFORE UPDATE ON operation_principals +BEGIN SELECT RAISE(ABORT,'immutable operation principal'); END; + CREATE TABLE IF NOT EXISTS app_operations ( id TEXT PRIMARY KEY, - subject TEXT NOT NULL REFERENCES app_users(subject), - course_id TEXT, + subject TEXT NOT NULL, + quota_class TEXT NOT NULL DEFAULT 'member' CHECK(quota_class IN ('member','guest')), + course_id TEXT CHECK(quota_class<>'guest' OR course_id IS NULL), kind TEXT NOT NULL CHECK(kind IN ('split','chat','guide')), idempotency_key TEXT NOT NULL, fingerprint TEXT NOT NULL, @@ -23,7 +94,8 @@ CREATE TABLE IF NOT EXISTS app_operations ( error_code TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, - UNIQUE(subject,kind,idempotency_key) + UNIQUE(subject,kind,idempotency_key), + FOREIGN KEY(subject,quota_class) REFERENCES operation_principals(subject,quota_class) ); CREATE INDEX IF NOT EXISTS app_operations_allowance ON app_operations(subject,kind,day,state); CREATE INDEX IF NOT EXISTS app_operations_queue ON app_operations(kind,phase,state,not_before,created_at); @@ -60,7 +132,7 @@ CREATE TRIGGER IF NOT EXISTS operation_terminal_fence BEFORE UPDATE ON app_opera WHEN OLD.state IN ('succeeded','partial','failed','cancelled') AND NEW.state<>OLD.state BEGIN SELECT RAISE(ABORT,'terminal operation'); END; CREATE TRIGGER IF NOT EXISTS operation_identity_fence BEFORE UPDATE ON app_operations -WHEN NEW.subject<>OLD.subject OR NEW.kind<>OLD.kind OR NEW.day<>OLD.day OR +WHEN NEW.subject<>OLD.subject OR NEW.quota_class<>OLD.quota_class OR NEW.kind<>OLD.kind OR NEW.day<>OLD.day OR NEW.fingerprint<>OLD.fingerprint OR NEW.idempotency_key<>OLD.idempotency_key OR NEW.course_id IS NOT OLD.course_id BEGIN SELECT RAISE(ABORT,'immutable operation identity'); END; CREATE TABLE IF NOT EXISTS operation_attempts ( diff --git a/public/account.html b/public/account.html index bdd3b1e..5d81b36 100644 --- a/public/account.html +++ b/public/account.html @@ -18,7 +18,7 @@

My account

Daily split allowance unavailable.

Model usage unavailable.

- +