diff --git a/.github/issue-evidence/pr-370/prompt-idempotency.png b/.github/issue-evidence/pr-370/prompt-idempotency.png new file mode 100644 index 00000000..b7887704 Binary files /dev/null and b/.github/issue-evidence/pr-370/prompt-idempotency.png differ diff --git a/tests/web/app-render.test.ts b/tests/web/app-render.test.ts index f7aeb4a4..cb574310 100644 --- a/tests/web/app-render.test.ts +++ b/tests/web/app-render.test.ts @@ -1,8 +1,15 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import test from "node:test"; +import test, { afterEach } from "node:test"; import vm from "node:vm"; +const appDisposers = new Set<() => Promise>(); + +afterEach(async () => { + await Promise.all([...appDisposers].map((dispose) => dispose())); + appDisposers.clear(); +}); + /** * Executes web/ui/app.js in a stubbed DOM and feeds it a representative * session snapshot. Source-text regex assertions in web-host.test.ts cannot @@ -294,6 +301,10 @@ async function renderApp( setItem: (key: string, value: string) => stored.set(key, value), }; const replaced: string[] = []; + const windowListeners = new Map< + string, + Array<(event?: Record) => void> + >(); let eventFetches = 0; let snapshotFetches = 0; let readerCancellations = 0; @@ -323,6 +334,7 @@ async function renderApp( eventFetches++; const connectionEvents = eventFetches === 1 ? encodedEvents : []; let index = 0; + let resolveRead: ((value: { done: boolean }) => void) | undefined; return { ok: true, status: 200, @@ -330,6 +342,7 @@ async function renderApp( getReader: () => ({ cancel: async () => { readerCancellations++; + resolveRead?.({ done: true }); }, read: () => index < connectionEvents.length @@ -337,7 +350,9 @@ async function renderApp( done: false, value: connectionEvents[index++], }) - : new Promise(() => {}), + : new Promise((resolve) => { + resolveRead = resolve; + }), }), }, }; @@ -359,6 +374,7 @@ async function renderApp( URL, TextDecoder, TextEncoder, + AbortController, Element: class Element {}, }; context.window = { @@ -368,6 +384,15 @@ async function renderApp( clearInterval, setTimeout, clearTimeout, + addEventListener( + type: string, + listener: (event?: Record) => void, + ) { + windowListeners.set(type, [ + ...(windowListeners.get(type) ?? []), + listener, + ]); + }, matchMedia: () => systemTheme, }; context.globalThis = context; @@ -386,6 +411,17 @@ async function renderApp( ); vm.runInContext(source, context as vm.Context, { filename: "app.js" }); await new Promise((resolve) => setTimeout(resolve, 200)); + const dispatchWindowEvent = ( + type: string, + event?: Record, + ) => { + for (const listener of windowListeners.get(type) ?? []) listener(event); + }; + const dispose = async () => { + dispatchWindowEvent("pagehide", { persisted: false }); + await new Promise((resolve) => setTimeout(resolve, 0)); + }; + appDisposers.add(dispose); return { elements, stored, @@ -418,6 +454,17 @@ async function renderApp( "sendPrompt", context as vm.Context, ) as () => Promise, + api: vm.runInContext("api", context as vm.Context) as ( + path: string, + options?: Record, + ) => Promise, + readEventChunk: vm.runInContext( + "readEventChunk", + context as vm.Context, + ) as ( + reader: { read(): Promise }, + timeoutMs?: number, + ) => Promise, cancelActiveTurn: vm.runInContext( "cancelActiveTurn", context as vm.Context, @@ -433,6 +480,15 @@ async function renderApp( resetCursor?: boolean; epoch?: number; }) => Promise, + dispose, + suspendForPageCache: async () => { + dispatchWindowEvent("pagehide", { persisted: true }); + await new Promise((resolve) => setTimeout(resolve, 0)); + }, + resumeFromPageCache: async () => { + dispatchWindowEvent("pageshow", { persisted: true }); + await new Promise((resolve) => setTimeout(resolve, 0)); + }, }; } @@ -515,6 +571,189 @@ test("app.js resnapshots on an SSE cursor gap", async () => { assert.ok(app.readerCancellations() >= 1); }); +test("app.js uses quiet-stream heartbeats for bounded snapshot recovery", async () => { + const heartbeat = ": heartbeat\n\n"; + const app = await renderApp({ + eventRecords: [heartbeat.repeat(4)], + }); + assert.equal(app.eventFetches(), 1); + assert.ok(app.snapshotFetches() >= 2); + assert.equal(app.state.cursor, SNAPSHOT.cursor); +}); + +test("app.js keeps its one SSE loop across a back-forward cache restore", async () => { + const app = await renderApp(); + assert.equal(app.eventFetches(), 1); + await app.suspendForPageCache(); + await app.resumeFromPageCache(); + assert.equal(app.eventFetches(), 1); +}); + +test("app.js bounds API waits and explains duplicate prompt admission", async () => { + const app = await renderApp(); + app.context.fetch = async ( + _url: unknown, + options?: { signal?: AbortSignal }, + ) => + new Promise((_resolve, reject) => { + options?.signal?.addEventListener("abort", () => { + const error = new Error("aborted"); + error.name = "AbortError"; + reject(error); + }); + }); + await assert.rejects( + app.api("/api/stuck", { timeoutMs: 5, timeoutMessage: "bounded timeout" }), + /bounded timeout/u, + ); + await assert.rejects( + app.readEventChunk({ read: () => new Promise(() => {}) }, 5), + /event stream stalled/u, + ); + + app.state.promptAdmissionPending = true; + const input = app.elements.get("prompt-input"); + assert.ok(input); + input.value = "another message"; + await app.sendPrompt(); + assert.equal( + app.elements.get("composer-hint")?.textContent, + "OpenPI is still accepting the previous message.", + ); +}); + +test("app.js retries a timed-out admission with the exact same command", async () => { + const app = await renderApp(); + const input = app.elements.get("prompt-input"); + assert.ok(input); + const requests: Array> = []; + ( + app.context.window as { + setTimeout( + callback: () => void, + delay: number, + ): ReturnType; + } + ).setTimeout = (callback: () => void, delay: number) => + setTimeout(callback, delay === 30_000 ? 1 : delay); + app.context.fetch = ( + url: unknown, + options?: { + body?: string; + signal?: AbortSignal; + }, + ) => { + if (String(url) === "/api/prompt") { + requests.push(JSON.parse(options?.body || "{}")); + if (requests.length === 1) { + return new Promise((_resolve, reject) => { + options?.signal?.addEventListener("abort", () => { + const error = new Error("aborted"); + error.name = "AbortError"; + reject(error); + }); + }); + } + return Promise.resolve( + response({ id: requests[0]?.commandId, accepted: true }), + ); + } + if (String(url).startsWith("/api/snapshot")) + return Promise.resolve(response(SNAPSHOT)); + throw new Error(`unexpected request: ${String(url)}`); + }; + + input.value = "admit this once"; + await app.sendPrompt(); + assert.equal(app.state.promptAdmissionPending, false); + assert.ok(app.state.promptAdmission); + + await app.sendPrompt(); + assert.equal(requests.length, 2); + assert.deepEqual(requests[0], { + sessionId: "s1", + content: "admit this once", + commandId: requests[0]?.commandId, + retry: false, + }); + assert.deepEqual(requests[1], { + ...requests[0], + retry: true, + }); + assert.equal(app.state.promptAdmission, null); +}); + +test("app.js preserves an uncertain transport admission without resetting an active turn", async () => { + const app = await renderApp(); + const input = app.elements.get("prompt-input"); + assert.ok(input); + const requests: Array> = []; + app.context.fetch = (url: unknown, options?: { body?: string }) => { + if (String(url) === "/api/prompt") { + requests.push(JSON.parse(options?.body || "{}")); + if (requests.length === 1) + return Promise.reject(new TypeError("network dropped")); + return Promise.resolve( + response({ id: requests[0]?.commandId, accepted: true }), + ); + } + if (String(url).startsWith("/api/snapshot")) + return Promise.resolve(response(SNAPSHOT)); + throw new Error(`unexpected request: ${String(url)}`); + }; + app.state.liveRunning = true; + app.state.livePhase = "running"; + input.value = "do not duplicate this"; + + await app.sendPrompt(); + assert.equal(app.state.livePhase, "running"); + assert.ok(app.state.promptAdmission); + + await app.sendPrompt(); + assert.equal(requests.length, 2); + assert.equal(requests[0]?.commandId, requests[1]?.commandId); + assert.equal(requests[0]?.retry, false); + assert.equal(requests[1]?.retry, true); +}); + +test("app.js starts a new attempt after admission capacity rejects before dispatch", async () => { + const app = await renderApp(); + const input = app.elements.get("prompt-input"); + assert.ok(input); + const requests: Array> = []; + app.context.fetch = (url: unknown, options?: { body?: string }) => { + if (String(url) === "/api/prompt") { + requests.push(JSON.parse(options?.body || "{}")); + if (requests.length === 1) { + return Promise.resolve({ + ok: false, + status: 503, + json: async () => ({ + code: "PROMPT_ADMISSION_CAPACITY", + error: "prompt admission capacity is full", + }), + }); + } + return Promise.resolve( + response({ id: requests[1]?.commandId, accepted: true }), + ); + } + if (String(url).startsWith("/api/snapshot")) + return Promise.resolve(response(SNAPSHOT)); + throw new Error(`unexpected request: ${String(url)}`); + }; + input.value = "try after capacity opens"; + + await app.sendPrompt(); + assert.equal(app.state.promptAdmission, null); + await app.sendPrompt(); + + assert.equal(requests.length, 2); + assert.notEqual(requests[0]?.commandId, requests[1]?.commandId); + assert.equal(requests[0]?.retry, false); + assert.equal(requests[1]?.retry, false); +}); + test("app.js invalidates snapshots for cross-tab session metadata events", async () => { const app = await renderApp({ eventRecords: [ @@ -1154,7 +1393,12 @@ test("app.js accepts an unbound snapshot and preserves a chosen workspace throug activated.selectedSession.cwd = chosenPath; let currentSnapshot: SnapshotFixture = chosen; let sessionCreations = 0; - const prompts: Array<{ sessionId: string; content: string }> = []; + const prompts: Array<{ + sessionId: string; + content: string; + commandId?: string; + retry?: boolean; + }> = []; app.context.fetch = async (url: unknown, options?: { body?: string }) => { if (String(url) === "/api/workspaces/select") { return response({ cancelled: false, path: chosenPath }); @@ -1193,7 +1437,13 @@ test("app.js accepts an unbound snapshot and preserves a chosen workspace throug await new Promise((resolve) => setTimeout(resolve, 0)); } assert.equal(sessionCreations, 1); - assert.deepEqual(prompts, [{ sessionId: "s1", content: "first task" }]); + assert.equal(prompts.length, 1); + assert.deepEqual(prompts[0], { + sessionId: "s1", + content: "first task", + commandId: prompts[0]?.commandId, + retry: false, + }); assert.equal(app.state.selectedWorkspace, chosenPath); assert.equal(app.state.selectedPath, "/tmp/s1.jsonl"); assert.equal(input.value, ""); diff --git a/tests/web/web-host.test.ts b/tests/web/web-host.test.ts index f0f9f862..febbd3f9 100644 --- a/tests/web/web-host.test.ts +++ b/tests/web/web-host.test.ts @@ -1007,6 +1007,63 @@ test("keeps unexpected Web Host failures classified as server errors", async () } }); +test("quiet SSE clients receive heartbeats without advancing the event cursor", { + timeout: 2_000, +}, async () => { + const cwd = await mkdtemp(join(tmpdir(), "openpi-web-heartbeat-")); + const host = new WebHost({ + runtime: testRuntime(cwd), + sseHeartbeatMs: 10, + }); + let request: ReturnType | undefined; + try { + await host.start(); + const launched = new URL(host.url); + const token = new URLSearchParams(launched.hash.slice(1)).get("token"); + assert.ok(token); + const headers = { Authorization: `Bearer ${token}` }; + const before = (await ( + await fetch(`${launched.origin}/api/snapshot`, { headers }) + ).json()) as { cursor: number }; + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error("timed out waiting for an SSE heartbeat")), + 1_000, + ); + request = httpRequest( + { + hostname: launched.hostname, + port: Number(launched.port), + path: `/events?cursor=${before.cursor}`, + headers, + }, + (response) => { + assert.equal(response.statusCode, 200); + response.setEncoding("utf8"); + let received = ""; + response.on("data", (chunk: string) => { + received += chunk; + if (!received.includes(": heartbeat\n\n")) return; + clearTimeout(timeout); + resolve(); + }); + response.once("error", reject); + }, + ); + request.once("error", reject); + request.end(); + }); + const after = (await ( + await fetch(`${launched.origin}/api/snapshot`, { headers }) + ).json()) as { cursor: number }; + assert.equal(after.cursor, before.cursor); + } finally { + request?.destroy(); + await host.stop(); + await rm(cwd, { recursive: true, force: true }); + } +}); + test("adapter initialization fails before the Host starts listening", async () => { const cwd = await mkdtemp(join(tmpdir(), "openpi-web-startup-failure-")); const runtime = testRuntime(cwd); @@ -1075,7 +1132,9 @@ test("rejects prompt admission with the runtime's typed receipt", async () => { "PROMPT_REJECTED", 422, ); + let sendCalls = 0; const runtime = testRuntime(cwd, async () => { + sendCalls++; throw rejection; }); const { host, launched, headers } = await startTestHost(runtime); @@ -1086,6 +1145,8 @@ test("rejects prompt admission with the runtime's typed receipt", async () => { body: JSON.stringify({ sessionId: runtime.sessionManager.getSessionId(), content: "reject me", + commandId: "rejected-admission", + retry: false, }), }); assert.equal(response.status, 422); @@ -1093,7 +1154,187 @@ test("rejects prompt admission with the runtime's typed receipt", async () => { code: "PROMPT_REJECTED", error: "Pi rejected this prompt", }); + const replay = await fetch(`${launched.origin}/api/prompt`, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionId: runtime.sessionManager.getSessionId(), + content: "reject me", + commandId: "rejected-admission", + retry: true, + }), + }); + assert.equal(replay.status, 422); + assert.deepEqual(await replay.json(), { + code: "PROMPT_REJECTED", + error: "Pi rejected this prompt", + }); + assert.equal(sendCalls, 1); + } finally { + await host.stop(); + await rm(cwd, { recursive: true, force: true }); + } +}); + +test("replays one prompt admission after a browser timeout", async () => { + const cwd = await mkdtemp(join(tmpdir(), "openpi-web-prompt-retry-")); + let sendCalls = 0; + let releaseAdmission!: () => void; + const admitted = new Promise((resolve) => { + releaseAdmission = resolve; + }); + let runtimePendingFollowUps = 2; + const runtime = testRuntime(cwd, async () => { + sendCalls++; + await admitted; + const receipt = { pendingFollowUps: runtimePendingFollowUps }; + runtimePendingFollowUps = 0; + return receipt; + }); + const { host, launched, headers } = await startTestHost(runtime); + const commandId = "browser-timeout-retry"; + const prompt = { + sessionId: runtime.sessionManager.getSessionId(), + content: "send this exactly once", + commandId, + }; + try { + const abort = new AbortController(); + const timedOut = fetch(`${launched.origin}/api/prompt`, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ ...prompt, retry: false }), + signal: abort.signal, + }); + while (sendCalls === 0) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + abort.abort(); + await assert.rejects(timedOut, /abort/u); + // The host can accept after the client loses its receipt. The retry must + // replay that completed admission rather than dispatch it again. + releaseAdmission(); + while ( + ( + (await ( + await fetch(`${launched.origin}/api/snapshot`, { headers }) + ).json()) as { cursor: number } + ).cursor < 2 + ) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + + const replay = fetch(`${launched.origin}/api/prompt`, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ ...prompt, retry: true }), + }); + const response = await replay; + assert.equal(response.status, 202); + assert.deepEqual(await response.json(), { + id: commandId, + accepted: true, + state: "accepted", + pendingFollowUps: 2, + cursor: 2, + }); + assert.equal(runtimePendingFollowUps, 0); + assert.equal(sendCalls, 1); + + const conflict = await fetch(`${launched.origin}/api/prompt`, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ + ...prompt, + content: "different body", + retry: true, + }), + }); + assert.equal(conflict.status, 409); + assert.deepEqual(await conflict.json(), { + code: "COMMAND_CONFLICT", + error: "commandId is already bound to a different prompt", + }); + + const unknown = await fetch(`${launched.origin}/api/prompt`, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ + ...prompt, + commandId: "after-host-restart", + retry: true, + }), + }); + assert.equal(unknown.status, 409); + assert.deepEqual(await unknown.json(), { + code: "COMMAND_ADMISSION_UNKNOWN", + error: + "previous prompt admission is unknown; refresh canonical state before sending a new request", + }); + assert.equal(sendCalls, 1); + } finally { + releaseAdmission?.(); + await host.stop(); + await rm(cwd, { recursive: true, force: true }); + } +}); + +test("fails closed instead of evicting pending prompt admissions", { + timeout: 10_000, +}, async () => { + const cwd = await mkdtemp(join(tmpdir(), "openpi-web-prompt-capacity-")); + let sendCalls = 0; + let releaseAdmissions!: () => void; + const held = new Promise((resolve) => { + releaseAdmissions = resolve; + }); + const runtime = testRuntime(cwd, async () => { + sendCalls++; + await held; + return { pendingFollowUps: 0 }; + }); + const { host, launched, headers } = await startTestHost(runtime); + try { + const requests = Array.from({ length: 128 }, (_, index) => + fetch(`${launched.origin}/api/prompt`, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionId: runtime.sessionManager.getSessionId(), + content: `pending ${index}`, + commandId: `pending-${index}`, + retry: false, + }), + }), + ); + while (sendCalls !== 128) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + const overflow = await fetch(`${launched.origin}/api/prompt`, { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionId: runtime.sessionManager.getSessionId(), + content: "must not replace a pending admission", + commandId: "overflow", + retry: false, + }), + }); + assert.equal(overflow.status, 503); + assert.deepEqual(await overflow.json(), { + code: "PROMPT_ADMISSION_CAPACITY", + error: + "prompt admission capacity is full; wait for a pending admission to settle", + }); + assert.equal(sendCalls, 128); + releaseAdmissions(); + assert.ok( + (await Promise.all(requests)).every( + (response) => response.status === 202, + ), + ); } finally { + releaseAdmissions?.(); await host.stop(); await rm(cwd, { recursive: true, force: true }); } diff --git a/web/host/web-host.ts b/web/host/web-host.ts index c06ec666..805d03c6 100644 --- a/web/host/web-host.ts +++ b/web/host/web-host.ts @@ -37,10 +37,24 @@ const MAX_COMMAND_BYTES = 16 * 1024; const MAX_SSE_CLIENTS = 8; const MAX_SSE_BUFFER_BYTES = 256 * 1024; const MAX_SSE_REPLAY_BYTES = MAX_SSE_BUFFER_BYTES; +const DEFAULT_SSE_HEARTBEAT_MS = 15_000; const SERVER_CLOSE_DRAIN_MS = 500; const DEFAULT_SHUTDOWN_TIMEOUT_MS = 5_000; +const MAX_PROMPT_ADMISSIONS = 128; const execFileAsync = promisify(execFile); +type PromptAdmissionResponse = { + readonly status: number; + readonly body: Record; +}; + +type PromptAdmission = { + readonly sessionId: string; + readonly content: string; + readonly completion: Promise; + result?: PromptAdmissionResponse; +}; + type WebRequestErrorCode = | "INVALID_REQUEST_BODY" | "REQUEST_BODY_TOO_LARGE"; @@ -72,6 +86,7 @@ export interface WebHostOptions { allowedOrigins?: readonly string[]; directoryChooser?: (signal: AbortSignal) => Promise; shutdownTimeoutMs?: number; + sseHeartbeatMs?: number; } export class WebHost { @@ -79,6 +94,10 @@ export class WebHost { private readonly token: Buffer; private readonly adapter: PiWebAdapter; private readonly clients = new Set(); + private readonly clientHeartbeats = new Map< + ServerResponse, + ReturnType + >(); private readonly events: WebEvent[] = []; private sequence = 0; private port = 0; @@ -90,11 +109,16 @@ export class WebHost { WebHostOptions["directoryChooser"] >; private readonly shutdownTimeoutMs: number; + private readonly sseHeartbeatMs: number; private readonly unsubscribeCapabilities: () => void; private readonly unsubscribeRuntime: () => void; private readonly chooserAbort = new AbortController(); private readonly leaseSensitiveRequests = new Set>(); private readonly leaseSensitiveMessages = new Set(); + private readonly promptAdmissions = new Map< + string, + PromptAdmission + >(); private stopping = false; private stopPromise?: Promise; @@ -117,6 +141,14 @@ export class WebHost { ) { throw new Error("Web host shutdown timeout must be a positive integer"); } + this.sseHeartbeatMs = + options.sseHeartbeatMs ?? DEFAULT_SSE_HEARTBEAT_MS; + if ( + !Number.isSafeInteger(this.sseHeartbeatMs) || + this.sseHeartbeatMs <= 0 + ) { + throw new Error("SSE heartbeat interval must be a positive integer"); + } this.adapter = new PiWebAdapter(options.runtime); this.onEvent = options.onEvent; this.unsubscribeCapabilities = subscribeWebCapabilities((scope) => { @@ -212,8 +244,7 @@ export class WebHost { client.writableLength > MAX_SSE_BUFFER_BYTES || !client.write(record) ) { - this.clients.delete(client); - client.destroy(); + this.removeSseClient(client, "destroy"); } } this.onEvent?.(event.type, event.detail); @@ -231,8 +262,7 @@ export class WebHost { this.unsubscribeCapabilities(); this.unsubscribeRuntime(); this.chooserAbort.abort(); - for (const client of this.clients) client.end(); - this.clients.clear(); + for (const client of [...this.clients]) this.removeSseClient(client, "end"); const closeServer = this.server.listening ? new Promise((resolve) => { const forceClose = setTimeout( @@ -523,6 +553,51 @@ export class WebHost { error: "prompt must be 1-12000 characters", }); } + const commandId = + typeof body.commandId === "string" && body.commandId.length > 0 + ? body.commandId + : randomUUID(); + if (commandId.length > 128) { + return this.json(response, 400, { + error: "commandId must be at most 128 characters", + }); + } + if (body.retry !== undefined && typeof body.retry !== "boolean") { + return this.json(response, 400, { + error: "retry must be a boolean when provided", + }); + } + if (typeof body.sessionId !== "string") { + return this.json(response, 400, { + error: "sessionId is required", + }); + } + const existing = this.promptAdmissions.get(commandId); + if (existing) { + if ( + existing.sessionId !== body.sessionId || + existing.content !== content + ) { + return this.json(response, 409, { + code: "COMMAND_CONFLICT", + error: "commandId is already bound to a different prompt", + }); + } + const result = await existing.completion; + traceWeb("prompt_admission_replayed", { + commandId, + sessionId: body.sessionId, + status: result.status, + elapsedMs: elapsed(requestStarted), + }); + return this.json(response, result.status, result.body); + } + if (body.retry === true) { + return this.json(response, 409, { + code: "COMMAND_ADMISSION_UNKNOWN", + error: "previous prompt admission is unknown; refresh canonical state before sending a new request", + }); + } if (this.runtime.workspaceSelected !== true) { return this.json(response, 409, { code: "WORKSPACE_REQUIRED", @@ -530,7 +605,6 @@ export class WebHost { }); } if ( - typeof body.sessionId !== "string" || body.sessionId !== this.runtime.sessionManager.getSessionId() ) { return this.json(response, 409, { @@ -538,51 +612,19 @@ export class WebHost { error: "Only the active Web session accepts messages", }); } - const commandId = randomUUID(); - traceWeb("prompt_received", { - commandId, - sessionId: body.sessionId, - chars: content.length, - }); - let admission: { pendingFollowUps: number }; - try { - admission = await this.runtime.sendPrompt(content, { - commandId, - expectedSessionId: body.sessionId, - }); - traceWeb("prompt_admission_finished", { - commandId, - elapsedMs: elapsed(requestStarted), - }); - } catch (error) { - const failure = this.runtimeRequestFailure(error); - traceWeb("prompt_admission_failed", { - commandId, - elapsedMs: elapsed(requestStarted), - error: failure.error, - }); - return this.json(response, failure.status, { - code: failure.code, - error: failure.error, + if (!this.makePromptAdmissionSpace()) { + return this.json(response, 503, { + code: "PROMPT_ADMISSION_CAPACITY", + error: "prompt admission capacity is full; wait for a pending admission to settle", }); } - this.publish("prompt_accepted", { + const admission = this.beginPromptAdmission( commandId, - sessionId: body.sessionId, - pendingFollowUps: admission.pendingFollowUps, - }); - traceWeb("prompt_response_sent", { - commandId, - sessionId: body.sessionId, - elapsedMs: elapsed(requestStarted), - }); - return this.json(response, 202, { - id: commandId, - accepted: true, - state: "accepted", - pendingFollowUps: admission.pendingFollowUps, - cursor: this.sequence, - }); + body.sessionId, + content, + ); + const result = await admission.completion; + return this.json(response, result.status, result.body); } if (url.pathname === "/api/turns/cancel" && request.method === "POST") { const body = await this.readJson(request); @@ -728,6 +770,108 @@ export class WebHost { } } + private makePromptAdmissionSpace() { + while (this.promptAdmissions.size >= MAX_PROMPT_ADMISSIONS) { + const settled = [...this.promptAdmissions.entries()].find( + ([, admission]) => admission.result !== undefined, + ); + if (!settled) return false; + this.promptAdmissions.delete(settled[0]); + } + return true; + } + + private beginPromptAdmission( + commandId: string, + sessionId: string, + content: string, + ) { + let settle!: (result: PromptAdmissionResponse) => void; + const admission: PromptAdmission = { + sessionId, + content, + completion: new Promise((resolve) => { + settle = resolve; + }), + }; + // Store before dispatch: a client retry can only replay this record. + this.promptAdmissions.set(commandId, admission); + try { + traceWeb("prompt_received", { + commandId, + sessionId, + chars: content.length, + }); + } catch {} + void Promise.resolve() + .then(() => + this.runtime.sendPrompt(content, { + commandId, + expectedSessionId: sessionId, + }), + ) + .then( + (receipt) => { + const result: PromptAdmissionResponse = { + status: 202, + body: { + id: commandId, + accepted: true, + state: "accepted", + pendingFollowUps: receipt.pendingFollowUps, + cursor: this.sequence, + }, + }; + try { + this.publish("prompt_accepted", { + commandId, + sessionId, + pendingFollowUps: receipt.pendingFollowUps, + }); + result.body.cursor = this.sequence; + } catch {} + return result; + }, + (error) => { + const failure = this.runtimeRequestFailure(error); + return { + status: failure.status, + body: { code: failure.code, error: failure.error }, + }; + }, + ) + .then((result: PromptAdmissionResponse) => { + admission.result = result; + settle(result); + try { + traceWeb( + result.status === 202 + ? "prompt_admission_finished" + : "prompt_admission_failed", + { + commandId, + sessionId, + status: result.status, + ...(typeof result.body.error === "string" + ? { error: result.body.error } + : {}), + }, + ); + } catch {} + }) + .catch((error) => { + if (admission.result) return; + const failure = this.runtimeRequestFailure(error); + const result: PromptAdmissionResponse = { + status: failure.status, + body: { code: failure.code, error: failure.error }, + }; + admission.result = result; + settle(result); + }); + return admission; + } + private async readJson(request: IncomingMessage) { const chunks: Buffer[] = []; let bytes = 0; @@ -848,7 +992,31 @@ export class WebHost { // ordering without treating normal backpressure as a broken client. for (const record of replay) response.write(record); this.clients.add(response); - response.on("close", () => this.clients.delete(response)); + const heartbeat = setInterval(() => { + if ( + response.destroyed || + response.writableEnded || + response.writableLength > MAX_SSE_BUFFER_BYTES || + !response.write(": heartbeat\n\n") + ) { + this.removeSseClient(response, "destroy"); + } + }, this.sseHeartbeatMs); + heartbeat.unref(); + this.clientHeartbeats.set(response, heartbeat); + response.on("close", () => this.removeSseClient(response)); + } + + private removeSseClient( + response: ServerResponse, + close?: "destroy" | "end", + ) { + this.clients.delete(response); + const heartbeat = this.clientHeartbeats.get(response); + if (heartbeat) clearInterval(heartbeat); + this.clientHeartbeats.delete(response); + if (close === "destroy" && !response.destroyed) response.destroy(); + else if (close === "end" && !response.writableEnded) response.end(); } private parseCursor(value: string | undefined | null) { diff --git a/web/ui/app.js b/web/ui/app.js index 442ea25b..0c73e4b3 100644 --- a/web/ui/app.js +++ b/web/ui/app.js @@ -25,6 +25,7 @@ const state = { promptAdmissionPending: false, promptAdmissionToken: null, promptAdmissionSequence: 0, + promptAdmission: null, terminalPromptIds: new Set(), sessionEpoch: 0, sessionSwitching: false, @@ -33,6 +34,7 @@ const state = { snapshotGeneration: 0, livePhase: "idle", liveRetry: null, + composerFeedback: null, pendingFollowUpsReceipt: null, themePreference: "system", query: "", @@ -72,6 +74,10 @@ const translations = { enterHint: "Enter to send, Shift+Enter for a new line.", activeOnlyHint: "Only the active Web session accepts messages.", acceptedHint: "Message accepted by OpenPI Web.", + admissionPendingHint: "OpenPI is still accepting the previous message.", + admissionTimeout: "Prompt admission timed out. Retrying will check the same message.", + requestTimeout: "The Web request timed out.", + reconnectingHint: "Live updates were interrupted. Reconnecting and checking canonical state...", pendingFollowUpsHint: "Message received; {count} follow-up messages were waiting when it was received.", stopTurn: "Stop turn", stoppingTurn: "Stopping current turn...", @@ -112,6 +118,10 @@ const translations = { enterHint: "按 Enter 发送,Shift+Enter 换行。", activeOnlyHint: "只有当前 Web 会话可以接收消息。", acceptedHint: "OpenPI Web 已接收消息。", + admissionPendingHint: "OpenPI 仍在接收上一条消息,请稍候。", + admissionTimeout: "消息接收超时;重试会核对同一条消息。", + requestTimeout: "Web 请求已超时。", + reconnectingHint: "实时更新已中断,正在重连并核对权威状态……", pendingFollowUpsHint: "消息已接收;接收时有 {count} 条后续消息等待处理。", stopTurn: "停止当前回合", stoppingTurn: "正在停止当前回合...", @@ -153,6 +163,10 @@ systemTheme?.addEventListener?.("change", () => { if (state.themePreference === "system") applyThemePreference("system"); }); const tokenStorageKey = "openpi.web.token"; +const DEFAULT_API_TIMEOUT_MS = 15_000; +const PROMPT_ADMISSION_TIMEOUT_MS = 30_000; +const SSE_STALE_TIMEOUT_MS = 45_000; +const HEARTBEATS_PER_SNAPSHOT = 4; const fragmentToken = new URLSearchParams(location.hash.slice(1)).get("token"); let token = fragmentToken; if (fragmentToken) { @@ -165,6 +179,21 @@ const headers = (json = false) => ({ Authorization: `Bearer ${token}`, ...(json ? { "Content-Type": "application/json" } : {}), }); + +function setComposerFeedback(message, kind = "status") { + state.composerFeedback = message ? { message, kind } : null; + const hint = $("composer-hint"); + if (!hint) return; + hint.textContent = message || ""; + for (const candidate of ["status", "error", "connection"]) { + hint.classList.toggle(candidate, candidate === kind && Boolean(message)); + } +} + +function clearComposerFeedback(kind) { + if (kind && state.composerFeedback?.kind !== kind) return; + setComposerFeedback(""); +} const escapeHtml = (value) => String(value ?? "").replace( /[&<>"']/g, @@ -222,13 +251,45 @@ function renderMarkdown(value) { } async function api(path, options = {}) { - const response = await fetch(path, { - ...options, - headers: { ...headers(Boolean(options.body)), ...options.headers }, - }); - const body = await response.json().catch(() => ({})); - if (!response.ok) throw new Error(body.error || `Request failed (${response.status})`); - return body; + const { + timeoutMs = DEFAULT_API_TIMEOUT_MS, + timeoutMessage = t("requestTimeout"), + ...requestOptions + } = options; + const controller = new AbortController(); + let timedOut = false; + const timer = window.setTimeout(() => { + timedOut = true; + controller.abort(); + }, timeoutMs); + try { + const response = await fetch(path, { + ...requestOptions, + signal: controller.signal, + headers: { + ...headers(Boolean(requestOptions.body)), + ...requestOptions.headers, + }, + }); + const body = await response.json().catch(() => ({})); + if (!response.ok) { + const failure = new Error(body.error || `Request failed (${response.status})`); + failure.name = "WebApiResponseError"; + if (typeof body.code === "string") failure.code = body.code; + failure.status = response.status; + throw failure; + } + return body; + } catch (error) { + if (timedOut) { + const timeout = new Error(timeoutMessage); + timeout.name = "WebRequestTimeout"; + throw timeout; + } + throw error; + } finally { + window.clearTimeout(timer); + } } function sessionTitle(session) { @@ -575,25 +636,30 @@ function updateComposer() { : active ? t("promptMessage") : t("promptReadonly"); - const composerHint = $("composer-hint"); - composerHint.classList.toggle("receipt", state.pendingFollowUpsReceipt > 0); - composerHint.textContent = - state.turnCancellationPending - ? t("stoppingTurn") - : state.turnTerminalStatus === "cancelled" - ? t("stoppedTurn") - : state.pendingFollowUpsReceipt !== null - ? state.pendingFollowUpsReceipt > 0 - ? t("pendingFollowUpsHint").replace( - "{count}", - String(state.pendingFollowUpsReceipt), - ) - : t("acceptedHint") - : canCompose - ? state.snapshot.runtime.status === "running" || state.liveRunning - ? t("queuedHint") - : t("enterHint") - : t("activeOnlyHint"); + const defaultHint = canCompose + ? state.snapshot.runtime.status === "running" || state.liveRunning + ? t("queuedHint") + : t("enterHint") + : t("activeOnlyHint"); + const feedback = state.composerFeedback; + const hint = $("composer-hint"); + const receipt = state.pendingFollowUpsReceipt; + hint.textContent = state.turnCancellationPending + ? t("stoppingTurn") + : state.turnTerminalStatus === "cancelled" + ? t("stoppedTurn") + : receipt !== null + ? receipt > 0 + ? t("pendingFollowUpsHint").replace("{count}", String(receipt)) + : t("acceptedHint") + : feedback?.message || defaultHint; + hint.classList.toggle("receipt", receipt > 0); + for (const candidate of ["status", "error", "connection"]) { + hint.classList.toggle( + candidate, + receipt === null && feedback?.kind === candidate, + ); + } } async function selectModel(value) { @@ -673,7 +739,8 @@ async function refreshSnapshot({ applyThemePreference(snapshot.preferences?.theme); if ( state.snapshot.runtime.status !== "running" && - !state.promptAdmissionPending + !state.promptAdmissionPending && + !state.promptAdmission ) { state.liveRunning = false; state.livePhase = "idle"; @@ -705,8 +772,7 @@ async function refreshSnapshot({ ) return false; $("connection-state").textContent = "Unavailable"; $("connection-state").classList.add("reconnecting"); - $("composer-hint").textContent = error.message; - $("composer-hint").classList.add("error"); + setComposerFeedback(error.message, "error"); return false; } } @@ -719,6 +785,7 @@ async function selectSession(path) { state.selectedPath = path; state.promptAdmissionPending = false; state.promptAdmissionToken = null; + state.promptAdmission = null; resetLiveState(); document.body.classList.remove("sidebar-open"); renderWorkspaces(); @@ -783,11 +850,14 @@ async function sendPrompt() { await chooseWorkspace(); } const content = $("prompt-input").value.trim(); + if (state.promptAdmissionPending) { + setComposerFeedback(t("admissionPendingHint")); + return; + } if ( !content || !state.selectedWorkspace || - state.sessionSwitching || - state.promptAdmissionPending + state.sessionSwitching ) return; if (!state.snapshot?.selectedSession?.id) { await createSession(state.selectedWorkspace); @@ -796,40 +866,79 @@ async function sendPrompt() { if (!sessionId || state.sessionSwitching || state.promptAdmissionPending) return; const epoch = state.sessionEpoch; const admissionToken = ++state.promptAdmissionSequence; - const optimisticKey = `optimistic-${Date.now()}`; - state.liveMessages = [ - ...state.liveMessages, - { key: optimisticKey, message: { role: "user", content } }, - ].slice(-8); + const retrying = + state.promptAdmission?.sessionId === sessionId && + state.promptAdmission?.content === content; + const commandId = retrying + ? state.promptAdmission.commandId + : globalThis.crypto?.randomUUID?.() || + `web-prompt-${Date.now()}-${admissionToken}`; + const optimisticKey = retrying + ? state.promptAdmission.optimisticKey + : `optimistic-${commandId}`; + if (!retrying) { + state.liveMessages = [ + ...state.liveMessages, + { key: optimisticKey, message: { role: "user", content } }, + ].slice(-8); + } + // Keep this attempt before dispatch. A timeout may be a lost receipt, not a + // failed admission, so the next submit must replay this exact request. + state.promptAdmission = { sessionId, content, commandId, optimisticKey }; state.promptAdmissionPending = true; state.promptAdmissionToken = admissionToken; + clearComposerFeedback(); state.pendingFollowUpsReceipt = null; state.turnTerminalStatus = null; renderConversation(); - $("composer-hint").classList.remove("error"); try { const receipt = await api("/api/prompt", { method: "POST", - body: JSON.stringify({ sessionId, content }), + body: JSON.stringify({ sessionId, content, commandId, retry: retrying }), + timeoutMs: PROMPT_ADMISSION_TIMEOUT_MS, + timeoutMessage: t("admissionTimeout"), }); if (epoch !== state.sessionEpoch || state.promptAdmissionToken !== admissionToken) return; const alreadySettled = state.terminalPromptIds.has(receipt.id); applyPromptAcceptedState(alreadySettled); + if (state.promptAdmission?.commandId === commandId) { + state.promptAdmission = null; + } state.pendingFollowUpsReceipt = receipt.pendingFollowUps; $("prompt-input").value = ""; resizePrompt(); - $("composer-hint").textContent = t("acceptedHint"); + clearComposerFeedback(); scheduleSnapshotRefresh(120); } catch (error) { if (epoch !== state.sessionEpoch || state.promptAdmissionToken !== admissionToken) return; - state.liveRunning = false; - state.livePhase = "idle"; - state.liveRetry = null; + const knownRejection = + error?.name === "WebApiResponseError" && + [ + "WORKSPACE_REQUIRED", + "SESSION_CONFLICT", + "PROMPT_REJECTED", + "COMMAND_CONFLICT", + "PROMPT_ADMISSION_CAPACITY", + ].includes(error?.code); + if (!knownRejection) { + if (state.livePhase !== "running") { + state.liveRunning = true; + state.livePhase = "preparing"; + } + state.liveRetry = null; + setComposerFeedback( + error?.name === "WebRequestTimeout" ? t("admissionTimeout") : error.message, + "connection", + ); + return; + } + if (state.promptAdmission?.commandId === commandId) { + state.promptAdmission = null; + } state.liveMessages = state.liveMessages.filter( (entry) => entry.key !== optimisticKey, ); - $("composer-hint").textContent = error.message; - $("composer-hint").classList.add("error"); + setComposerFeedback(error.message, "error"); } finally { if (epoch === state.sessionEpoch && state.promptAdmissionToken === admissionToken) { state.promptAdmissionPending = false; @@ -950,6 +1059,7 @@ async function createSession(workspacePath) { state.selectedPath = null; state.promptAdmissionPending = false; state.promptAdmissionToken = null; + state.promptAdmission = null; resetLiveState(); document.body.classList.remove("sidebar-open"); renderWorkspaces(); @@ -994,8 +1104,7 @@ async function createSession(workspacePath) { } function showNotice(message) { - $("composer-hint").textContent = message; - $("composer-hint").classList.add("error"); + setComposerFeedback(message, "error"); } function openWorkspaceMenu(path, anchor) { @@ -1297,6 +1406,17 @@ function rememberCompletedActivation(commandId) { } let eventLoopStarted = false; +let eventLoopStopped = false; +let activeEventReader = null; + +window.addEventListener("pagehide", (event) => { + // A bfcache entry resumes this same document and its event loop. Do not + // create a second SSE connection on pageshow. + if (event.persisted) return; + eventLoopStopped = true; + void activeEventReader?.cancel().catch(() => undefined); +}); + function resetLiveState() { state.liveMessages = []; state.liveRunning = false; @@ -1312,7 +1432,7 @@ async function connectEvents() { if (eventLoopStarted) return; eventLoopStarted = true; let reconnectDelay = 500; - while (true) { + while (!eventLoopStopped) { let reader = null; let recoveryAttempted = false; try { @@ -1334,17 +1454,28 @@ async function connectEvents() { if (!response.ok || !response.body) throw new Error("event connection failed"); $("connection-state").textContent = "Connected"; $("connection-state").classList.remove("reconnecting"); + clearComposerFeedback("connection"); reconnectDelay = 500; reader = response.body.getReader(); + activeEventReader = reader; const decoder = new TextDecoder(); let buffer = ""; + let heartbeatCount = 0; while (true) { - const { done, value } = await reader.read(); + const { done, value } = await readEventChunk(reader); if (done) throw new Error("event connection closed"); buffer += decoder.decode(value, { stream: true }); const records = buffer.split("\n\n"); buffer = records.pop() || ""; for (const record of records) { + if (record.split("\n").some((item) => item === ": heartbeat")) { + heartbeatCount++; + if (heartbeatCount >= HEARTBEATS_PER_SNAPSHOT) { + heartbeatCount = 0; + scheduleSnapshotRefresh(0); + } + continue; + } const line = record.split("\n").find((item) => item.startsWith("data: ")); if (!line) continue; const event = JSON.parse(line.slice(6)); @@ -1358,21 +1489,42 @@ async function connectEvents() { } } catch { await reader?.cancel().catch(() => undefined); + if (activeEventReader === reader) activeEventReader = null; reader = null; + if (eventLoopStopped) break; $("connection-state").textContent = "Reconnecting"; $("connection-state").classList.add("reconnecting"); const recovered = recoveryAttempted ? false : await refreshSnapshot({ resetCursor: true }); if (!recovered) resetLiveState(); + setComposerFeedback(t("reconnectingHint"), "connection"); await new Promise((resolve) => setTimeout(resolve, reconnectDelay)); reconnectDelay = Math.min(reconnectDelay * 2, 5_000); } finally { await reader?.cancel().catch(() => undefined); + if (activeEventReader === reader) activeEventReader = null; } } } +async function readEventChunk(reader, timeoutMs = SSE_STALE_TIMEOUT_MS) { + let timer; + try { + return await Promise.race([ + reader.read(), + new Promise((_, reject) => { + timer = window.setTimeout( + () => reject(new Error("event stream stalled")), + timeoutMs, + ); + }), + ]); + } finally { + if (timer !== undefined) window.clearTimeout(timer); + } +} + const collapseButton = $("collapse-sidebar"); const collapsedStorageKey = "openpi.sidebar-collapsed"; const setSidebarCollapsed = (collapsed) => { @@ -1528,7 +1680,12 @@ $("composer")?.addEventListener("submit", (event) => { $("stop-turn")?.addEventListener("click", () => { void cancelActiveTurn(); }); -$("prompt-input")?.addEventListener("input", resizePrompt); +$("prompt-input")?.addEventListener("input", () => { + state.pendingFollowUpsReceipt = null; + if (!state.promptAdmissionPending) clearComposerFeedback(); + resizePrompt(); + updateComposer(); +}); $("prompt-input")?.addEventListener("keydown", (event) => { if (event.isComposing || event.keyCode === 229) return; if (event.key === "Enter" && !event.shiftKey) { diff --git a/web/ui/styles.css b/web/ui/styles.css index f4c8f4ec..f9f216f5 100644 --- a/web/ui/styles.css +++ b/web/ui/styles.css @@ -582,8 +582,10 @@ body.sidebar-collapsed .icon-button { width: 36px; height: 36px; } .stop-button[hidden] { display: none; } .stop-button svg { width: 16px; height: 16px; fill: currentColor; stroke: none; } .composer-hint { display: none; } +.composer-hint.status, .composer-hint.error, .composer-hint.connection { display: block; margin: 7px 2px 0; color: var(--muted); font-size: 11px; line-height: 1.35; } .composer-hint.receipt { display: block; color: var(--subtle); } .composer-hint.error { color: var(--error); } +.composer-hint.connection { color: #9a6700; } .sidebar-scrim { display: none; } @media (max-width: 760px) { @@ -627,6 +629,8 @@ body.sidebar-collapsed .icon-button { width: 36px; height: 36px; } .conversation { scroll-behavior: auto; } } +.composer-hint.status, .composer-hint.error, .composer-hint.connection { display: block; } + /* The conversation view should start with the actual messages. On narrow screens keep only the sidebar trigger as an overlay so navigation remains available without restoring the session header. */