diff --git a/replicas-matrix-bridge/src/poller.ts b/replicas-matrix-bridge/src/poller.ts index 4255f13c..1dfc9e6c 100644 --- a/replicas-matrix-bridge/src/poller.ts +++ b/replicas-matrix-bridge/src/poller.ts @@ -90,7 +90,14 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -const MAX_WATCH_DURATION_MS = 30 * 60 * 1000; +// Idle timeout — "stuck" is measured by NO events from /history for +// IDLE_TIMEOUT_MS, not by total wall-clock runtime. As long as the +// agent keeps emitting events (tool calls, thinking, text), the +// watcher keeps watching. Heavy ops can run for hours legitimately; +// only truly stuck/silent agents hit the cap. Updated from the old +// 60-min hard cap after Jaden flagged that wall-clock is the wrong +// signal for "stuck". +const IDLE_TIMEOUT_MS = 30 * 60 * 1000; // matrix.org's per-room send rate is roughly 30/min (one every 2s). // 1000ms edits were tripping M_LIMIT_EXCEEDED on long turns, fragmenting // the status frame. 2000ms / 4000ms ticker keeps us safely under the @@ -411,16 +418,41 @@ export class ReplicaPoller { if (!watch) return; const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now(); - if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) { - // Pre-fix: deleteAll() ran silently, leaving the user-facing - // status pane frozen at whatever mid-progress frame was last - // rendered. Now we land a terminal "timed out" frame first so - // the pane resolves cleanly and the room doesn't look stuck. + const lastEventAtForTimeout = + (await this.state.storage.get("lastEventAt")) ?? startedAt; + const idleMs = Date.now() - lastEventAtForTimeout; + if (idleMs > IDLE_TIMEOUT_MS) { + // Idle timeout — no /history events for IDLE_TIMEOUT_MS means + // the agent is actually stuck (not just thinking). Wall-clock + // cap removed entirely; heavy ops can run as long as they keep + // emitting events. When this fires we: + // 1. DELETE the upstream replica so it stops processing + // (without this, watcher gives up but Replicas workspace + // keeps running and burning credits silently). + // 2. Flush room: + replica: KV mappings so the next message + // spawns fresh instead of following up on the dead one. + // 3. Land a terminal Failed frame so the pane resolves + // cleanly and the room doesn't look stuck. const seconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); + try { + await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { + method: "DELETE", + headers: replicasHeaders(this.env), + }); + console.log(`[poller] idle-timeout: deleted upstream replica ${watch.replicaId}`); + } catch (e) { + console.log(`[poller] idle-timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + } + try { + await this.env.MAP.delete(`room:${watch.roomId}`); + await this.env.MAP.delete(`replica:${watch.replicaId}`); + } catch (e) { + console.log(`[poller] idle-timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + } await this.setTerminal(watch, { kind: "failed", durationSec: seconds, - errorMsg: `Watch timed out — agent ran longer than ${Math.round(MAX_WATCH_DURATION_MS / 60_000)} min.`, + errorMsg: `Agent went silent for ${Math.round(idleMs / 60_000)} min — treating as stuck. Send a new message to start a fresh session.`, }); await this.state.storage.deleteAll(); return; diff --git a/replicas-telegram-bridge/src/poller.ts b/replicas-telegram-bridge/src/poller.ts index badad4bb..6571906e 100644 --- a/replicas-telegram-bridge/src/poller.ts +++ b/replicas-telegram-bridge/src/poller.ts @@ -64,7 +64,10 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -const MAX_WATCH_DURATION_MS = 30 * 60 * 1000; +// Idle timeout — "stuck" measured by no /history events for +// IDLE_TIMEOUT_MS, not by wall-clock runtime. Heavy ops can run as +// long as the agent keeps emitting events. Wall-clock cap removed. +const IDLE_TIMEOUT_MS = 30 * 60 * 1000; // Telegram permits ~1 editMessageText per chat per second; we leave a small // margin so a burst of polls coalesces into at most one edit per ~900ms. const EDIT_MIN_INTERVAL_MS = 500; @@ -257,7 +260,29 @@ export class ReplicaPoller { if (!watch) return; const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now(); - if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) { + const lastFreshAt = + (await this.state.storage.get("lastFreshAt")) ?? startedAt; + const idleMs = Date.now() - lastFreshAt; + if (idleMs > IDLE_TIMEOUT_MS) { + // Idle timeout. Wall-clock cap removed; treats "no new + // /history events for IDLE_TIMEOUT_MS" as stuck. Cancel the + // upstream replica + flush KV so the next message spawns + // fresh. + try { + await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { + method: "DELETE", + headers: replicasHeaders(this.env), + }); + console.log(`[poller] idle-timeout: deleted upstream replica ${watch.replicaId}`); + } catch (e) { + console.log(`[poller] idle-timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + } + try { + const chatKey = `chat:${watch.chatId}:thread:${watch.threadId ?? "main"}`; + await this.env.MAP.delete(chatKey); + } catch (e) { + console.log(`[poller] idle-timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + } await this.state.storage.deleteAll(); return; } @@ -291,6 +316,12 @@ export class ReplicaPoller { const body = (await r.json()) as HistoryResponse; const events = body.events ?? []; const fresh = events.slice(lastSeenCount); + // Stamp lastFreshAt whenever new events arrive so the idle + // timeout above measures actual silence, not wall-clock since + // the turn began. + if (fresh.length > 0) { + await this.state.storage.put("lastFreshAt", Date.now()); + } const lines = (snap.get("lines") as string[] | undefined) ?? []; let phase = (snap.get("phase") as Phase | undefined) ?? "STARTING";