From cac5ed011ed052352efbb4a71f5763b8f25f81c8 Mon Sep 17 00:00:00 2001 From: "replicas-connector[bot]" Date: Sat, 30 May 2026 18:56:42 +0000 Subject: [PATCH 1/2] =?UTF-8?q?fix(bridges):=2030=E2=86=9260=20min=20watch?= =?UTF-8?q?=20cap=20+=20cancel=20upstream=20replica=20on=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings from Lark Admin room review: last turn there hit the 30-min timeout while the agent was *actively* making progress through multi- step MCP config cleanups. Session has 317 cumulative steps / 40 turns — this is real heavy ops work, not a runaway loop. Two fixes: **#1 Bump MAX_WATCH_DURATION_MS from 30 → 60 min** (both bridges) Lark Admin's "Watch timed out — agent ran longer than 30 min" was a false-positive cap on legitimate work. 60 min keeps runaway-loop protection while letting heavy ops finish. **#2 Cancel upstream replica + flush KV on timeout** When the watcher gave up at the cap, the upstream Replicas workspace kept processing silently — burning credits indefinitely. The bridge had no visibility into what state the replica ended up in. On timeout we now: - DELETE /v1/replica/{id} so the workspace stops - Flush `room:{roomId}` / `replica:{id}` (or `chat:{id}:thread:{tid}` for TG) KV mappings so the next message in the room spawns fresh - Then setTerminal + deleteAll as before - Failed-frame message updated: "Send a new message to start a fresh session." so the user knows the room is recoverable Matches the behavior we already had on claude-result 404/410 paths. **Validation** - Matrix: 173/173 ✅ · typecheck clean · deployed `1d6a1f2d` - Telegram: 99/99 ✅ · typecheck clean · deployed `944a6aa3` Co-Authored-By: itsablabla Co-Authored-By: Claude Opus 4.7 (1M context) --- replicas-matrix-bridge/src/poller.ts | 33 ++++++++++++++++++++++++-- replicas-telegram-bridge/src/poller.ts | 23 +++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/replicas-matrix-bridge/src/poller.ts b/replicas-matrix-bridge/src/poller.ts index 4255f13c..fc3a29dc 100644 --- a/replicas-matrix-bridge/src/poller.ts +++ b/replicas-matrix-bridge/src/poller.ts @@ -90,7 +90,12 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -const MAX_WATCH_DURATION_MS = 30 * 60 * 1000; +// 60 min hard cap. Was 30 min, bumped after Lark Admin power-user +// ops chats legitimately needed >30 min for multi-step infra work +// (observed: 30-min timeout on a turn that was actively making +// progress through MCP config cleanups). At 60 min, runaway-loop +// protection still kicks in but realistic heavy work fits. +const MAX_WATCH_DURATION_MS = 60 * 60 * 1000; // matrix.org's per-room send rate is roughly 30/min (one every 2s). // 1000ms edits were tripping M_LIMIT_EXCEEDED on long turns, fragmenting // the status frame. 2000ms / 4000ms ticker keeps us safely under the @@ -416,11 +421,35 @@ export class ReplicaPoller { // status pane frozen at whatever mid-progress frame was last // rendered. Now we land a terminal "timed out" frame first so // the pane resolves cleanly and the room doesn't look stuck. + // + // Also: cancel the upstream replica so it stops processing. + // Without this, the watcher gave up at the cap but the + // Replicas workspace kept running and burning credits silently + // (observed in Lark Admin room: 30-min watch timeout left an + // active replica running for hours afterward). const seconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); + try { + await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { + method: "DELETE", + headers: replicasHeaders(this.env), + }); + console.log(`[poller] timeout: deleted upstream replica ${watch.replicaId}`); + } catch (e) { + console.log(`[poller] timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + } + // Also flush the room→replica mapping so the next message + // in this room spawns a fresh workspace instead of trying to + // follow up on the deleted one. + try { + await this.env.MAP.delete(`room:${watch.roomId}`); + await this.env.MAP.delete(`replica:${watch.replicaId}`); + } catch (e) { + console.log(`[poller] timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + } await this.setTerminal(watch, { kind: "failed", durationSec: seconds, - errorMsg: `Watch timed out — agent ran longer than ${Math.round(MAX_WATCH_DURATION_MS / 60_000)} min.`, + errorMsg: `Watch timed out — agent ran longer than ${Math.round(MAX_WATCH_DURATION_MS / 60_000)} min. Send a new message to start a fresh session.`, }); await this.state.storage.deleteAll(); return; diff --git a/replicas-telegram-bridge/src/poller.ts b/replicas-telegram-bridge/src/poller.ts index badad4bb..79b6bc25 100644 --- a/replicas-telegram-bridge/src/poller.ts +++ b/replicas-telegram-bridge/src/poller.ts @@ -64,7 +64,9 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -const MAX_WATCH_DURATION_MS = 30 * 60 * 1000; +// 60 min hard cap (ported from matrix bridge — heavy ops chats +// legitimately need >30 min for multi-step work). +const MAX_WATCH_DURATION_MS = 60 * 60 * 1000; // Telegram permits ~1 editMessageText per chat per second; we leave a small // margin so a burst of polls coalesces into at most one edit per ~900ms. const EDIT_MIN_INTERVAL_MS = 500; @@ -258,6 +260,25 @@ export class ReplicaPoller { const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now(); if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) { + // Cancel the upstream replica so it stops processing — the + // watcher giving up doesn't auto-stop the Replicas workspace. + try { + await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { + method: "DELETE", + headers: replicasHeaders(this.env), + }); + console.log(`[poller] timeout: deleted upstream replica ${watch.replicaId}`); + } catch (e) { + console.log(`[poller] timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + } + // Flush the chat→replica mapping so the next message spawns + // fresh instead of following up on the deleted replica. + try { + const chatKey = `chat:${watch.chatId}:thread:${watch.threadId ?? "main"}`; + await this.env.MAP.delete(chatKey); + } catch (e) { + console.log(`[poller] timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + } await this.state.storage.deleteAll(); return; } From 1ace05b68c58724fcc7813b3e701c1277b998611 Mon Sep 17 00:00:00 2001 From: "replicas-connector[bot]" Date: Sat, 30 May 2026 19:01:41 +0000 Subject: [PATCH 2/2] fix(bridges): replace wall-clock cap with idle timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Jaden's question: why have a wall-clock cap at all? The cap was the wrong signal — wall-clock fires regardless of whether the agent is actually stuck or just doing legitimate long work. Lark Admin hit the 30-min cap mid-progress; bumping to 60 was a temporary patch. The correct signal for "stuck" is *no /history events for N minutes*. While the agent keeps emitting events (tool calls, thinking, text), the watcher keeps watching — no matter how long. When events go silent for IDLE_TIMEOUT_MS, the agent is actually stuck and we clean up. **Changes (both bridges):** - `MAX_WATCH_DURATION_MS` → `IDLE_TIMEOUT_MS` (30 min idle) - Matrix: reuse existing `lastEventAt` (already tracked for the idle indicator) - TG: add `lastFreshAt` storage write whenever `fresh.length > 0` on the alarm tick - Failed-frame copy: "Agent went silent for N min — treating as stuck. Send a new message to start a fresh session." Same cleanup behavior on timeout: DELETE upstream replica, flush KV mappings, setTerminal + deleteAll. **Validation** - Matrix: 173/173 ✅ · typecheck clean · deployed `5419497d` - Telegram: 99/99 ✅ · typecheck clean · deployed `18cfe9d0` Co-Authored-By: itsablabla Co-Authored-By: Claude Opus 4.7 (1M context) --- replicas-matrix-bridge/src/poller.ts | 51 ++++++++++++++------------ replicas-telegram-bridge/src/poller.ts | 32 ++++++++++------ 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/replicas-matrix-bridge/src/poller.ts b/replicas-matrix-bridge/src/poller.ts index fc3a29dc..1dfc9e6c 100644 --- a/replicas-matrix-bridge/src/poller.ts +++ b/replicas-matrix-bridge/src/poller.ts @@ -90,12 +90,14 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -// 60 min hard cap. Was 30 min, bumped after Lark Admin power-user -// ops chats legitimately needed >30 min for multi-step infra work -// (observed: 30-min timeout on a turn that was actively making -// progress through MCP config cleanups). At 60 min, runaway-loop -// protection still kicks in but realistic heavy work fits. -const MAX_WATCH_DURATION_MS = 60 * 60 * 1000; +// Idle timeout — "stuck" is measured by NO events from /history for +// IDLE_TIMEOUT_MS, not by total wall-clock runtime. As long as the +// agent keeps emitting events (tool calls, thinking, text), the +// watcher keeps watching. Heavy ops can run for hours legitimately; +// only truly stuck/silent agents hit the cap. Updated from the old +// 60-min hard cap after Jaden flagged that wall-clock is the wrong +// signal for "stuck". +const IDLE_TIMEOUT_MS = 30 * 60 * 1000; // matrix.org's per-room send rate is roughly 30/min (one every 2s). // 1000ms edits were tripping M_LIMIT_EXCEEDED on long turns, fragmenting // the status frame. 2000ms / 4000ms ticker keeps us safely under the @@ -416,40 +418,41 @@ export class ReplicaPoller { if (!watch) return; const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now(); - if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) { - // Pre-fix: deleteAll() ran silently, leaving the user-facing - // status pane frozen at whatever mid-progress frame was last - // rendered. Now we land a terminal "timed out" frame first so - // the pane resolves cleanly and the room doesn't look stuck. - // - // Also: cancel the upstream replica so it stops processing. - // Without this, the watcher gave up at the cap but the - // Replicas workspace kept running and burning credits silently - // (observed in Lark Admin room: 30-min watch timeout left an - // active replica running for hours afterward). + const lastEventAtForTimeout = + (await this.state.storage.get("lastEventAt")) ?? startedAt; + const idleMs = Date.now() - lastEventAtForTimeout; + if (idleMs > IDLE_TIMEOUT_MS) { + // Idle timeout — no /history events for IDLE_TIMEOUT_MS means + // the agent is actually stuck (not just thinking). Wall-clock + // cap removed entirely; heavy ops can run as long as they keep + // emitting events. When this fires we: + // 1. DELETE the upstream replica so it stops processing + // (without this, watcher gives up but Replicas workspace + // keeps running and burning credits silently). + // 2. Flush room: + replica: KV mappings so the next message + // spawns fresh instead of following up on the dead one. + // 3. Land a terminal Failed frame so the pane resolves + // cleanly and the room doesn't look stuck. const seconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); try { await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { method: "DELETE", headers: replicasHeaders(this.env), }); - console.log(`[poller] timeout: deleted upstream replica ${watch.replicaId}`); + console.log(`[poller] idle-timeout: deleted upstream replica ${watch.replicaId}`); } catch (e) { - console.log(`[poller] timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + console.log(`[poller] idle-timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); } - // Also flush the room→replica mapping so the next message - // in this room spawns a fresh workspace instead of trying to - // follow up on the deleted one. try { await this.env.MAP.delete(`room:${watch.roomId}`); await this.env.MAP.delete(`replica:${watch.replicaId}`); } catch (e) { - console.log(`[poller] timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + console.log(`[poller] idle-timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); } await this.setTerminal(watch, { kind: "failed", durationSec: seconds, - errorMsg: `Watch timed out — agent ran longer than ${Math.round(MAX_WATCH_DURATION_MS / 60_000)} min. Send a new message to start a fresh session.`, + errorMsg: `Agent went silent for ${Math.round(idleMs / 60_000)} min — treating as stuck. Send a new message to start a fresh session.`, }); await this.state.storage.deleteAll(); return; diff --git a/replicas-telegram-bridge/src/poller.ts b/replicas-telegram-bridge/src/poller.ts index 79b6bc25..6571906e 100644 --- a/replicas-telegram-bridge/src/poller.ts +++ b/replicas-telegram-bridge/src/poller.ts @@ -64,9 +64,10 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -// 60 min hard cap (ported from matrix bridge — heavy ops chats -// legitimately need >30 min for multi-step work). -const MAX_WATCH_DURATION_MS = 60 * 60 * 1000; +// Idle timeout — "stuck" measured by no /history events for +// IDLE_TIMEOUT_MS, not by wall-clock runtime. Heavy ops can run as +// long as the agent keeps emitting events. Wall-clock cap removed. +const IDLE_TIMEOUT_MS = 30 * 60 * 1000; // Telegram permits ~1 editMessageText per chat per second; we leave a small // margin so a burst of polls coalesces into at most one edit per ~900ms. const EDIT_MIN_INTERVAL_MS = 500; @@ -259,25 +260,28 @@ export class ReplicaPoller { if (!watch) return; const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now(); - if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) { - // Cancel the upstream replica so it stops processing — the - // watcher giving up doesn't auto-stop the Replicas workspace. + const lastFreshAt = + (await this.state.storage.get("lastFreshAt")) ?? startedAt; + const idleMs = Date.now() - lastFreshAt; + if (idleMs > IDLE_TIMEOUT_MS) { + // Idle timeout. Wall-clock cap removed; treats "no new + // /history events for IDLE_TIMEOUT_MS" as stuck. Cancel the + // upstream replica + flush KV so the next message spawns + // fresh. try { await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { method: "DELETE", headers: replicasHeaders(this.env), }); - console.log(`[poller] timeout: deleted upstream replica ${watch.replicaId}`); + console.log(`[poller] idle-timeout: deleted upstream replica ${watch.replicaId}`); } catch (e) { - console.log(`[poller] timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + console.log(`[poller] idle-timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); } - // Flush the chat→replica mapping so the next message spawns - // fresh instead of following up on the deleted replica. try { const chatKey = `chat:${watch.chatId}:thread:${watch.threadId ?? "main"}`; await this.env.MAP.delete(chatKey); } catch (e) { - console.log(`[poller] timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + console.log(`[poller] idle-timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); } await this.state.storage.deleteAll(); return; @@ -312,6 +316,12 @@ export class ReplicaPoller { const body = (await r.json()) as HistoryResponse; const events = body.events ?? []; const fresh = events.slice(lastSeenCount); + // Stamp lastFreshAt whenever new events arrive so the idle + // timeout above measures actual silence, not wall-clock since + // the turn began. + if (fresh.length > 0) { + await this.state.storage.put("lastFreshAt", Date.now()); + } const lines = (snap.get("lines") as string[] | undefined) ?? []; let phase = (snap.get("phase") as Phase | undefined) ?? "STARTING";