Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 39 additions & 7 deletions replicas-matrix-bridge/src/poller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,14 @@ interface HistoryResponse {
const FIRST_POLL_DELAY_MS = 80;
const ACTIVE_POLL_INTERVAL_MS = 180;
const BACKOFF_POLL_INTERVAL_MS = 3000;
const MAX_WATCH_DURATION_MS = 30 * 60 * 1000;
// Idle timeout — "stuck" is measured by NO events from /history for
// IDLE_TIMEOUT_MS, not by total wall-clock runtime. As long as the
// agent keeps emitting events (tool calls, thinking, text), the
// watcher keeps watching. Heavy ops can run for hours legitimately;
// only truly stuck/silent agents hit the cap. Updated from the old
// 60-min hard cap after Jaden flagged that wall-clock is the wrong
// signal for "stuck".
const IDLE_TIMEOUT_MS = 30 * 60 * 1000;
// matrix.org's per-room send rate is roughly 30/min (one every 2s).
// 1000ms edits were tripping M_LIMIT_EXCEEDED on long turns, fragmenting
// the status frame. 2000ms / 4000ms ticker keeps us safely under the
Expand Down Expand Up @@ -411,16 +418,41 @@ export class ReplicaPoller {
if (!watch) return;

const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now();
if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) {
// Pre-fix: deleteAll() ran silently, leaving the user-facing
// status pane frozen at whatever mid-progress frame was last
// rendered. Now we land a terminal "timed out" frame first so
// the pane resolves cleanly and the room doesn't look stuck.
const lastEventAtForTimeout =
(await this.state.storage.get<number>("lastEventAt")) ?? startedAt;
const idleMs = Date.now() - lastEventAtForTimeout;
if (idleMs > IDLE_TIMEOUT_MS) {
// Idle timeout — no /history events for IDLE_TIMEOUT_MS means
// the agent is actually stuck (not just thinking). Wall-clock
// cap removed entirely; heavy ops can run as long as they keep
// emitting events. When this fires we:
// 1. DELETE the upstream replica so it stops processing
// (without this, watcher gives up but Replicas workspace
// keeps running and burning credits silently).
// 2. Flush room: + replica: KV mappings so the next message
// spawns fresh instead of following up on the dead one.
// 3. Land a terminal Failed frame so the pane resolves
// cleanly and the room doesn't look stuck.
const seconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000));
try {
await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, {
method: "DELETE",
headers: replicasHeaders(this.env),
});
console.log(`[poller] idle-timeout: deleted upstream replica ${watch.replicaId}`);
} catch (e) {
console.log(`[poller] idle-timeout: replica delete failed: ${e instanceof Error ? e.message : e}`);
}
try {
await this.env.MAP.delete(`room:${watch.roomId}`);
await this.env.MAP.delete(`replica:${watch.replicaId}`);
} catch (e) {
console.log(`[poller] idle-timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`);
}
await this.setTerminal(watch, {
kind: "failed",
durationSec: seconds,
errorMsg: `Watch timed out — agent ran longer than ${Math.round(MAX_WATCH_DURATION_MS / 60_000)} min.`,
errorMsg: `Agent went silent for ${Math.round(idleMs / 60_000)} min — treating as stuck. Send a new message to start a fresh session.`,
});
await this.state.storage.deleteAll();
return;
Expand Down
35 changes: 33 additions & 2 deletions replicas-telegram-bridge/src/poller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,10 @@ interface HistoryResponse {
const FIRST_POLL_DELAY_MS = 80;
const ACTIVE_POLL_INTERVAL_MS = 180;
const BACKOFF_POLL_INTERVAL_MS = 3000;
const MAX_WATCH_DURATION_MS = 30 * 60 * 1000;
// Idle timeout — "stuck" measured by no /history events for
// IDLE_TIMEOUT_MS, not by wall-clock runtime. Heavy ops can run as
// long as the agent keeps emitting events. Wall-clock cap removed.
const IDLE_TIMEOUT_MS = 30 * 60 * 1000;
// Telegram permits ~1 editMessageText per chat per second; we leave a small
// margin so a burst of polls coalesces into at most one edit per ~900ms.
const EDIT_MIN_INTERVAL_MS = 500;
Expand Down Expand Up @@ -257,7 +260,29 @@ export class ReplicaPoller {
if (!watch) return;

const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now();
if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) {
const lastFreshAt =
(await this.state.storage.get<number>("lastFreshAt")) ?? startedAt;
const idleMs = Date.now() - lastFreshAt;
if (idleMs > IDLE_TIMEOUT_MS) {
// Idle timeout. Wall-clock cap removed; treats "no new
// /history events for IDLE_TIMEOUT_MS" as stuck. Cancel the
// upstream replica + flush KV so the next message spawns
// fresh.
try {
await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, {
method: "DELETE",
headers: replicasHeaders(this.env),
});
console.log(`[poller] idle-timeout: deleted upstream replica ${watch.replicaId}`);
} catch (e) {
console.log(`[poller] idle-timeout: replica delete failed: ${e instanceof Error ? e.message : e}`);
}
try {
const chatKey = `chat:${watch.chatId}:thread:${watch.threadId ?? "main"}`;
await this.env.MAP.delete(chatKey);
} catch (e) {
console.log(`[poller] idle-timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`);
}
await this.state.storage.deleteAll();
return;
}
Expand Down Expand Up @@ -291,6 +316,12 @@ export class ReplicaPoller {
const body = (await r.json()) as HistoryResponse;
const events = body.events ?? [];
const fresh = events.slice(lastSeenCount);
// Stamp lastFreshAt whenever new events arrive so the idle
// timeout above measures actual silence, not wall-clock since
// the turn began.
if (fresh.length > 0) {
await this.state.storage.put("lastFreshAt", Date.now());
}

const lines = (snap.get("lines") as string[] | undefined) ?? [];
let phase = (snap.get("phase") as Phase | undefined) ?? "STARTING";
Expand Down