diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 1f4efbe..c7bd73f 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -490,12 +490,18 @@ async function restartDaemon(): Promise { warn(`Simlock daemon is not answering; restarting it (${used + 1}/${DAEMON_RESTART_LIMIT})`); await state.simlock!.logs(); await state.simlock!.stop().catch(() => {}); - try { - await state.simlock!.start(); - } catch (err) { - return `Simlock daemon restart failed: ${message(err).slice(0, 150)}`; + // A start right after a slow stop can still find the old daemon going away + // ("Daemon connection closed"), so it gets one more try. + for (let attempt = 1; ; attempt++) { + try { + await state.simlock!.start(); + return null; + } catch (err) { + if (attempt >= 2) return `Simlock daemon restart failed: ${message(err).slice(0, 150)}`; + warn(`Simlock daemon start failed (${message(err).slice(0, 150)}); retrying`); + await interruptibleSleep(15_000); + } } - return null; } async function checkHardExit(): Promise { @@ -529,13 +535,19 @@ async function supervise(): Promise { const status = await state.simlock!.status(); if (!status) { daemonProbeFailures += 1; - warn(`simlock status failed (${daemonProbeFailures}/${DAEMON_PROBE_FAILURE_LIMIT})`); + warn( + `simlock status failed (${daemonProbeFailures}/${DAEMON_PROBE_FAILURE_LIMIT}): ${state.simlock!.statusProblem()}`, + ); // Unknown load is not evidence of idleness. lastWorkAt = Date.now(); if (daemonProbeFailures >= DAEMON_PROBE_FAILURE_LIMIT) { daemonProbeFailures = 0; - const failure = await restartDaemon(); - if (failure) await shutdown(failure, 1); + if (await state.simlock!.alive()) { + warn("Simlock is slow but answers its health check; not restarting it"); + } else { + const failure = await restartDaemon(); + if (failure) await shutdown(failure, 1); + } } } else { daemonProbeFailures = 0; diff --git a/packages/agent/src/simlock.ts b/packages/agent/src/simlock.ts index 3b79403..6bd2159 100644 --- a/packages/agent/src/simlock.ts +++ b/packages/agent/src/simlock.ts @@ -482,6 +482,23 @@ export function createSimlock(bins: SimlockBins) { status, logs, + /** Why the last status read failed; empty after a good read. */ + statusProblem: () => statusProblem, + + /** + * Whether the daemon answers its health route within `timeoutMs`. A host + * busy wiping and re-warming a device can make `simlock status` outlast + * its limit while the daemon is fine; restarting it then lost the run. + */ + async alive(timeoutMs = 60_000): Promise { + try { + const res = await fetch(SIMLOCK_HEALTH_URL, { signal: AbortSignal.timeout(timeoutMs) }); + return res.ok; + } catch { + return false; + } + }, + async configure(options: Parameters[0]): Promise { for (const [key, value] of simlockSettings(options)) { const res = await simlock(["config", "set", key, value]); @@ -526,7 +543,7 @@ export function createSimlock(bins: SimlockBins) { }, async stop(): Promise { - await simlock(["daemon", "stop"], 20_000); + await simlock(["daemon", "stop"], 90_000); }, /** @@ -590,6 +607,9 @@ export function simlockLoad(status: SimlockStatus): { leases: number; queued: nu return { leases: status.leases?.length ?? 0, queued: status.queueDepth ?? 0 }; } +/** Simlock's anonymous liveness route on the run's daemon. */ +export const SIMLOCK_HEALTH_URL = "http://127.0.0.1:4701/v1/healthz"; + /** Warm devices ready to lease right now. */ export function warmReady(status: SimlockStatus): number { return (status.warmPool?.targets ?? []).reduce((sum, target) => sum + target.ready, 0);