From bb184d3c215276000cc53f874b88bb37f8e999aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20=C5=98=C3=A1dek?= Date: Sun, 22 Nov 2026 09:20:09 +0100 Subject: [PATCH] fix(server): keep-awake stops retrying when logind denies the inhibitor On a systemd user service with no interactive session, polkit refuses `systemd-inhibit --mode=block` every time. The hold retried under backoff forever and logged a full stack on each attempt (about 7,600 lines in hub's boot-service.log on 2026-11-19). An inhibitor exit whose stderr reports an access or authentication denial now logs one warning without a stack and turns every later hold in the process into a no-op. Other failures keep the existing backoff retry. The retry policy moves into an exported makeHold so tests drive it with a fake holdOnce on the TestClock. --- apps/server/src/background/KeepAwake.test.ts | 56 ++++++++++++++++++++ apps/server/src/background/KeepAwake.ts | 39 ++++++++++++-- docs/progress.md | 8 +++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/apps/server/src/background/KeepAwake.test.ts b/apps/server/src/background/KeepAwake.test.ts index 8984102729ce..ec7a942addd1 100644 --- a/apps/server/src/background/KeepAwake.test.ts +++ b/apps/server/src/background/KeepAwake.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; import * as Effect from "effect/Effect"; +import * as Logger from "effect/Logger"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; @@ -34,3 +35,58 @@ describe("KeepAwake.holdWhileWanted", () => { }), ); }); + +describe("KeepAwake.makeHold", () => { + const warningsLogger = (warnings: Array>) => + Logger.layer( + [Logger.make((entry) => (entry.logLevel === "Warn" ? warnings.push(entry) : undefined))], + { mergeWithExisting: false }, + ); + + it.effect( + "stops for good after logind denies the inhibitor, with one warning and no stack", + () => { + const warnings: Array> = []; + return Effect.gen(function* () { + const attempts = yield* Queue.unbounded(); + const hold = yield* KeepAwake.makeHold( + Queue.offer(attempts, 1).pipe( + Effect.andThen( + new KeepAwake.KeepAwakeInhibitorExited({ + exitCode: 1, + stderr: "Failed to inhibit: Access denied (interactive authentication required)", + }), + ), + ), + ); + + yield* hold; + yield* TestClock.adjust("1 hour"); + yield* hold; + + expect(yield* Queue.size(attempts)).toBe(1); + expect(warnings).toHaveLength(1); + expect(warnings[0]!.cause.reasons).toHaveLength(0); + }).pipe(Effect.provide(warningsLogger(warnings))); + }, + ); + + it.effect("retries any other inhibitor exit under backoff", () => + Effect.gen(function* () { + const attempts = yield* Queue.unbounded(); + const hold = yield* KeepAwake.makeHold( + Queue.offer(attempts, 1).pipe( + Effect.andThen(new KeepAwake.KeepAwakeInhibitorExited({ exitCode: 1, stderr: "" })), + ), + ); + yield* hold.pipe(Effect.forkScoped); + + yield* Queue.take(attempts); + yield* TestClock.adjust("5 seconds"); + yield* Queue.take(attempts); + yield* TestClock.adjust("10 seconds"); + yield* Queue.take(attempts); + expect(yield* Queue.size(attempts)).toBe(0); + }), + ); +}); diff --git a/apps/server/src/background/KeepAwake.ts b/apps/server/src/background/KeepAwake.ts index f9d2cf3da753..7ac36801360e 100644 --- a/apps/server/src/background/KeepAwake.ts +++ b/apps/server/src/background/KeepAwake.ts @@ -125,7 +125,7 @@ const retryWithBackoff = Effect.retry(backoff), ); -class KeepAwakeInhibitorExited extends Schema.TaggedError()( +export class KeepAwakeInhibitorExited extends Schema.TaggedError()( "KeepAwakeInhibitorExited", { exitCode: Schema.Number, stderr: Schema.String }, ) { @@ -134,6 +134,38 @@ class KeepAwakeInhibitorExited extends Schema.TaggedError( + error: E | KeepAwakeInhibitorExited, +): error is KeepAwakeInhibitorExited => + isInhibitorExit(error) && + /access denied|interactive authentication|not authorized/i.test(error.stderr); + +/** + * Retries `holdOnce` under backoff until interrupted, except after a permission denial, which + * makes this and every later hold a no-op. + */ +export const makeHold = (holdOnce: Effect.Effect) => + Ref.make(false).pipe( + Effect.map((denied) => + holdOnce.pipe( + Effect.catchIf(isPermissionDenied, (error) => + Ref.set(denied, true).pipe( + Effect.andThen( + Effect.logWarning(`keep-awake disabled until restart: ${error.message}`), + ), + ), + ), + retryWithBackoff("keep-awake sleep inhibitor stopped"), + Effect.ignore, + Effect.when(Effect.map(Ref.get(denied), (isDenied) => !isDenied)), + Effect.asVoid, + ), + ), + ); + export const layer = Layer.effectDiscard( Effect.gen(function* () { const platform = yield* HostProcess.Platform; @@ -238,10 +270,7 @@ export const layer = Layer.effectDiscard( stderr: (yield* Fiber.join(stderr)).trim(), }); }).pipe(Effect.scoped); - const hold = holdOnce.pipe( - retryWithBackoff("keep-awake sleep inhibitor stopped"), - Effect.ignore, - ); + const hold = yield* makeHold(holdOnce); yield* forkParked( holdWhileWanted( diff --git a/docs/progress.md b/docs/progress.md index c5b48cf633ec..71a49391f927 100644 --- a/docs/progress.md +++ b/docs/progress.md @@ -299,3 +299,11 @@ - Real run: a copy of the live database was taken through a read-only connection (`VACUUM INTO`, 370 MB, written under the worktree's ignored `.t3` because `/tmp` is tmpfs on hub2, and deleted after seeding), and `vp run migrate-dev-db --base-dir .t3/verify --source ` seeded a scratch data dir from it (65.6 MB, 5 projects, 29 threads), applying migrations 103-105 from earlier syncs. The server booted on that dir on port 16615, completed V2 orchestration recovery and answered HTTP 302. The harness stopped its process tree and the port was released. No agent turn was sent. The only warnings were the expected two: no Rust build for the resource monitor, and the scratch home inside the Git repo. - Switching the hubs: these 7 commits change no migrations, lockfile or package manifests; hub2's live database is still at 102, so its switch applies 103-105 from earlier syncs. - Next: Tomáš merges the PR, then switches hub2 and hub to the new build. Upstream main moved 3 commits during the checks (`11224f63d2`: #39822, #40154, #40703); they touch 8 files the fork does not change (the Antigravity provider, `ChatView.browser.test.tsx`, client-runtime `commands.ts`), merge without conflicts, and go in the next sync. + +## 2026-11-22: Keep-awake stops retrying when logind denies the sleep inhibitor + +- What: `KeepAwake.ts` now treats a `systemd-inhibit` exit whose stderr says "Access denied", "interactive authentication" or "not authorized" as permanent. The hold logs one warning without a stack (`keep-awake disabled until restart: ...`) and every later hold in that process does nothing. Other inhibitor failures keep the old retry with exponential backoff (5 s, capped at 10 min). `keepAwakeWhileAgentsWork`, the platform commands and run detection are unchanged. +- Why: the hubs run T3 as a systemd user service with no interactive session, so polkit refuses `--mode=block` every time. The old hold logged a full stack on every retry, forever: about 7,600 lines in hub's `boot-service.log` on 2026-11-19. +- Numbers: `KeepAwake.test.ts` gains two `makeHold` cases on the TestClock: a denied hold run twice across an hour spawns once and logs one warning with an empty cause; a plain non-zero exit retries at 5 s and again 10 s later. Both failed before the change (no `makeHold`), 3/3 pass after. +- What worked: pulling the retry policy into an exported `makeHold(holdOnce)` with a `denied` Ref, so the test drives it with a fake `holdOnce` instead of spawning `systemd-inhibit`. +- Next: Tomáš merges the PR and switches hub2 and hub to the new build; their logs should show one `keep-awake disabled until restart` line per server start instead of a stack every few minutes.