From 91a930c7d5a04609f57a21d596f18399a6da9b9c Mon Sep 17 00:00:00 2001 From: scgopi Date: Sat, 3 Oct 2026 08:52:25 -0700 Subject: [PATCH 1/2] Keep redialing a restarted codespace and bring back all its loops A codespace that stayed unreachable for four minutes paused every dial until a human selected one of its loops, so a restart that took longer left every loop on it dead. The pause now lets one dial through every five minutes, in the daemon and in an open pane. The dial that reaches the codespace again asks for the reboot probe, so finished loops are restored even with no pane open, and the liveness sweep now also re-ensures the workers of a piloted or armed composite. Co-Authored-By: Claude Opus 5.5 Signed-off-by: scgopi --- .../Domain/CodespaceDialSchedule.swift | 14 ++++-- .../Sources/Domain/SSHReconnectLoop.swift | 32 ++++++++------ GraphcodeKit/Sources/GraphStore.swift | 10 +++++ .../Sessions/CodespaceDialBreaker.swift | 39 ++++++++++++++--- .../Sources/Sessions/ZmxSessionLauncher.swift | 12 ++++++ .../Tests/CodespaceDialScheduleTests.swift | 43 ++++++++++++++++++- graphcode/Tests/CodespaceSelectionTests.swift | 25 +++++++++++ .../Tests/RemoteSessionResumeTests.swift | 32 ++++++++++++++ 8 files changed, 183 insertions(+), 24 deletions(-) diff --git a/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift b/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift index b71b86be..ab9cecc1 100644 --- a/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift +++ b/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift @@ -6,8 +6,11 @@ import Foundation /// human's per-user Codespaces rate limit — around a hundred while a codespace is /// starting. Retrying on each reader's own clock spent that limit during every outage, /// so all dialers share this one schedule, counted from the first failure: retry freely -/// for a minute, hold until the third, retry again until the fourth, then pause until a -/// human asks to reconnect. +/// for a minute, hold until the third, retry again until the fourth, then pause: one dial +/// every `slowRetryInterval` until the codespace answers, or at once when a human asks to +/// reconnect. The pause never ends in silence, because a codespace restarted from +/// outside graphcode can take longer than four minutes to come back, and its loops +/// should recover without a human finding them dead. /// /// The daemon applies it in `CodespaceDialBreaker`; a terminal pane applies the same /// numbers inside its shell loop (`SSHReconnectLoop`), which runs in another process. @@ -15,11 +18,16 @@ public struct CodespaceDialSchedule: Equatable, Sendable { public var freeRetryWindow: Int public var holdUntil: Int public var pauseAfter: Int + public var slowRetryInterval: Int - public init(freeRetryWindow: Int = 60, holdUntil: Int = 180, pauseAfter: Int = 240) { + public init( + freeRetryWindow: Int = 60, holdUntil: Int = 180, pauseAfter: Int = 240, + slowRetryInterval: Int = 300 + ) { self.freeRetryWindow = freeRetryWindow self.holdUntil = holdUntil self.pauseAfter = pauseAfter + self.slowRetryInterval = slowRetryInterval } public static let standard = CodespaceDialSchedule() diff --git a/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift b/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift index 3dc68e2a..c635a478 100644 --- a/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift +++ b/GraphcodeKit/Sources/Domain/SSHReconnectLoop.swift @@ -41,11 +41,12 @@ public enum SSHReconnectLoop { /// A Codespace surface's loop: the same dials and exit handling, retried on /// `CodespaceDialSchedule` instead of forever, because every gh run spends the human's - /// Codespaces rate limit (issue #480). Past `schedule.pauseAfter` it waits for Enter, - /// which also touches `pauseMarker` so `graphcoded`'s `CodespaceDialBreaker` resumes - /// the codespace's reads and ensures with it. The app touches the same marker when a - /// loop of this codespace is selected, which restarts the schedule and redials from any - /// wait — held or paused — within a second. + /// Codespaces rate limit (issue #480). Past `schedule.pauseAfter` it redials once per + /// `schedule.slowRetryInterval`, or at once on Enter, which also touches `pauseMarker` + /// so `graphcoded`'s `CodespaceDialBreaker` resumes the codespace's reads and ensures + /// with it. The app touches the same marker when a loop of this codespace is selected, + /// which restarts the schedule and redials from any wait — held or paused — within a + /// second. /// /// The outage clock restarts only after a dial that lasted `upAfter`, longer than the /// five minutes gh can spend waiting for a codespace to start before failing — a @@ -74,13 +75,15 @@ public enum SSHReconnectLoop { // does not count. let asked = "{ [ -e \"$gc_stamp\" ] && [ \(marker) -nt \"$gc_stamp\" ]; }" let enter = "mkdir -p \(directory) 2>/dev/null; touch \(marker) 2>/dev/null" - // On a tty the pause polls, so a selection can end it too. Off one, `read` blocks as - // it always did: bash 3.2's `read -t` answers 1 for a timeout and for end of input - // alike, and a closed stdin would spin. A pane always has a tty. + // On a tty the pause polls, so a selection can end it too, and runs out after the slow + // retry interval to redial on the same outage clock. Off one, `read` blocks as it + // always did: bash 3.2's `read -t` answers 1 for a timeout and for end of input alike, + // and a closed stdin would spin. A pane always has a tty. let pause = - "if [ -t 0 ]; then while :; do \(asked) && break; " - + "read -t 1 gc_line && { \(enter); break; }; done; " - + "else read gc_line || exit 0; \(enter); fi; " + "gc_ask=; if [ -t 0 ]; then gc_left=\(schedule.slowRetryInterval); " + + "while [ \"$gc_left\" -gt 0 ]; do \(asked) && { gc_ask=1; break; }; " + + "read -t 1 gc_line && { \(enter); gc_ask=1; break; }; gc_left=$((gc_left - 1)); done; " + + "else read gc_line || exit 0; \(enter); gc_ask=1; fi; " let waitOrAsk = "gc_wait_or_ask() { gc_left=$1; while [ \"$gc_left\" -gt 0 ]; do " + "\(asked) && return 0; sleep 1; gc_left=$((gc_left - 1)); done; return 1; }; " @@ -93,9 +96,10 @@ public enum SSHReconnectLoop { + "while :; do gc_out=$(($(date +%s) - gc_down)); " + "if [ \"$gc_out\" -ge \(schedule.pauseAfter) ]; then " + #"printf '\033[1;33m── Codespace still unreachable (exit %s). Paused to save your "# - + #"Codespaces API quota. Press Enter or select the loop to reconnect, Ctrl-C to "# - + #"close. ──\033[0m\r\n' "$gc_rc"; "# - + pause + "\(restart); " + + #"Codespaces API quota; retrying every %ss. Press Enter or select the loop to "# + + #"reconnect now, Ctrl-C to close. ──\033[0m\r\n' "$gc_rc" "# + + "\(schedule.slowRetryInterval); " + + pause + "[ -n \"$gc_ask\" ] && { \(restart); }; " + "else " + "if [ \"$gc_out\" -ge \(schedule.freeRetryWindow) ] " + "&& [ \"$gc_out\" -lt \(schedule.holdUntil) ]; then " diff --git a/GraphcodeKit/Sources/GraphStore.swift b/GraphcodeKit/Sources/GraphStore.swift index 77f124cf..3b7f7c63 100644 --- a/GraphcodeKit/Sources/GraphStore.swift +++ b/GraphcodeKit/Sources/GraphStore.swift @@ -4863,6 +4863,16 @@ public actor GraphStore { for node in graph.nodes where node.runsUnattended && !node.isResolved { ensureSession(node) } + // The children `pilotComposite` started live on the composite's sub-graph, not in + // `graph.nodes`, and a reboot kills their sessions just the same. + for composite in graph.nodes + where !composite.isResolved + && (composite.pilotState == .piloted || composite.pilotState == .armed) + { + for child in composite.subGraph?.nodes ?? [] where child.runsUnattended && !child.isResolved { + ensureSession(child) + } + } let finished = graph.nodes.filter { $0.runsUnattended && $0.isResolved && $0.launchFailure == nil } diff --git a/GraphcodeKit/Sources/Sessions/CodespaceDialBreaker.swift b/GraphcodeKit/Sources/Sessions/CodespaceDialBreaker.swift index 02146329..a1f47b1a 100644 --- a/GraphcodeKit/Sources/Sessions/CodespaceDialBreaker.swift +++ b/GraphcodeKit/Sources/Sessions/CodespaceDialBreaker.swift @@ -10,19 +10,30 @@ import Foundation /// newer than the outage clears it — holding or paused alike — so the next read or /// ensure dials at once. A file rather than a daemon command because the pane is a shell /// loop in the app's process, and a `stat` spends nothing. +/// +/// Paused, one dial per `slowRetryInterval` still goes through, whichever reader or +/// ensure asks first. The one that reaches the codespace clears the outage for all of +/// them and calls `onRecovered`, which is how the finished loops on the codespace get +/// restored as well as the running ones. public actor CodespaceDialBreaker { - static let shared = CodespaceDialBreaker() + static let shared = CodespaceDialBreaker(onRecovered: { location in + ZmxSessionLauncher.markRedialed(location) + }) private let schedule: CodespaceDialSchedule private let markerDirectory: URL + private let onRecovered: @Sendable (RemoteProjectLocation) -> Void private var downSince: [String: Date] = [:] + private var lastPausedDial: [String: Date] = [:] init( schedule: CodespaceDialSchedule = .standard, - markerDirectory: URL = CodespaceDialBreaker.defaultMarkerDirectory + markerDirectory: URL = CodespaceDialBreaker.defaultMarkerDirectory, + onRecovered: @escaping @Sendable (RemoteProjectLocation) -> Void = { _ in } ) { self.schedule = schedule self.markerDirectory = markerDirectory + self.onRecovered = onRecovered } public static var defaultMarkerDirectory: URL { @@ -60,21 +71,39 @@ public actor CodespaceDialBreaker { func permits(_ location: RemoteProjectLocation, now: Date = Date()) -> Bool { guard location.isCodespace, let since = downSince[location.host] else { return true } if reconnectRequested(for: location, after: since) { - downSince.removeValue(forKey: location.host) + clearOutage(location.host) + return true + } + switch schedule.verdict(secondsDown: now.timeIntervalSince(since)) { + case .dial: return true + case .hold: return false + case .paused: + let pausedAt = since.addingTimeInterval(TimeInterval(schedule.pauseAfter)) + let last = lastPausedDial[location.host] ?? pausedAt + guard now.timeIntervalSince(last) >= TimeInterval(schedule.slowRetryInterval) else { + return false + } + lastPausedDial[location.host] = now return true } - return schedule.verdict(secondsDown: now.timeIntervalSince(since)) == .dial } func record(_ location: RemoteProjectLocation, reached: Bool, now: Date = Date()) { guard location.isCodespace else { return } if reached { - downSince.removeValue(forKey: location.host) + guard downSince[location.host] != nil else { return } + clearOutage(location.host) + onRecovered(location) } else if downSince[location.host] == nil { downSince[location.host] = now } } + private func clearOutage(_ host: String) { + downSince.removeValue(forKey: host) + lastPausedDial.removeValue(forKey: host) + } + private func reconnectRequested(for location: RemoteProjectLocation, after since: Date) -> Bool { let marker = Self.reconnectMarker(for: location, in: markerDirectory) guard diff --git a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift index f50be696..34310aae 100644 --- a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift +++ b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift @@ -2552,6 +2552,18 @@ public enum ZmxSessionLauncher { .appendingPathComponent("\(location.host).redial") } + /// Touches `redialStamp` from the daemon: a codespace that just answered after an + /// outage may have restarted under finished loops whose panes are closed, and nothing + /// else would ask `restoreRebootedRemote` to probe it. + static func markRedialed(_ location: RemoteProjectLocation) { + let stamp = redialStamp(for: location) + try? FileManager.default.createDirectory( + at: stamp.deletingLastPathComponent(), withIntermediateDirectories: true) + FileManager.default.createFile(atPath: stamp.path, contents: nil) + try? FileManager.default.setAttributes( + [.modificationDate: Date()], ofItemAtPath: stamp.path) + } + /// Whether a host's panes have redialed since its last answered probe — the only /// state `restoreRebootedRemote` keeps. Stamps from before this daemon started count /// once, so a pane left waiting across a daemon restart is still answered. diff --git a/graphcode/Tests/CodespaceDialScheduleTests.swift b/graphcode/Tests/CodespaceDialScheduleTests.swift index 15f31444..dde36c9e 100644 --- a/graphcode/Tests/CodespaceDialScheduleTests.swift +++ b/graphcode/Tests/CodespaceDialScheduleTests.swift @@ -1,10 +1,11 @@ +import ComposableArchitecture import Foundation import Testing @testable import GraphcodeKit /// One outage schedule for every Codespace dialer (issue #480): retry freely for a -/// minute, hold until the third, retry until the fourth, then pause until a human asks. +/// minute, hold until the third, retry until the fourth, then pause to a slow retry. /// Every gh run spends the human's Codespaces rate limit, so a dialer that retried on /// its own clock forever spent it during every outage. /// @@ -91,7 +92,45 @@ struct CodespaceDialScheduleTests { #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(90))) #expect(await breaker.permits(codespace, now: down.addingTimeInterval(200))) #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(250))) - #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(86_400))) + } + + @Test + func aPausedCodespaceIsStillDialedOncePerSlowInterval() async throws { + // A codespace restarted from outside graphcode can take longer than the four minutes + // before the pause; its loops have to come back without a human selecting one. + let breaker = CodespaceDialBreaker(markerDirectory: try scratch()) + let down = Date(timeIntervalSince1970: 1_000_000) + await breaker.record(codespace, reached: false, now: down) + + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(539))) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(540))) + // One dial per interval for the whole codespace, not one per reader or loop. + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(540))) + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(839))) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(840))) + // A failed slow dial leaves the outage clock where it was. + await breaker.record(codespace, reached: false, now: down.addingTimeInterval(845)) + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(900))) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(86_400))) + } + + @Test + func aCodespaceThatAnswersAfterAnOutageAsksForTheRebootProbe() async throws { + let recovered = LockIsolated<[String]>([]) + let breaker = CodespaceDialBreaker( + markerDirectory: try scratch(), + onRecovered: { location in recovered.withValue { $0.append(location.host) } }) + let down = Date(timeIntervalSince1970: 1_000_000) + + await breaker.record(codespace, reached: true, now: down) + #expect(recovered.value.isEmpty) + + await breaker.record(codespace, reached: false, now: down) + await breaker.record(codespace, reached: true, now: down.addingTimeInterval(600)) + await breaker.record(codespace, reached: true, now: down.addingTimeInterval(601)) + + #expect(recovered.value == [codespace.host]) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(602))) } @Test diff --git a/graphcode/Tests/CodespaceSelectionTests.swift b/graphcode/Tests/CodespaceSelectionTests.swift index a6178d13..d4eaa529 100644 --- a/graphcode/Tests/CodespaceSelectionTests.swift +++ b/graphcode/Tests/CodespaceSelectionTests.swift @@ -212,4 +212,29 @@ struct CodespaceSelectionTests { #expect(await waitFor(within: .seconds(5)) { lines(in: log) > pausedAt }) } + + @Test + func aPausedPaneRedialsOnItsOwnAfterTheSlowInterval() async throws { + let directory = try scratch() + defer { try? FileManager.default.removeItem(at: directory) } + let log = directory.appendingPathComponent("dials") + let marker = directory.appendingPathComponent("space.reconnect") + let dial = "(echo dial >> \(RemoteProjectLocation.shellQuoted(log.path)); exit 1)" + let slow = CodespaceDialSchedule( + freeRetryWindow: 1, holdUntil: 3, pauseAfter: 4, slowRetryInterval: 3) + let pane = try startPane( + SSHReconnectLoop.codespaceScript( + connect: dial, reconnect: dial, pauseMarker: marker.path, schedule: slow), + onATTY: true) + defer { pane.process.terminate() } + + #expect(await waitFor { lines(in: log) >= 3 }) + try await Task.sleep(for: .seconds(5)) + let pausedAt = lines(in: log) + + // Nobody presses Enter or selects the loop. + #expect(await waitFor(within: .seconds(10)) { lines(in: log) >= pausedAt + 2 }) + #expect(pane.process.isRunning) + #expect(!FileManager.default.fileExists(atPath: marker.path)) + } } diff --git a/graphcode/Tests/RemoteSessionResumeTests.swift b/graphcode/Tests/RemoteSessionResumeTests.swift index d904bf6d..fa33e6a6 100644 --- a/graphcode/Tests/RemoteSessionResumeTests.swift +++ b/graphcode/Tests/RemoteSessionResumeTests.swift @@ -466,6 +466,38 @@ struct RemoteSessionResumeTests { #expect(started.value.isEmpty) } + + @Test + func theLivenessSweepRestartsTheLoopsInsideARunningComposite() async { + // A codespace restart kills a piloted composite's workers with everything else, and + // they live on its sub-graph, where a sweep of `graph.nodes` never looked. + let started = LockIsolated<[UUID]>([]) + let worker = LoopNode( + title: "Worker", loopType: .timeBased, triggerPrompt: "/loop 1h Check") + let finishedWorker = LoopNode( + title: "Done", loopType: .goalBased, goal: GoalSpec(summary: "ship"), state: .succeeded) + let reviewer = LoopNode(title: "Review", loopType: .turnBased, checkDescription: "Sound?") + let piloted = LoopNode( + title: "Routine", loopType: .composite, + subGraph: LoopGraph( + project: ProjectRef(path: "sub", name: "sub"), + nodes: [worker, finishedWorker, reviewer]), + pilotState: .piloted) + let draftWorker = LoopNode( + title: "Draft worker", loopType: .timeBased, triggerPrompt: "/loop 1h Draft") + let draft = LoopNode( + title: "Draft", loopType: .composite, + subGraph: LoopGraph(project: ProjectRef(path: "draft", name: "draft"), nodes: [draftWorker])) + let store = GraphStore( + graph: LoopGraph( + scope: LoopGraphScope(projectPath: location.projectPath, name: "widget"), + nodes: [piloted, draft]), + onEnsureSession: { node, _ in started.withValue { $0.append(node.id) } }) + + await store.ensureUnattendedSessionsAlive() + + #expect(started.value == [worker.id]) + } } /// A finished unattended loop across a remote reboot: its session comes back as the From 9936b0e3501ede498d197f5a9094c8acf3233d54 Mon Sep 17 00:00:00 2001 From: scgopi Date: Sat, 3 Oct 2026 09:02:20 -0700 Subject: [PATCH 2/2] Retry a paused codespace every minute and sweep ssh hosts every 30s The paused codespace now redials once a minute instead of every five. Plain ssh hosts are swept every 30 seconds, and their finished loops are probed on every sweep rather than only after a pane redials: their dials ride the ControlMaster and spend no quota. Codespaces stay on a minute. Co-Authored-By: Claude Opus 5.5 Signed-off-by: scgopi --- .../Domain/CodespaceDialSchedule.swift | 2 +- GraphcodeKit/Sources/ProjectRegistry.swift | 19 ++++++++-- .../Sources/Sessions/ZmxSessionLauncher.swift | 15 +++++--- .../Tests/CodespaceDialScheduleTests.swift | 23 ++++++------ .../Tests/RemoteSessionResumeTests.swift | 37 ++++++++++++++++--- 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift b/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift index ab9cecc1..5505a4f1 100644 --- a/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift +++ b/GraphcodeKit/Sources/Domain/CodespaceDialSchedule.swift @@ -22,7 +22,7 @@ public struct CodespaceDialSchedule: Equatable, Sendable { public init( freeRetryWindow: Int = 60, holdUntil: Int = 180, pauseAfter: Int = 240, - slowRetryInterval: Int = 300 + slowRetryInterval: Int = 60 ) { self.freeRetryWindow = freeRetryWindow self.holdUntil = holdUntil diff --git a/GraphcodeKit/Sources/ProjectRegistry.swift b/GraphcodeKit/Sources/ProjectRegistry.swift index f50c282b..0b5feba8 100644 --- a/GraphcodeKit/Sources/ProjectRegistry.swift +++ b/GraphcodeKit/Sources/ProjectRegistry.swift @@ -353,15 +353,20 @@ public actor ProjectRegistry { /// dial is a bare `zmx get` — `remoteEnsureInvocation` keeps the hooks write and the /// file delivery behind that check precisely so this can be cheap — multiplexed onto /// the host's existing `ControlMaster` connection. - static let remoteLivenessSweepInterval: Duration = .seconds(60) + /// + /// A codespace is swept on every `codespaceSweepTicks`th tick only: any of its dials can + /// fall back to gh and spend the human's Codespaces API quota (issue #480). + static let remoteLivenessSweepInterval: Duration = .seconds(30) + static let codespaceSweepTicks = 2 - /// Generous next to the remote sweep's minute: on a healthy machine the condemned + /// Generous next to the remote sweep's interval: on a healthy machine the condemned /// list is empty and a tick is one file read, but a tick that finds work spawns /// processes, and a session that survived three confirmed kill attempts is not going /// to die to a faster clock. static let condemnedReapInterval: Duration = .seconds(300) private var remoteSweeper: Task? + private var remoteSweepTick = 0 private var condemnedReaper: Task? /// Started by the first remote project this daemon loads and left running: a store is @@ -393,11 +398,19 @@ public actor ProjectRegistry { /// `RemoteEnsureGate`: one ensure per node at a time, so a slow tick cannot pile a /// second dial onto the same session. private func sweepRemoteSessions() async { - for (path, store) in stores where RemoteProjectLocation.parse(projectPath: path) != nil { + remoteSweepTick += 1 + for (path, store) in stores { + guard let location = RemoteProjectLocation.parse(projectPath: path), + Self.sweeps(location, onTick: remoteSweepTick) + else { continue } await store.ensureUnattendedSessionsAlive() } } + static func sweeps(_ location: RemoteProjectLocation, onTick tick: Int) -> Bool { + !location.isCodespace || tick % codespaceSweepTicks == 0 + } + /// Takes or drops the sleep assertion to match what is running right now /// (`AwakeAssertion`), across every open project. /// diff --git a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift index 34310aae..be4a2d37 100644 --- a/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift +++ b/GraphcodeKit/Sources/Sessions/ZmxSessionLauncher.swift @@ -2515,10 +2515,11 @@ public enum ZmxSessionLauncher { /// sessions that are missing *and* were last seen alive in an earlier boot; only those /// are dialed again, each behind the same boot gate. /// - /// The probe itself runs only when a pane of that host has redialed since the last - /// probe that answered (`redialStamp`): the one thing left dialing a finished loop's - /// host is its pane, and a healthy host has no pane redialing, so the sweep spends - /// nothing — a codespace dial spends the human's API quota (issue #480). + /// On a codespace the probe runs only when a pane of that host has redialed since the + /// last probe that answered (`redialStamp`), or the codespace answered after an outage: + /// a codespace dial spends the human's API quota (issue #480). A plain ssh host is + /// probed on every sweep, multiplexed over its `ControlMaster`, so its finished loops + /// come back with no pane open. /// /// `nodes` are already the quiet copies the store made (`GraphStore.rebootRestoreCopy`): /// the create resumes the banked conversation, or opens on a note, never on the task. @@ -2565,8 +2566,9 @@ public enum ZmxSessionLauncher { } /// Whether a host's panes have redialed since its last answered probe — the only - /// state `restoreRebootedRemote` keeps. Stamps from before this daemon started count - /// once, so a pane left waiting across a daemon restart is still answered. + /// state `restoreRebootedRemote` keeps, and always yes for a plain ssh host. Stamps from + /// before this daemon started count once, so a pane left waiting across a daemon + /// restart is still answered. actor RebootProbeGate { static let shared = RebootProbeGate() @@ -2582,6 +2584,7 @@ public enum ZmxSessionLauncher { } func panesRedialed(_ location: RemoteProjectLocation) -> Bool { + guard location.isCodespace else { return true } guard let touched = (try? FileManager.default.attributesOfItem( diff --git a/graphcode/Tests/CodespaceDialScheduleTests.swift b/graphcode/Tests/CodespaceDialScheduleTests.swift index dde36c9e..e143c09a 100644 --- a/graphcode/Tests/CodespaceDialScheduleTests.swift +++ b/graphcode/Tests/CodespaceDialScheduleTests.swift @@ -77,6 +77,7 @@ struct CodespaceDialScheduleTests { #expect(schedule.verdict(secondsDown: 239) == .dial) #expect(schedule.verdict(secondsDown: 240) == .paused) #expect(schedule.verdict(secondsDown: 86_400) == .paused) + #expect(schedule.slowRetryInterval == 60) } // MARK: - The daemon's breaker @@ -102,15 +103,15 @@ struct CodespaceDialScheduleTests { let down = Date(timeIntervalSince1970: 1_000_000) await breaker.record(codespace, reached: false, now: down) - #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(539))) - #expect(await breaker.permits(codespace, now: down.addingTimeInterval(540))) + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(299))) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(300))) // One dial per interval for the whole codespace, not one per reader or loop. - #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(540))) - #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(839))) - #expect(await breaker.permits(codespace, now: down.addingTimeInterval(840))) + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(300))) + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(359))) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(360))) // A failed slow dial leaves the outage clock where it was. - await breaker.record(codespace, reached: false, now: down.addingTimeInterval(845)) - #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(900))) + await breaker.record(codespace, reached: false, now: down.addingTimeInterval(365)) + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(400))) #expect(await breaker.permits(codespace, now: down.addingTimeInterval(86_400))) } @@ -167,13 +168,13 @@ struct CodespaceDialScheduleTests { FileManager.default.createFile(atPath: marker.path, contents: nil) try FileManager.default.setAttributes( [.modificationDate: down.addingTimeInterval(-60)], ofItemAtPath: marker.path) - #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(300))) + #expect(await !breaker.permits(codespace, now: down.addingTimeInterval(250))) try FileManager.default.setAttributes( - [.modificationDate: down.addingTimeInterval(299)], ofItemAtPath: marker.path) - #expect(await breaker.permits(codespace, now: down.addingTimeInterval(300))) + [.modificationDate: down.addingTimeInterval(249)], ofItemAtPath: marker.path) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(250))) // Resumed, not a one-off: the next read goes through too. - #expect(await breaker.permits(codespace, now: down.addingTimeInterval(301))) + #expect(await breaker.permits(codespace, now: down.addingTimeInterval(251))) } @Test diff --git a/graphcode/Tests/RemoteSessionResumeTests.swift b/graphcode/Tests/RemoteSessionResumeTests.swift index fa33e6a6..8ecf0b84 100644 --- a/graphcode/Tests/RemoteSessionResumeTests.swift +++ b/graphcode/Tests/RemoteSessionResumeTests.swift @@ -649,25 +649,52 @@ struct RemoteRebootRestoreTests { } @Test - func aHealthyHostIsNeverProbed() async throws { + func aHealthyCodespaceIsNeverProbed() async throws { // The probe is a dial, and on a codespace a dial spends the human's API quota. Only // a pane redialing its host is worth one; a host nobody is redialing costs nothing. + let codespace = RemoteProjectLocation( + host: "fluffy-space-waddle", remotePath: "/workspaces/widget", isCodespace: true) let stamp = FileManager.default.temporaryDirectory .appendingPathComponent("redial-\(UUID().uuidString)") defer { try? FileManager.default.removeItem(at: stamp) } let gate = ZmxSessionLauncher.RebootProbeGate(stampFor: { _ in stamp }) - #expect(await !gate.panesRedialed(location)) + #expect(await !gate.panesRedialed(codespace)) FileManager.default.createFile(atPath: stamp.path, contents: nil) - #expect(await gate.panesRedialed(location)) + #expect(await gate.panesRedialed(codespace)) - await gate.probed(location, at: Date().addingTimeInterval(1)) - #expect(await !gate.panesRedialed(location)) + await gate.probed(codespace, at: Date().addingTimeInterval(1)) + #expect(await !gate.panesRedialed(codespace)) try FileManager.default.setAttributes( [.modificationDate: Date().addingTimeInterval(5)], ofItemAtPath: stamp.path) + #expect(await gate.panesRedialed(codespace)) + } + + @Test + func aPlainSSHHostIsProbedOnEverySweepWithNoPaneOpen() async { + // Its dials ride the host's ControlMaster and spend no quota, so finished loops on a + // rebooted ssh host come back without waiting for a pane to redial. + let stamp = FileManager.default.temporaryDirectory + .appendingPathComponent("redial-\(UUID().uuidString)") + let gate = ZmxSessionLauncher.RebootProbeGate(stampFor: { _ in stamp }) + #expect(await gate.panesRedialed(location)) + await gate.probed(location, at: Date().addingTimeInterval(1)) + #expect(await gate.panesRedialed(location)) + } + + @Test + func plainSSHHostsAreSweptEveryThirtySecondsAndCodespacesEveryMinute() { + let codespace = RemoteProjectLocation( + host: "fluffy-space-waddle", remotePath: "/workspaces/widget", isCodespace: true) + + #expect(ProjectRegistry.remoteLivenessSweepInterval == .seconds(30)) + #expect( + (1...4).map { ProjectRegistry.sweeps(location, onTick: $0) } == [true, true, true, true]) + #expect( + (1...4).map { ProjectRegistry.sweeps(codespace, onTick: $0) } == [false, true, false, true]) } @Test(arguments: [false, true])