Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 115 additions & 1 deletion desktop/src/main/__tests__/watcher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ function makeWatcher(
runProviderList: () => Promise<Record<string, unknown>>,
workspaces: Array<{ id: string; status?: string }> = [],
runRaw: (args: string[]) => Promise<unknown> = async () => ({}),
getMainWindow: () => unknown = () => null,
) {
const state = {
updateProviders: vi.fn().mockReturnValue(false),
Expand Down Expand Up @@ -43,7 +44,7 @@ function makeWatcher(
const watcher = new Watcher({
cli: cli as never,
state: state as never,
getMainWindow: () => null,
getMainWindow: getMainWindow as never,
providerJobs: providerJobs as never,
workspaceJobs: workspaceJobs as never,
})
Expand Down Expand Up @@ -531,3 +532,116 @@ describe("Watcher workspace list freshness", () => {
expect(daemonList).toHaveBeenCalledExactlyOnceWith({ fresh: false })
})
})

describe("Watcher scheduled status polling", () => {
const visibleWindow = {
isDestroyed: () => false,
isVisible: () => true,
isMinimized: () => false,
}
const trayWindow = { ...visibleWindow, isVisible: () => false }
const scheduled = (watcher: Watcher) =>
(
watcher as unknown as { scheduleWorkspaceStatusPoll: () => void }
).scheduleWorkspaceStatusPoll()
const settle = () => new Promise((resolve) => setTimeout(resolve, 10))

it("skips scheduled status polls while the window is hidden", async () => {
const { watcher, cli } = makeWatcher(
async () => ({}),
[{ id: "ws-1" }],
async () => JSON.stringify({ state: "running" }),
() => trayWindow,
)

scheduled(watcher)
await settle()

expect(cli.runRaw).not.toHaveBeenCalled()
})

it("backs off a failing workspace on scheduled polls only", async () => {
const { watcher, cli } = makeWatcher(
async () => ({}),
[{ id: "ws-1" }],
async () => {
throw new Error("offline")
},
() => visibleWindow,
)

scheduled(watcher)
await settle()
scheduled(watcher)
await settle()
expect(cli.runRaw).toHaveBeenCalledTimes(1)

await watcher.refreshWorkspaceStatuses()
expect(cli.runRaw).toHaveBeenCalledTimes(2)
})

it("resumes scheduled polls after the backoff expires", async () => {
const { watcher, cli } = makeWatcher(
async () => ({}),
[{ id: "ws-1" }],
async () => {
throw new Error("offline")
},
() => visibleWindow,
)

scheduled(watcher)
await settle()
expect(cli.runRaw).toHaveBeenCalledTimes(1)

const now = Date.now()
const nowSpy = vi.spyOn(Date, "now").mockReturnValue(now + 6 * 60_000)
scheduled(watcher)
await settle()
nowSpy.mockRestore()

expect(cli.runRaw).toHaveBeenCalledTimes(2)
})

it("drops backoff for workspaces that leave the list", async () => {
const { watcher, cli } = makeWatcher(
async () => ({}),
[{ id: "ws-1" }],
async () => {
throw new Error("offline")
},
() => visibleWindow,
)

scheduled(watcher)
await settle()
await watcher.refreshWorkspaces()
scheduled(watcher)
await settle()

expect(cli.runRaw).toHaveBeenCalledTimes(2)
})

it("does not rerun immediately when a tick overlaps a running sweep", async () => {
let release: () => void = () => {}
const gate = new Promise<void>((resolve) => {
release = resolve
})
const { watcher, cli } = makeWatcher(
async () => ({}),
[{ id: "ws-1" }],
async () => {
await gate
return JSON.stringify({ state: "running" })
},
() => visibleWindow,
)

scheduled(watcher)
scheduled(watcher)
release()
await settle()

expect(cli.runRaw).toHaveBeenCalledTimes(1)
})
})
85 changes: 63 additions & 22 deletions desktop/src/main/watcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ import type { DaemonState, Machine, Workspace } from "./state.js"
import type { WorkspaceJobs } from "./workspace-jobs.js"
import { normalizeWorkspaceStatus } from "./workspace-status.js"

const WORKSPACE_POLL_INTERVAL_MS = 3000
const WORKSPACE_STATUS_INTERVAL_MS = 10_000
const STATUS_BACKOFF_MAX_MS = 300_000
interface WatcherDeps {
cli: CliRunner
daemon?: DaemonClient
Expand Down Expand Up @@ -81,15 +84,18 @@ export class Watcher {
private workspacePollChain: Promise<void> = Promise.resolve()
private workspaceStatusChain: Promise<void> = Promise.resolve()
private workspaceStatusPolling = false
private workspaceStatusQueued = false
private statusFailures = new Map<string, { count: number; retryAt: number }>()
Comment thread
skevetter marked this conversation as resolved.
Resolved

constructor(private deps: WatcherDeps) {}

start(): Promise<void> {
this.pollTimer = setInterval(() => this.schedulePoll(), 3000)
this.pollTimer = setInterval(() => {
if (this.windowActive()) this.schedulePoll()
else this.retryReconcilingJobs()
}, WORKSPACE_POLL_INTERVAL_MS)
Comment thread
skevetter marked this conversation as resolved.
Resolved
this.workspaceStatusTimer = setInterval(
() => this.scheduleWorkspaceStatusPoll(),
10_000,
WORKSPACE_STATUS_INTERVAL_MS,
)

const devsyDir = join(homedir(), ".devsy")
Expand All @@ -116,7 +122,6 @@ export class Watcher {
clearInterval(this.workspaceStatusTimer)
this.workspaceStatusTimer = null
}
this.workspaceStatusQueued = false
if (this.fsWatcher) {
this.fsWatcher.close()
this.fsWatcher = null
Expand All @@ -140,12 +145,7 @@ export class Watcher {
this.pollMachines(),
this.pollContexts(),
])
for (const [id, job] of Object.entries(
this.deps.workspaceJobs.snapshot(),
)) {
if (job.state === "reconciling" && job.refreshError)
void this.deps.workspaceJobs.retryRefresh(id)
}
this.retryReconcilingJobs()
} finally {
this.polling = false
if (this.pollQueued) {
Expand All @@ -155,6 +155,15 @@ export class Watcher {
}
}

private retryReconcilingJobs(): void {
for (const [id, job] of Object.entries(
this.deps.workspaceJobs.snapshot(),
)) {
if (job.state === "reconciling" && job.refreshError)
void this.deps.workspaceJobs.retryRefresh(id)
}
}

private async queryWithFallback<T>(
daemonFn: (() => Promise<T>) | undefined,
cliFn: () => Promise<T>,
Expand Down Expand Up @@ -182,34 +191,61 @@ export class Watcher {
await this.queueWorkspaceStatusPoll()
}

private windowActive(): boolean {
const win = this.deps.getMainWindow()
return (
win !== null &&
!win.isDestroyed() &&
win.isVisible() &&
!win.isMinimized()
)
}

private scheduleWorkspaceStatusPoll(): void {
if (this.workspaceStatusPolling) {
this.workspaceStatusQueued = true
return
}
void this.refreshWorkspaceStatuses()
if (this.workspaceStatusPolling || !this.windowActive()) return
this.workspaceStatusPolling = true
void this.queueWorkspaceStatusPoll(undefined, false, true)
}

private queueWorkspaceStatusPoll(
workspaceIds?: string[],
strict = false,
scheduled = false,
): Promise<void> {
const run = this.workspaceStatusChain.then(() =>
this.pollWorkspaceStatuses(workspaceIds, strict),
this.pollWorkspaceStatuses(workspaceIds, strict, scheduled),
)
this.workspaceStatusChain = run.catch(() => {})
return run
}

private statusBackoffElapsed(id: string): boolean {
const failure = this.statusFailures.get(id)
return !failure || Date.now() >= failure.retryAt
}

private recordStatusFailure(id: string): void {
const count = (this.statusFailures.get(id)?.count ?? 0) + 1
const delay = Math.min(
STATUS_BACKOFF_MAX_MS,
WORKSPACE_STATUS_INTERVAL_MS * 2 ** count,
)
this.statusFailures.set(id, { count, retryAt: Date.now() + delay })
}

private async pollWorkspaceStatuses(
workspaceIds?: string[],
strict = false,
scheduled = false,
): Promise<void> {
this.workspaceStatusPolling = true
let changed = false
try {
const ids =
const requested =
workspaceIds ?? this.deps.state.workspaceList().map((ws) => ws.id)
const ids = scheduled
? requested.filter((id) => this.statusBackoffElapsed(id))
: requested
const concurrency = 6
let next = 0
const worker = async (): Promise<void> => {
Expand All @@ -234,12 +270,13 @@ export class Watcher {
const status = normalizeWorkspaceStatus(raw)
if (!status && strict)
throw new Error("Workspace status unavailable")
if (status) this.statusFailures.delete(id)
if (status && this.deps.state.updateWorkspaceStatus(id, status)) {
changed = true
}
} catch (error) {
if (strict) throw error
// Preserve the last known status and retry on the next sweep.
this.recordStatusFailure(id)
}
}
}
Expand All @@ -249,10 +286,6 @@ export class Watcher {
if (changed) this.broadcastWorkspaces()
} finally {
this.workspaceStatusPolling = false
if (this.workspaceStatusQueued) {
this.workspaceStatusQueued = false
void this.refreshWorkspaceStatuses()
}
}
}

Expand Down Expand Up @@ -282,6 +315,7 @@ export class Watcher {
throw new Error("Workspace operation changed during refresh")
return
}
this.pruneStatusFailures(workspaces)
const changed = this.deps.state.updateWorkspaces(workspaces)
if (changed) {
this.broadcastWorkspaces()
Expand All @@ -294,6 +328,13 @@ export class Watcher {
}
}

private pruneStatusFailures(workspaces: Workspace[]): void {
const listed = new Set(workspaces.map((ws) => ws.id))
for (const id of this.statusFailures.keys()) {
if (!listed.has(id)) this.statusFailures.delete(id)
}
}

/**
* Push the workspace list plus in-flight job state. Sent on one channel so
* the two can't arrive out of order and show a deleted-but-still-listed
Expand Down
Loading