From 18769977d2d085f4f46d124eb7b6919447721e6c Mon Sep 17 00:00:00 2001 From: me2seeks Date: Fri, 4 Sep 2026 18:44:26 +0800 Subject: [PATCH] refactor(runtime-host): split residency kinds and give idle visibility to handshakes The never-released process-retention marker blocked graceful close: the drain waited for every residency, so a poisoned Host always fell through to the shutdown deadline and never released the writer lease. Residencies now split into drain (work in flight; blocks idle exit and must settle before close) and idle (markers that only block idle exit). The maintenance probe no longer filters the marker label out. The idle timer and the local-owner true-idle takeover were blind to in-flight handshakes after the first accepted connection, so the Host could exit under a connecting Client. #isTrueIdle now counts handshaking transports, the idle timer re-evaluates when a handshake settles, and the takeover excludes only the transport being admitted. The replacement advice in rejections keeps ignoring handshakes: it is what a stale Client acts on, and millisecond probes must not flip it. Closes #4760. Generated-by: Maka --- .../src/__tests__/host-kernel.test.ts | 116 +++++++++++++++--- .../__tests__/host-residency-registry.test.ts | 52 ++++++++ .../runtime-host/src/server/host-kernel.ts | 41 +++++-- .../src/server/host-residency-registry.ts | 68 +++++++--- 4 files changed, 234 insertions(+), 43 deletions(-) diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 2e330f305a..b3c6ff9d85 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -1268,7 +1268,7 @@ describe('non-serving Runtime Host kernel', () => { }); }); - test('process-exit retention closes admission before requiring termination without releasing ownership', async () => { + test('process-exit retention neither stalls the graceful close nor retains ownership', async () => { await withHostPaths(async (paths) => { const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); const owner = await tryAcquireInteractiveRootOwner(capability); @@ -1276,7 +1276,7 @@ describe('non-serving Runtime Host kernel', () => { const host = await RuntimeHostKernel.start({ owner, idleGraceMs: 10_000, - shutdownGraceMs: 50, + shutdownGraceMs: 10_000, composition: defineInteractiveRuntimeHostComposition(async (context) => { context.retainUntilProcessExit(); context.retainUntilProcessExit(); @@ -1286,11 +1286,12 @@ describe('non-serving Runtime Host kernel', () => { }); try { - await assert.rejects( + // The anti-idle marker is not work: the drain it accompanies closes + // gracefully, long before the shutdown deadline. + await withTimeout( host.closed, - (error: unknown) => - error instanceof RuntimeHostProcessTerminationRequiredError && - error.code === 'process_termination_required', + 2_000, + 'retained Host waited out its shutdown deadline instead of closing gracefully', ); await assert.rejects( () => openSocket(host.endpoint), @@ -1300,7 +1301,9 @@ describe('non-serving Runtime Host kernel', () => { ((error as NodeJS.ErrnoException).code === 'ENOENT' || (error as NodeJS.ErrnoException).code === 'ECONNREFUSED'), ); - assert.equal(await tryAcquireInteractiveRootOwner(capability), undefined); + const successor = await tryAcquireInteractiveRootOwner(capability); + assert.ok(successor, 'graceful close must release the State Root writer lease'); + await successor?.close(); } finally { await owner.close(); } @@ -1405,6 +1408,75 @@ describe('non-serving Runtime Host kernel', () => { }); }); + test('an in-flight handshake keeps an ephemeral Host alive past the idle deadline', async () => { + await withHostPaths(async (paths) => { + const candidate = await startTestRuntimeHostCandidate(paths, { + rootPath: paths.root, + idleGraceMs: 250, + initialConnectionTimeoutMs: 5_000, + handshakeTimeoutMs: 5_000, + }); + assert.equal(candidate.kind, 'winner'); + if (candidate.kind !== 'winner') return; + const host = candidate.host; + + // The first accepted connection leaves and the idle timer arms; a + // handshake that begins now is the phase the idle timer used to be + // blind to. + const first = await retryConnect(paths, CURRENT_PROTOCOL); + assert.equal(first.kind, 'connected'); + if (first.kind !== 'connected') return; + await first.connection.close(); + + const silent = await openSocket(host.endpoint); + await new Promise((resolve) => setTimeout(resolve, 50)); + try { + // Past the idle deadline with the handshake in flight: the Host must + // not drain under a connecting Client. + await new Promise((resolve) => setTimeout(resolve, 500)); + assert.equal(host.state, 'ready'); + } finally { + silent.destroy(); + } + // Once the handshake settles, the idle timer re-arms and the Host exits. + await withTimeout( + host.closed, + 5_000, + 'ephemeral Host never idle-exited after the handshake settled', + ); + }); + }); + + test('a poisoned Host closes gracefully without waiting out the shutdown deadline', async () => { + await withHostPaths(async (paths) => { + const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + const host = await RuntimeHostKernel.start({ + owner, + lifecycleMode: 'service', + shutdownGraceMs: 10_000, + composition: defineInteractiveRuntimeHostComposition(async (context) => { + // Mirror the poison/fatal path: the anti-idle marker must not stall + // the drain it accompanies. + context.retainUntilProcessExit(); + context.requestDrain(); + return { + handlers: createUnavailableDomainOperationHandlers(), + beginDrain() {}, + async recover() {}, + async close() {}, + }; + }), + }); + await withTimeout( + host.closed, + 2_000, + 'poisoned Host waited out its shutdown deadline instead of closing gracefully', + ); + }); + }); + test('drain requested before factory completion begins drain before recovery exactly once', async () => { await withHostPaths(async (paths) => { const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' }); @@ -1591,18 +1663,26 @@ describe('non-serving Runtime Host kernel', () => { staleWhileResident.abort(); await staleWhileResident.closed; - const blocked = await connectOrSpawnRuntimeHost({ - ...paths, - rootPath: paths.root, - protocol: LEGACY_PROTOCOL, - compositionId: KERNEL_COMPOSITION.descriptor.id, - candidateEntrypoint: KERNEL_CANDIDATE_ENTRYPOINT, - electionDeadlineMs: 2_000, - }); - assert.equal(blocked.kind, 'incompatible'); - if (blocked.kind === 'incompatible') { - assert.equal(blocked.handshake.replacement, 'blocked_by_residency'); + const blockedWhileResident = new FramedTransport(await openSocket(candidate.host.endpoint)); + await writeRawLocalIpc( + blockedWhileResident, + encodeLegacyProtocolFrame({ + kind: 'hello', + clientInstanceId: 'blocked-legacy-resident', + protocolMin: LEGACY_PROTOCOL.min, + protocolMax: LEGACY_PROTOCOL.max, + }), + ); + const blockedResponse = decodeHostFrame(await blockedWhileResident.read(1_000)); + assert.ok('kind' in blockedResponse && blockedResponse.kind === 'incompatible'); + if ('kind' in blockedResponse && blockedResponse.kind === 'incompatible') { + assert.equal(blockedResponse.replacement, 'blocked_by_residency'); } + blockedWhileResident.abort(); + await blockedWhileResident.closed; + // The rejected handshake's teardown is asynchronous Host-side; let it + // settle so only the next probe's own handshake remains in flight. + await sleep(50); await resident.connection.close(); const staleAtIdle = new FramedTransport(await openSocket(candidate.host.endpoint)); diff --git a/packages/runtime-host/src/__tests__/host-residency-registry.test.ts b/packages/runtime-host/src/__tests__/host-residency-registry.test.ts index 09130ed308..bb914b9381 100644 --- a/packages/runtime-host/src/__tests__/host-residency-registry.test.ts +++ b/packages/runtime-host/src/__tests__/host-residency-registry.test.ts @@ -46,3 +46,55 @@ test('Host residency registry explains liveness and drains on exact release', as assert.equal(registry.activeCount, 0); assert.deepEqual(registry.snapshot(), []); }); + +test('idle-kind residencies block liveness but never the drain', async () => { + const registry = new HostResidencyRegistry(); + const marker = registry.acquire('process-retention', 'idle'); + const work = registry.acquire('hosted-execution'); + + assert.equal(registry.activeCount, 2); + assert.equal(registry.drainCount, 1); + assert.deepEqual(registry.snapshot(), [ + { label: 'hosted-execution', count: 1 }, + { label: 'process-retention', count: 1 }, + ]); + + let drained = false; + const drain = registry.waitForEmpty().then(() => { + drained = true; + }); + await Promise.resolve(); + assert.equal(drained, false); + work.release(); + await drain; + assert.equal(drained, true); + assert.equal(registry.activeCount, 1); + assert.equal(registry.drainCount, 0); + + const resource = registry.acquire('runtime-resource'); + let exceptResolved = false; + const except = registry.waitForEmptyExcept('runtime-resource').then(() => { + exceptResolved = true; + }); + await Promise.resolve(); + assert.equal(exceptResolved, true); + resource.release(); + await except; + marker.release(); + assert.equal(registry.activeCount, 0); + assert.deepEqual(registry.snapshot(), []); +}); + +test('idle-kind residency release resolves only drain waiters when nothing drains', async () => { + const registry = new HostResidencyRegistry(); + const marker = registry.acquire('process-retention', 'idle'); + let drained = false; + const drain = registry.waitForEmpty().then(() => { + drained = true; + }); + await drain; + assert.equal(drained, true); + assert.equal(registry.activeCount, 1); + marker.release(); + assert.equal(registry.activeCount, 0); +}); diff --git a/packages/runtime-host/src/server/host-kernel.ts b/packages/runtime-host/src/server/host-kernel.ts index 9655398dda..81de1c498e 100644 --- a/packages/runtime-host/src/server/host-kernel.ts +++ b/packages/runtime-host/src/server/host-kernel.ts @@ -407,6 +407,9 @@ export class RuntimeHostKernel { void this.#serveConnection(connection).finally(() => { this.#handshakingTransports.delete(transport); this.#transportAuthorities.delete(transport); + // A handshake that never completes keeps the Host visible to the idle + // timer while it is in flight; once it settles, idle must re-evaluate. + this.#scheduleIdleIfNeeded(); }); } @@ -516,7 +519,7 @@ export class RuntimeHostKernel { hello.generation !== undefined && hello.generation !== this.#options.generation; if (generationMismatch && hello.takeover?.expectedHostEpoch === this.hostEpoch) { - if (authority.principalKind === 'local_owner' && this.#isTrueIdle()) { + if (authority.principalKind === 'local_owner' && this.#isTrueIdle(transport)) { this.#requestDrain(); return { kind: 'draining', @@ -543,7 +546,7 @@ export class RuntimeHostKernel { ...(this.#options.generation === undefined ? {} : { generation: this.#options.generation }), state: admittedState, replacement: - this.#lifecycle.kind === 'ephemeral' && this.#isTrueIdle() + this.#lifecycle.kind === 'ephemeral' && this.#isSettledForReplacementAdvice() ? 'wait_for_idle_exit' : 'blocked_by_residency', ...(generationMismatch && authority.principalKind === 'local_owner' @@ -670,7 +673,9 @@ export class RuntimeHostKernel { #retainUntilProcessExit(): void { if (this.#retainedUntilProcessExit) return; this.#retainedUntilProcessExit = true; - this.#residencies.acquire('process-retention'); + // Not work in flight: the marker only blocks idle exit, so it must not + // stall the drain it accompanies. + this.#residencies.acquire('process-retention', 'idle'); this.#cancelIdle(); } @@ -866,7 +871,7 @@ export class RuntimeHostKernel { // requires explicit interruption authority before retirement. if (this.#acceptedTransports.size > 1) return true; if (this.#activeCommandOperations > 1) return true; - return this.#residencies.snapshot().some(({ label }) => label !== 'process-retention'); + return this.#residencies.drainCount > 0; } #beginCompositionDrain(): void { @@ -893,8 +898,9 @@ export class RuntimeHostKernel { if (this.#shutdownRequested) return; // One timer authority per lifecycle phase: until the first connection is // accepted, only #initialConnectionDeadline governs (it defers under an - // in-flight handshake, which #isTrueIdle() cannot see); afterwards the - // idle timer owns the idleGraceMs exit. + // in-flight handshake up to a bounded number of times); afterwards the + // idle timer owns the idleGraceMs exit, with in-flight handshakes visible + // to #isTrueIdle(). if (!this.#hasAcceptedConnection) return; if (!this.#isTrueIdle() || this.#idleTimer) return; this.#idleTimer = setTimeout(() => { @@ -904,7 +910,28 @@ export class RuntimeHostKernel { }, this.#lifecycle.idleGraceMs); } - #isTrueIdle(): boolean { + #isTrueIdle(exceptHandshaking?: RuntimeHostMessageTransport): boolean { + // A transport mid-handshake keeps the Host busy, except the one whose + // admission is being decided right now: counting it would make every + // true-idle takeover observe itself as activity. + const handshaking = + exceptHandshaking !== undefined && this.#handshakingTransports.has(exceptHandshaking) + ? this.#handshakingTransports.size - 1 + : this.#handshakingTransports.size; + return ( + this.#state === 'ready' && + this.#acceptedTransports.size === 0 && + handshaking === 0 && + this.#activeOperations === 0 && + this.#residencies.activeCount === 0 + ); + } + + // The replacement advice in a rejection is what a stale Client acts on. + // In-flight handshakes resolve within milliseconds and must not flip that + // advice, so unlike the idle timer and the takeover decision it ignores + // the handshaking set entirely. + #isSettledForReplacementAdvice(): boolean { return ( this.#state === 'ready' && this.#acceptedTransports.size === 0 && diff --git a/packages/runtime-host/src/server/host-residency-registry.ts b/packages/runtime-host/src/server/host-residency-registry.ts index cbb94fc4fe..069635b7c4 100644 --- a/packages/runtime-host/src/server/host-residency-registry.ts +++ b/packages/runtime-host/src/server/host-residency-registry.ts @@ -21,33 +21,61 @@ import type { OperationResidency } from './operation-dispatcher.js'; const RESIDENCY_LABEL_PATTERN = /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)*$/; +/** + * Residencies answer two different questions and the kinds must not be + * conflated: + * + * - `drain` residencies mark real work in flight. They block idle exit and + * must settle before a graceful close completes. + * - `idle` residencies only block idle exit. A marker such as + * process-retention is not work: it must not stall a drain or count as + * activity that blocks a maintenance probe. + */ +export type HostResidencyKind = 'drain' | 'idle'; + export interface HostResidencySnapshot { readonly label: string; readonly count: number; } +interface HostResidencyCounts { + total: number; + drain: number; +} + export class HostResidencyRegistry { - readonly #counts = new Map(); + readonly #counts = new Map(); readonly #drainWaiters = new Set<{ readonly excludedLabel: string | undefined; readonly resolve: () => void; }>(); #activeCount = 0; + #drainCount = 0; + /** Residencies of either kind keep the process alive against idle exit. */ get activeCount(): number { return this.#activeCount; } - acquire(label: string): OperationResidency { + /** Only drain-kind residencies block maintenance probes and graceful close. */ + get drainCount(): number { + return this.#drainCount; + } + + acquire(label: string, kind: HostResidencyKind = 'drain'): OperationResidency { requireResidencyLabel(label); this.#activeCount += 1; - this.#counts.set(label, (this.#counts.get(label) ?? 0) + 1); + if (kind === 'drain') this.#drainCount += 1; + const counts = this.#counts.get(label) ?? { total: 0, drain: 0 }; + counts.total += 1; + if (kind === 'drain') counts.drain += 1; + this.#counts.set(label, counts); let active = true; return { release: () => { if (!active) return; active = false; - this.#release(label); + this.#release(label, kind); }, }; } @@ -55,41 +83,45 @@ export class HostResidencyRegistry { snapshot(): readonly HostResidencySnapshot[] { return [...this.#counts] .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([label, count]) => Object.freeze({ label, count })); + .map(([label, counts]) => Object.freeze({ label, count: counts.total })); } waitForEmpty(): Promise { - return this.#waitForEmptyExcept(undefined); + return this.#waitForDrainEmptyExcept(undefined); } waitForEmptyExcept(excludedLabel: string): Promise { requireResidencyLabel(excludedLabel); - return this.#waitForEmptyExcept(excludedLabel); + return this.#waitForDrainEmptyExcept(excludedLabel); } - #release(label: string): void { - const count = this.#counts.get(label); - if (count === undefined || count === 0 || this.#activeCount === 0) { + #release(label: string, kind: HostResidencyKind): void { + const counts = this.#counts.get(label); + if (counts === undefined || counts.total === 0 || this.#activeCount === 0) { throw new Error('Runtime Host residency underflow'); } - if (count === 1) this.#counts.delete(label); - else this.#counts.set(label, count - 1); + counts.total -= 1; + if (kind === 'drain') { + counts.drain -= 1; + this.#drainCount -= 1; + } this.#activeCount -= 1; + if (counts.total === 0) this.#counts.delete(label); for (const waiter of this.#drainWaiters) { - if (!this.#isEmptyExcept(waiter.excludedLabel)) continue; + if (!this.#isDrainEmptyExcept(waiter.excludedLabel)) continue; this.#drainWaiters.delete(waiter); waiter.resolve(); } } - #waitForEmptyExcept(excludedLabel: string | undefined): Promise { - if (this.#isEmptyExcept(excludedLabel)) return Promise.resolve(); + #waitForDrainEmptyExcept(excludedLabel: string | undefined): Promise { + if (this.#isDrainEmptyExcept(excludedLabel)) return Promise.resolve(); return new Promise((resolve) => this.#drainWaiters.add({ excludedLabel, resolve })); } - #isEmptyExcept(excludedLabel: string | undefined): boolean { - for (const [label, count] of this.#counts) { - if (count > 0 && label !== excludedLabel) return false; + #isDrainEmptyExcept(excludedLabel: string | undefined): boolean { + for (const [label, counts] of this.#counts) { + if (counts.drain > 0 && label !== excludedLabel) return false; } return true; }