refactor(runtime-host): split residency kinds and give idle visibility to handshakes

packages/runtime-host/src/__tests__/host-kernel.test.ts

@@ -1268,15 +1268,15 @@ describe('non-serving Runtime Host kernel', () => {
     });
   });
 
-  test('process-exit retention closes admission before requiring termination without releasing ownership', async () => {
+  test('process-exit retention neither stalls the graceful close nor retains ownership', async () => {
     await withHostPaths(async (paths) => {
       const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' });
       const owner = await tryAcquireInteractiveRootOwner(capability);
       assert.ok(owner);
       const host = await RuntimeHostKernel.start({
         owner,
         idleGraceMs: 10_000,
-        shutdownGraceMs: 50,
+        shutdownGraceMs: 10_000,
         composition: defineInteractiveRuntimeHostComposition(async (context) => {
           context.retainUntilProcessExit();
           context.retainUntilProcessExit();
@@ -1286,11 +1286,12 @@ describe('non-serving Runtime Host kernel', () => {
       });
 
       try {
-        await assert.rejects(
+        // The anti-idle marker is not work: the drain it accompanies closes
+        // gracefully, long before the shutdown deadline.
+        await withTimeout(
           host.closed,
-          (error: unknown) =>
-            error instanceof RuntimeHostProcessTerminationRequiredError &&
-            error.code === 'process_termination_required',
+          2_000,
+          'retained Host waited out its shutdown deadline instead of closing gracefully',
         );
         await assert.rejects(
           () => openSocket(host.endpoint),
@@ -1300,7 +1301,9 @@ describe('non-serving Runtime Host kernel', () => {
             ((error as NodeJS.ErrnoException).code === 'ENOENT' ||
               (error as NodeJS.ErrnoException).code === 'ECONNREFUSED'),
         );
-        assert.equal(await tryAcquireInteractiveRootOwner(capability), undefined);
+        const successor = await tryAcquireInteractiveRootOwner(capability);
+        assert.ok(successor, 'graceful close must release the State Root writer lease');
+        await successor?.close();
       } finally {
         await owner.close();
       }
@@ -1405,6 +1408,75 @@ describe('non-serving Runtime Host kernel', () => {
     });
   });
 
+  test('an in-flight handshake keeps an ephemeral Host alive past the idle deadline', async () => {
+    await withHostPaths(async (paths) => {
+      const candidate = await startTestRuntimeHostCandidate(paths, {
+        rootPath: paths.root,
+        idleGraceMs: 250,
+        initialConnectionTimeoutMs: 5_000,
+        handshakeTimeoutMs: 5_000,
+      });
+      assert.equal(candidate.kind, 'winner');
+      if (candidate.kind !== 'winner') return;
+      const host = candidate.host;
+
+      // The first accepted connection leaves and the idle timer arms; a
+      // handshake that begins now is the phase the idle timer used to be
+      // blind to.
+      const first = await retryConnect(paths, CURRENT_PROTOCOL);
+      assert.equal(first.kind, 'connected');
+      if (first.kind !== 'connected') return;
+      await first.connection.close();
+
+      const silent = await openSocket(host.endpoint);
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      try {
+        // Past the idle deadline with the handshake in flight: the Host must
+        // not drain under a connecting Client.
+        await new Promise((resolve) => setTimeout(resolve, 500));
+        assert.equal(host.state, 'ready');
+      } finally {
+        silent.destroy();
+      }
+      // Once the handshake settles, the idle timer re-arms and the Host exits.
+      await withTimeout(
+        host.closed,
+        5_000,
+        'ephemeral Host never idle-exited after the handshake settled',
+      );
+    });
+  });
+
+  test('a poisoned Host closes gracefully without waiting out the shutdown deadline', async () => {
+    await withHostPaths(async (paths) => {
+      const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' });
+      const owner = await tryAcquireInteractiveRootOwner(capability);
+      assert.ok(owner);
+      const host = await RuntimeHostKernel.start({
+        owner,
+        lifecycleMode: 'service',
+        shutdownGraceMs: 10_000,
+        composition: defineInteractiveRuntimeHostComposition(async (context) => {
+          // Mirror the poison/fatal path: the anti-idle marker must not stall
+          // the drain it accompanies.
+          context.retainUntilProcessExit();
+          context.requestDrain();
+          return {
+            handlers: createUnavailableDomainOperationHandlers(),
+            beginDrain() {},
+            async recover() {},
+            async close() {},
+          };
+        }),
+      });
+      await withTimeout(
+        host.closed,
+        2_000,
+        'poisoned Host waited out its shutdown deadline instead of closing gracefully',
+      );
+    });
+  });
+
   test('drain requested before factory completion begins drain before recovery exactly once', async () => {
     await withHostPaths(async (paths) => {
       const capability = await resolveStorageRoot({ path: paths.root, kind: 'interactive' });
@@ -1591,18 +1663,26 @@ describe('non-serving Runtime Host kernel', () => {
       staleWhileResident.abort();
       await staleWhileResident.closed;
 
-      const blocked = await connectOrSpawnRuntimeHost({
-        ...paths,
-        rootPath: paths.root,
-        protocol: LEGACY_PROTOCOL,
-        compositionId: KERNEL_COMPOSITION.descriptor.id,
-        candidateEntrypoint: KERNEL_CANDIDATE_ENTRYPOINT,
-        electionDeadlineMs: 2_000,
-      });
-      assert.equal(blocked.kind, 'incompatible');
-      if (blocked.kind === 'incompatible') {
-        assert.equal(blocked.handshake.replacement, 'blocked_by_residency');
+      const blockedWhileResident = new FramedTransport(await openSocket(candidate.host.endpoint));
+      await writeRawLocalIpc(
+        blockedWhileResident,
+        encodeLegacyProtocolFrame({
+          kind: 'hello',
+          clientInstanceId: 'blocked-legacy-resident',
+          protocolMin: LEGACY_PROTOCOL.min,
+          protocolMax: LEGACY_PROTOCOL.max,
+        }),
+      );
+      const blockedResponse = decodeHostFrame(await blockedWhileResident.read(1_000));
+      assert.ok('kind' in blockedResponse && blockedResponse.kind === 'incompatible');
+      if ('kind' in blockedResponse && blockedResponse.kind === 'incompatible') {
+        assert.equal(blockedResponse.replacement, 'blocked_by_residency');
       }
+      blockedWhileResident.abort();
+      await blockedWhileResident.closed;
+      // The rejected handshake's teardown is asynchronous Host-side; let it
+      // settle so only the next probe's own handshake remains in flight.
+      await sleep(50);
       await resident.connection.close();
 
       const staleAtIdle = new FramedTransport(await openSocket(candidate.host.endpoint));

packages/runtime-host/src/__tests__/host-residency-registry.test.ts

@@ -46,3 +46,55 @@ test('Host residency registry explains liveness and drains on exact release', as
   assert.equal(registry.activeCount, 0);
   assert.deepEqual(registry.snapshot(), []);
 });
+
+test('idle-kind residencies block liveness but never the drain', async () => {
+  const registry = new HostResidencyRegistry();
+  const marker = registry.acquire('process-retention', 'idle');
+  const work = registry.acquire('hosted-execution');
+
+  assert.equal(registry.activeCount, 2);
+  assert.equal(registry.drainCount, 1);
+  assert.deepEqual(registry.snapshot(), [
+    { label: 'hosted-execution', count: 1 },
+    { label: 'process-retention', count: 1 },
+  ]);
+
+  let drained = false;
+  const drain = registry.waitForEmpty().then(() => {
+    drained = true;
+  });
+  await Promise.resolve();
+  assert.equal(drained, false);
+  work.release();
+  await drain;
+  assert.equal(drained, true);
+  assert.equal(registry.activeCount, 1);
+  assert.equal(registry.drainCount, 0);
+
+  const resource = registry.acquire('runtime-resource');
+  let exceptResolved = false;
+  const except = registry.waitForEmptyExcept('runtime-resource').then(() => {
+    exceptResolved = true;
+  });
+  await Promise.resolve();
+  assert.equal(exceptResolved, true);
+  resource.release();
+  await except;
+  marker.release();
+  assert.equal(registry.activeCount, 0);
+  assert.deepEqual(registry.snapshot(), []);
+});
+
+test('idle-kind residency release resolves only drain waiters when nothing drains', async () => {
+  const registry = new HostResidencyRegistry();
+  const marker = registry.acquire('process-retention', 'idle');
+  let drained = false;
+  const drain = registry.waitForEmpty().then(() => {
+    drained = true;
+  });
+  await drain;
+  assert.equal(drained, true);
+  assert.equal(registry.activeCount, 1);
+  marker.release();
+  assert.equal(registry.activeCount, 0);
+});

packages/runtime-host/src/server/host-kernel.ts

@@ -407,6 +407,9 @@ export class RuntimeHostKernel {
     void this.#serveConnection(connection).finally(() => {
       this.#handshakingTransports.delete(transport);
       this.#transportAuthorities.delete(transport);
+      // A handshake that never completes keeps the Host visible to the idle
+      // timer while it is in flight; once it settles, idle must re-evaluate.
+      this.#scheduleIdleIfNeeded();
     });
   }
 
@@ -516,7 +519,7 @@ export class RuntimeHostKernel {
       hello.generation !== undefined &&
       hello.generation !== this.#options.generation;
     if (generationMismatch && hello.takeover?.expectedHostEpoch === this.hostEpoch) {
-      if (authority.principalKind === 'local_owner' && this.#isTrueIdle()) {
+      if (authority.principalKind === 'local_owner' && this.#isTrueIdle(transport)) {
         this.#requestDrain();
         return {
           kind: 'draining',
@@ -543,7 +546,7 @@ export class RuntimeHostKernel {
         ...(this.#options.generation === undefined ? {} : { generation: this.#options.generation }),
         state: admittedState,
         replacement:
-          this.#lifecycle.kind === 'ephemeral' && this.#isTrueIdle()
+          this.#lifecycle.kind === 'ephemeral' && this.#isSettledForReplacementAdvice()
             ? 'wait_for_idle_exit'
             : 'blocked_by_residency',
         ...(generationMismatch && authority.principalKind === 'local_owner'
@@ -670,7 +673,9 @@ export class RuntimeHostKernel {
   #retainUntilProcessExit(): void {
     if (this.#retainedUntilProcessExit) return;
     this.#retainedUntilProcessExit = true;
-    this.#residencies.acquire('process-retention');
+    // Not work in flight: the marker only blocks idle exit, so it must not
+    // stall the drain it accompanies.
+    this.#residencies.acquire('process-retention', 'idle');
     this.#cancelIdle();
   }
 
@@ -866,7 +871,7 @@ export class RuntimeHostKernel {
     // requires explicit interruption authority before retirement.
     if (this.#acceptedTransports.size > 1) return true;
     if (this.#activeCommandOperations > 1) return true;
-    return this.#residencies.snapshot().some(({ label }) => label !== 'process-retention');
+    return this.#residencies.drainCount > 0;
   }
 
   #beginCompositionDrain(): void {
@@ -893,8 +898,9 @@ export class RuntimeHostKernel {
     if (this.#shutdownRequested) return;
     // One timer authority per lifecycle phase: until the first connection is
     // accepted, only #initialConnectionDeadline governs (it defers under an
-    // in-flight handshake, which #isTrueIdle() cannot see); afterwards the
-    // idle timer owns the idleGraceMs exit.
+    // in-flight handshake up to a bounded number of times); afterwards the
+    // idle timer owns the idleGraceMs exit, with in-flight handshakes visible
+    // to #isTrueIdle().
     if (!this.#hasAcceptedConnection) return;
     if (!this.#isTrueIdle() || this.#idleTimer) return;
     this.#idleTimer = setTimeout(() => {
@@ -904,7 +910,28 @@ export class RuntimeHostKernel {
     }, this.#lifecycle.idleGraceMs);
   }
 
-  #isTrueIdle(): boolean {
+  #isTrueIdle(exceptHandshaking?: RuntimeHostMessageTransport): boolean {
+    // A transport mid-handshake keeps the Host busy, except the one whose
+    // admission is being decided right now: counting it would make every
+    // true-idle takeover observe itself as activity.
+    const handshaking =
+      exceptHandshaking !== undefined && this.#handshakingTransports.has(exceptHandshaking)
+        ? this.#handshakingTransports.size - 1
+        : this.#handshakingTransports.size;
+    return (
+      this.#state === 'ready' &&
+      this.#acceptedTransports.size === 0 &&
+      handshaking === 0 &&
+      this.#activeOperations === 0 &&
+      this.#residencies.activeCount === 0
+    );
+  }
+
+  // The replacement advice in a rejection is what a stale Client acts on.
+  // In-flight handshakes resolve within milliseconds and must not flip that
+  // advice, so unlike the idle timer and the takeover decision it ignores
+  // the handshaking set entirely.
+  #isSettledForReplacementAdvice(): boolean {
     return (
       this.#state === 'ready' &&
       this.#acceptedTransports.size === 0 &&