Pool: reconciler, controller, membership, fleet simulator and placement planner

README.md

@@ -122,6 +122,8 @@ lloom route q38fn                                                        # inspe
 lloom route q38fn --suspend-member qwen3.8-flash-next --apply --yes      # keep stable traffic on cloud
 lloom runtime-stop qwen38-flash-next-cluster                             # stop local TP-2 after suspension
 lloom route q38fn --resume-member qwen3.8-flash-next --apply --yes       # restore preferred-local recovery
+lloom pool plan --cordon spark-node-02                                   # dry-run placement across the cluster
+lloom pool members                                                       # nodes that joined with `lloom pool join`
 lloom integrate omp --apply --yes
 lloom add-model mlx-community/Qwen3.6-27B-OptiQ-4bit --keep-warm --default --go           # Mac: install, download, start
 lloom remove-model mlx-community/Qwen3.6-27B-OptiQ-4bit                                  # preview safe cleanup

bin/lloom.mjs

@@ -36,6 +36,21 @@ import {
   selectedRecipeIdFromCommunityPlan
 } from '../src/community-client.mjs';
 import { loadConfig } from '../src/config.mjs';
+import {
+  POOL_USAGE,
+  createPoolPlanReport,
+  formatPoolMembers,
+  formatPoolPlanReport,
+  poolPlanOptions,
+  poolTokenOptions
+} from '../src/pool-cli.mjs';
+import { joinPool } from '../src/pool-node.mjs';
+import {
+  POOL_SIMULATE_USAGE,
+  formatPoolSimulation,
+  poolSimulateOptions,
+  runPoolSimulateCommand
+} from '../src/pool-sim-cli.mjs';
 import { mutateConfigSource } from '../src/config-mutation.mjs';
 import { runtimeControlTimeoutMs } from '../src/control-timeout.mjs';
 import { createDoctorReport } from '../src/doctor.mjs';
@@ -136,6 +151,7 @@ const COMMAND_REGISTRY = [
   { name: 'runtimes', aliases: ['runtime-status'], tier: 'advanced', needsInstalledConfig: true },
   { name: 'service', aliases: [], tier: 'advanced', needsInstalledConfig: true },
   { name: 'cluster', aliases: ['cluster-status'], tier: 'advanced', needsInstalledConfig: true },
+  { name: 'pool', aliases: [], tier: 'advanced', needsInstalledConfig: true },
   { name: 'deployment', aliases: ['rollout'], tier: 'advanced', needsInstalledConfig: false },
   { name: 'node-agent', aliases: [], tier: 'advanced', needsInstalledConfig: false },
   { name: 'runtime-plan', aliases: [], tier: 'advanced', needsInstalledConfig: true },
@@ -228,6 +244,12 @@ Serving and discovery:
 Backends and runtimes:
   lloom down
   lloom cluster [status|doctor|discover] [--apply] [--id NAME] [--api-key-env NAME] [--json]
+  lloom pool plan [--cordon NODE[,NODE]] [--pin ALIAS=MODEL] [--profile NAME] [--consider-installs] [--headroom-gb N] [--churn-weight N] [--external fallback|allow|never] [--offline] [--json]
+  lloom pool token create [--node-id ID] [--ttl DURATION] [--uses N] [--label KEY=VALUE] [--json]
+  lloom pool members [--json]
+  lloom pool cordon|uncordon|drain|leave <node> [--reason TEXT]
+  lloom pool join <head-url> --token-stdin [--node-id ID] [--endpoint URL] [--apply --yes]
+  lloom pool simulate <scenario.json> [--mode off|advise|auto|compare] [--hours N] [--seed N] [--json]
   lloom deployment plan --plan FILE [--json]
   lloom deployment apply --plan FILE --nodes FILE [--operation-id ID] --apply --yes [--journal FILE] [--json]
   lloom deployment status --operation-id ID [--journal FILE] [--json]
@@ -370,6 +392,7 @@ function commandLine(command, parts = []) {
 
 const INSTALLED_CONFIG_COMMANDS = new Set([
   'add-model',
+  'pool',
   'bootstrap',
   'service',
   'cluster',
@@ -404,6 +427,7 @@ const INSTALLED_CONFIG_COMMANDS = new Set([
 
 const OPERATIONAL_CONFIG_COMMANDS = new Set([
   'add-model',
+  'pool',
   'service',
   'cluster',
   'cluster-status',
@@ -2495,6 +2519,101 @@ async function main() {
       if (!result) throw new Error(`route switch failed through ${gatewayUrlFor(config)}`);
       console.log(JSON.stringify({ ...result, applied: true }, null, 2));
     },
+    pool: async ({ args, config }) => {
+      const action = positional(args)[1] ?? 'plan';
+      const json = hasFlag(args, '--json');
+      if (action === 'token') {
+        if (positional(args)[2] !== 'create') throw new Error(POOL_USAGE);
+        const result = await gatewayRequest(config, '/gateway/pool/tokens', {
+          args,
+          method: 'POST',
+          body: poolTokenOptions({
+            ttl: argValue(args, '--ttl'),
+            uses: argValue(args, '--uses'),
+            labels: argValues(args, '--label'),
+            nodeId: argValue(args, '--node-id') ?? undefined
+          }),
+          timeoutMs: 10000
+        });
+        console.log(
+          json
+            ? JSON.stringify(result, null, 2)
+            : [
+                `Enrollment token ${result.id} (shown once; expires ${result.expiresAt}):`,
+                result.token,
+                '',
+                'On the joining node:',
+                `  lloom pool join <this-head-url> --token-stdin --endpoint <node-gateway-url> --apply --yes`
+              ].join('\n')
+        );
+        return;
+      }
+      if (action === 'members') {
+        const result = await gatewayRequest(config, '/gateway/pool/members', { args, timeoutMs: 10000 });
+        console.log(json ? JSON.stringify(result, null, 2) : formatPoolMembers(result));
+        return;
+      }
+      if (['cordon', 'uncordon', 'drain', 'leave'].includes(action)) {
+        const nodeId = positional(args)[2];
+        if (!nodeId) throw new Error(POOL_USAGE);
+        const reason = argValue(args, '--reason');
+        const result = await gatewayRequest(config, `/gateway/pool/nodes/${encodeURIComponent(nodeId)}/${action}`, {
+          args,
+          method: 'POST',
+          body: reason == null ? {} : { reason },
+          timeoutMs: 10000
+        });
+        console.log(JSON.stringify(result, null, 2));
+        return;
+      }
+      if (action === 'join') {
+        const headUrl = positional(args)[2];
+        if (!headUrl || !hasFlag(args, '--token-stdin')) throw new Error(POOL_USAGE);
+        const apply = hasFlag(args, '--apply');
+        const result = await joinPool(config, {
+          headUrl,
+          // The token is read only when it will be used.
+          token: apply && hasFlag(args, '--yes') ? await readBoundedStdinCredential(process.stdin) : null,
+          nodeId: argValue(args, '--node-id') ?? undefined,
+          endpoint: argValue(args, '--endpoint'),
+          apply,
+          yes: hasFlag(args, '--yes')
+        });
+        console.log(JSON.stringify(result, null, 2));
+        return;
+      }
+      if (action === 'simulate') {
+        const scenario = positional(args)[2];
+        if (!scenario) throw new Error(POOL_SIMULATE_USAGE);
+        const reports = await runPoolSimulateCommand(
+          scenario,
+          poolSimulateOptions({
+            mode: argValue(args, '--mode') ?? undefined,
+            hours: argValue(args, '--hours'),
+            seed: argValue(args, '--seed')
+          })
+        );
+        console.log(json ? JSON.stringify(reports, null, 2) : formatPoolSimulation(reports));
+        if (Object.values(reports).some((report) => !report.ok)) process.exitCode = 1;
+        return;
+      }
+      if (action !== 'plan') throw new Error(POOL_USAGE);
+      const options = poolPlanOptions({
+        cordon: argValues(args, '--cordon'),
+        pins: argValues(args, '--pin'),
+        profile: argValue(args, '--profile'),
+        considerInstalls: hasFlag(args, '--consider-installs'),
+        offline: hasFlag(args, '--offline'),
+        headroomGb: argValue(args, '--headroom-gb'),
+        churnWeight: argValue(args, '--churn-weight'),
+        external: argValue(args, '--external')
+      });
+      const report = await createPoolPlanReport(config, options, {
+        gatewayRequest: (target, pathname) => gatewayRequest(target, pathname, { args, timeoutMs: 30000 }),
+        gatewayUrl: gatewayUrlFor(config)
+      });
+      console.log(hasFlag(args, '--json') ? JSON.stringify(report, null, 2) : formatPoolPlanReport(report));
+    },
     fleet: async ({ args, config }) => {
       const action = positional(args)[1] ?? 'list';
       const name = positional(args)[2];

docs/architecture.md

@@ -220,7 +220,7 @@ The gateway expands `voice: "sample-speaker"` into the profile model, on-disk `r
 
 `/gateway/doctor` wraps setup status into a product-readiness report. It separates hard `blockers` from non-fatal `warnings`, groups install state into registry, recipe, backend, models, client, runtime, and benchmark phases, and returns ordered next actions that a UI can show directly.
 
-`/gateway/metrics` is the operational feed for dashboards, local tuning, and recipe evidence gathering. Aggregate totals plus 400 days of daily rollups are persisted by default in `~/.lloom/metrics-history.json` with atomic replacement and restored when the gateway starts, while active connections, rolling windows, and the bounded recent-request list remain process-local. `period=today|7d|30d|all` selects the durable window, and the topology HUD exposes the same choices. The persistence file contains aggregate counts and timing only—never prompts, responses, raw headers, callers, or client IPs. Before the first persistence-aware deployment, `node scripts/capture-live-metrics.mjs` can capture the old gateway's in-memory snapshot so its counters become the initial durable baseline.
+`/gateway/metrics` is the operational feed for dashboards, local tuning, and recipe evidence gathering. Aggregate totals plus 400 days of daily rollups are persisted by default in `~/.lloom/metrics-history.json` with atomic replacement and restored when the gateway starts, while active connections, rolling windows, and the bounded recent-request list remain process-local. Buckets are kept per resolved model, per HTTP route, and per requested id (`requested`, usually an alias), which is the demand signal for [pool planning](pool-planning.md). `period=today|7d|30d|all` selects the durable window, and the topology HUD exposes the same choices. The persistence file contains aggregate counts and timing only—never prompts, responses, raw headers, callers, or client IPs. Before the first persistence-aware deployment, `node scripts/capture-live-metrics.mjs` can capture the old gateway's in-memory snapshot so its counters become the initial durable baseline.
 
 Model-facing routes record requested model ID, resolved model ID, upstream model ID, kind, backend, runtime, status, stream flag, duration, first-content latency, last-content latency, response bytes, and normalized usage when the upstream reports it. Each completed streaming decode measurement follows the Spark Arena / `llama-benchy` timing convention: `(generated tokens - 1) / (last content token time - first content token time)`. The dashboard header shows `totals.activeDecodeTokensPerSecond`: completed decode tokens divided by the union of their first-to-last-content generation intervals within the selected period. Overlapping streams contribute tokens together but count elapsed generation time once; idle gaps and prefill are excluded. It updates as requests complete and remains visible during idle or buffered work. Daily windows select requests by completion date, as with token totals. Merged generation intervals are persisted but omitted from the HTTP metrics response. Legacy history without interval timing contributes its summed generation duration as an estimate; `activeDecodeRateEstimated` flags this or estimated token counts, and the HUD prefixes the rate with `~`. Model topology cards show instantaneous output rates; the model inspector shows `decodeTokensPerSecond`, the rolling mean of the latest ten measured streaming request rates in the selected period. LLooM uses reported output tokens when available and otherwise estimates them from observed output characters. Embedding responses have no generative token count, so their output contribution is estimated from observed response bytes for cumulative volume only. `estimatedDecodeSamples` makes estimated decode samples explicit. Non-streaming generative responses are excluded from decode speed because LLooM cannot observe their token-by-token decode interval; their tokens and end-to-end duration remain available in the other aggregate fields. Connection telemetry derives a short caller label from explicit client-name headers, recognized User-Agent families, or the SDK language header, but never stores raw headers or client IPs; unknown callers retain `conn_N`. Client disconnects before completion are recorded as status `499` so local timeouts do not look like backend crashes. The endpoint aggregates durable totals by model and route while retaining live-only rolling and recent windows.
 

docs/clusters.md

@@ -258,6 +258,8 @@ before contacting any node.
 
 The live dashboard renders node cards between the main LLooM chassis and model cards. Each node card reports architecture, accelerator type, CPU, RAM, GPU compute, GPU memory, temperature, and power when the host can report them. Unsupported fields render as unavailable rather than inventing values; Apple Silicon still reports its profile and shared-memory pressure even when per-GPU utilization is unavailable. Select a node for its complete profile. Replicated models connect to every target node and distributed models connect to every member node.
 
+`GET /gateway/node?inventory=1` and `GET /gateway/cluster?inventory=1` add each node's pool inventory (compute architectures, model-root disk, cached recipe weights, installed images and fabric links). `lloom pool plan` uses it to propose placements across the whole cluster; see [`pool-planning.md`](pool-planning.md). Nodes can also join a head with an enrollment token and keep a lease (`lloom pool join`); membership adds liveness and cordons to planning but does not replace `cluster add-node` for routing.
+
 Before sending model traffic, use `lloom cluster doctor --runtime ID` to verify the required nodes and serving endpoint. Whole-topology doctor reports optional offline nodes and missing memory telemetry as warnings; they do not make unrelated runtimes unavailable. Then start a small replicated lane, verify requests alternate under equal load, and only then canary the distributed engine.
 
 ## A note on example identifiers