From ad2887c83937d5c9d25b79de403309f9be1e88b2 Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 18 Nov 2026 21:01:25 -0500 Subject: [PATCH 1/6] feat: machine-wide heavy-work ledger and open-issue fixes (WIP) x heavy: memory ledger, core cap, worker/coordinator rule for many agents. setCookie(line), MAIL_TIMEOUT_MS, cache tier limits, replicator skips views, five-role rollout drill, queue-wake test diagnostics. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 20 +- CHANGELOG.md | 2 +- CLAUDE.md | 6 +- PUBLISHING.md | 5 +- docs/architecture/guards.md | 1 + docs/ops/06-runbooks.md | 2 +- docs/ops/07-deploy-contract.md | 18 +- llms.txt | 1 + .../action/src/session-cookie.e2e.test.ts | 104 +++++ packages/auth/README.md | 5 + packages/cache/CLAUDE.md | 6 +- packages/cache/README.md | 5 +- packages/cache/src/errors.test.ts | 19 +- packages/cache/src/errors.ts | 37 +- packages/cache/src/index.ts | 2 +- packages/cache/src/lru.test.ts | 4 + packages/cache/src/lru.ts | 11 +- packages/cache/src/redis.test.ts | 16 +- packages/cache/src/redis.ts | 25 +- packages/cache/src/tiers.ts | 19 +- packages/cache/src/ttl-jitter.test.ts | 11 + packages/cli/src/cmd-drill.ts | 21 +- packages/cli/src/cmd-heavy-spec.ts | 28 ++ packages/cli/src/cmd-heavy.test.ts | 170 +++++++++ packages/cli/src/cmd-heavy.ts | 160 ++++++++ packages/cli/src/cmd-test.ts | 16 +- .../cli/src/codemods/docs-many-agents.test.ts | 27 ++ packages/cli/src/codemods/docs-many-agents.ts | 30 ++ packages/cli/src/codemods/index.ts | 2 + packages/cli/src/error-codes-tooling.ts | 12 +- packages/cli/src/exec.ts | 14 +- packages/cli/src/heavy-cores.ts | 51 +++ packages/cli/src/heavy-env.ts | 19 + packages/cli/src/heavy-errors.ts | 36 ++ packages/cli/src/heavy-flock.ts | 73 ++++ packages/cli/src/heavy-ledger.test.ts | 234 ++++++++++++ packages/cli/src/heavy-ledger.ts | 358 ++++++++++++++++++ packages/cli/src/heavy-machine.test.ts | 121 ++++++ packages/cli/src/heavy-memory.ts | 67 ++++ packages/cli/src/heavy-weights.ts | 47 +++ packages/cli/src/heavy.test.ts | 142 +++++++ packages/cli/src/heavy.ts | 290 ++++++++++++++ packages/cli/src/local-cli.test.ts | 11 + packages/cli/src/local-cli.ts | 11 +- packages/cli/src/mcp-errors.ts | 3 + packages/cli/src/messages.ts | 11 + packages/cli/src/parse.test.ts | 2 +- packages/cli/src/registry.ts | 2 + packages/cli/src/role-replicator.test.ts | 20 + packages/cli/src/role-replicator.ts | 8 +- packages/cli/src/runtime-cache-limits.test.ts | 113 ++++++ packages/cli/src/runtime-cache.ts | 27 +- packages/cli/src/templates/scaffold-docs.ts | 9 +- packages/cli/src/test-batches.ts | 12 +- packages/cli/src/test-shards.ts | 6 +- packages/cli/src/test-slots.test.ts | 138 +++---- packages/cli/src/test-slots.ts | 196 ++-------- packages/cli/src/verify-checks.ts | 7 +- packages/cli/src/verify-coverage-run.test.ts | 5 +- packages/cli/src/verify-coverage-run.ts | 9 +- packages/cli/src/verify-deadline.ts | 20 +- packages/cli/src/verify-heavy-steps.ts | 26 ++ packages/cli/src/verify-test-run.ts | 5 +- packages/cli/src/verify-tests.ts | 47 ++- packages/core/src/framework-env.ts | 2 +- packages/http/README.md | 3 +- packages/http/src/set-cookie-line.test.ts | 151 ++++++++ packages/http/src/set-cookie.ts | 109 +++++- packages/jobs/src/queue-wake.live.test.ts | 48 ++- packages/mail/README.md | 2 +- packages/mail/src/driver-env.test.ts | 73 ++++ packages/mail/src/driver-env.ts | 24 ++ packages/mail/src/smtp-client.ts | 2 +- packages/mail/src/smtp-socket.ts | 2 +- packages/testing/README.md | 2 +- packages/testing/src/drill-fleet.ts | 2 + packages/testing/src/drill-services.test.ts | 43 +++ packages/testing/src/drill-services.ts | 28 +- packages/testing/src/drills.test.ts | 38 +- packages/testing/src/drills.ts | 58 ++- packages/testing/src/index.ts | 11 +- scripts/coverage-gate.ts | 25 +- scripts/heavy-peak.test.ts | 27 ++ scripts/heavy-peak.ts | 117 ++++++ scripts/lib/ci/test-weights.json | 1 - scripts/lib/file-lock.test.ts | 99 ----- scripts/lib/file-lock.ts | 96 ----- scripts/lib/release-preflight.test.ts | 14 +- scripts/lib/release-preflight.ts | 9 +- scripts/lib/run.ts | 6 +- scripts/lib/spawn-timeout-pins.ts | 4 +- scripts/preflight.ts | 27 +- scripts/template-codemods-rows.ts | 7 + scripts/template-codemods.lock.json | 40 +- scripts/typecheck-locked.ts | 62 ++- wiki/Building-Your-Own-Base.md | 17 +- wiki/CI-Parallel-Gate.md | 5 +- wiki/CLI-Reference.md | 34 +- wiki/Caching-And-Invalidation.md | 2 +- wiki/Configuration.md | 26 +- wiki/Error-Codes-cache.md | 6 +- wiki/Error-Codes-cli.md | 3 + wiki/Error-Codes.md | 5 +- wiki/Home.md | 1 + wiki/Interface-Rules.md | 2 +- wiki/Known-Gaps.md | 2 +- wiki/Mail.md | 2 +- wiki/Many-Agents.md | 60 +++ wiki/Testing.md | 10 +- wiki/Tutorial-01-First-App.md | 4 +- wiki/_Sidebar.md | 1 + 111 files changed, 3556 insertions(+), 681 deletions(-) create mode 100644 packages/action/src/session-cookie.e2e.test.ts create mode 100644 packages/cli/src/cmd-heavy-spec.ts create mode 100644 packages/cli/src/cmd-heavy.test.ts create mode 100644 packages/cli/src/cmd-heavy.ts create mode 100644 packages/cli/src/codemods/docs-many-agents.test.ts create mode 100644 packages/cli/src/codemods/docs-many-agents.ts create mode 100644 packages/cli/src/heavy-cores.ts create mode 100644 packages/cli/src/heavy-env.ts create mode 100644 packages/cli/src/heavy-errors.ts create mode 100644 packages/cli/src/heavy-flock.ts create mode 100644 packages/cli/src/heavy-ledger.test.ts create mode 100644 packages/cli/src/heavy-ledger.ts create mode 100644 packages/cli/src/heavy-machine.test.ts create mode 100644 packages/cli/src/heavy-memory.ts create mode 100644 packages/cli/src/heavy-weights.ts create mode 100644 packages/cli/src/heavy.test.ts create mode 100644 packages/cli/src/heavy.ts create mode 100644 packages/cli/src/runtime-cache-limits.test.ts create mode 100644 packages/cli/src/verify-heavy-steps.ts create mode 100644 packages/http/src/set-cookie-line.test.ts create mode 100644 scripts/heavy-peak.test.ts create mode 100644 scripts/heavy-peak.ts delete mode 100644 scripts/lib/file-lock.test.ts delete mode 100644 scripts/lib/file-lock.ts create mode 100644 wiki/Many-Agents.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3f1a82c0..e04f19ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1381,25 +1381,27 @@ jobs: run: mkdir -p "$WORK" && bash docker/deploy-proof/run.sh # Does the reference app keep the framework's declared behaviour while a dependency fails? `x - # drill` boots `examples/dummy`'s web and worker roles as separate processes on a scratch database, + # drill` boots `examples/dummy`'s roles as separate processes on a scratch database, # each dialling Postgres through a fault proxy, plays one fault under load (anonymous page reads, # and `contactSales` POSTs that enqueue a job each) and is red on any stated property that did not # hold: a coded answer, a bounded request, recovery with no restart, every job terminal, none run - # twice, a signalled role gone within its drain. The verdict is evidence, never a chart to read. + # twice, a signalled role gone within its drain, and — `roll-before-migrate`, the rollout order — every + # role started ahead of migrate waiting, then serving. The verdict is evidence, never a chart to read. # # Not a gate step: it needs a real Postgres, and `x verify` needs nothing (docs/idea/21-the-range.md). - # THREE of the eight, chosen for the 5-minute budget — measured locally 2026-11-06: `pg-blackhole` + # FOUR of the nine, chosen for the 5-minute budget — measured locally 2026-11-06: `pg-blackhole` # 67 s (plan 29's unanswered-statement bound; `drill-pg-blackhole.live.test.ts` proves it red with # the bound off), `worker-sigkill` 49 s (a reclaimed run, 1,012 jobs, none stuck or doubled), - # `web-sigterm` 6 s. `pg-slow` and `pg-restart` cost a minute each more and run by hand; + # `web-sigterm` 6 s. `pg-slow`, `pg-restart` and `worker-sigterm` run by hand; # `redis-down` and `nats-down` fault services this app does not declare, so they are refused - # (`X_DRILL_SERVICE_MISSING`), never run green. Postgres only, so only Postgres is started. + # (`X_DRILL_SERVICE_MISSING`), never run green. `roll-before-migrate` plays all five roles, so the bus and the logical Postgres + # (`postgres-tls`, `wal_level=logical`) start beside Postgres; a role it left out is red here. fault-drills: needs: [tested, changes] if: ${{ !cancelled() && needs.tested.outputs.tested != 'true' && needs.changes.outputs.heavy != 'false' }} runs-on: ubuntu-latest # A runaway kill-switch, not a budget: four drills plus setup is about three and a half - # minutes (roll-before-migrate measured 10 s locally, 2026-11-15). + # minutes (roll-before-migrate measured 10 s locally on three roles, 2026-11-15; five add two boots). timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1407,7 +1409,7 @@ jobs: persist-credentials: false - name: test services, in RAM — started run: | - nohup bash -c 'docker compose -f docker/docker-compose.test.yml up -d --wait postgres; echo $? > "$RUNNER_TEMP/services.exit"' \ + nohup bash -c 'docker compose -f docker/docker-compose.test.yml up -d --wait postgres postgres-tls nats; echo $? > "$RUNNER_TEMP/services.exit"' \ > "$RUNNER_TEMP/services.log" 2>&1 & echo $! > "$RUNNER_TEMP/services.pid" - uses: ./.github/actions/setup @@ -1417,7 +1419,7 @@ jobs: cat "$RUNNER_TEMP/services.log" [ "$(cat "$RUNNER_TEMP/services.exit" 2>/dev/null)" = 0 ] \ || { docker compose -f docker/docker-compose.test.yml ps -a; exit 1; } - grep -E '^TEST_DATABASE_URL=' docker/test-services.env >> "$GITHUB_ENV" + grep -E '^TEST_(DATABASE|REPLICATION|NATS)_URL=' docker/test-services.env >> "$GITHUB_ENV" # roll-before-migrate: the serving roles started on a database whose framework stamp is an # older build's, then ROLE=migrate — each must wait, say so once, and serve, never crash-loop. - name: x drill pg-blackhole, worker-sigkill, web-sigterm, roll-before-migrate on examples/dummy @@ -1428,6 +1430,8 @@ jobs: bun run x -- drill "$drill" --cwd examples/dummy --json > "$doc" || status=1 jq -r '.summary, (.findings[]? | " \(.code)\n cause: \(.cause)\n fix: \(.fix)")' "$doc" \ || { echo "x drill $drill wrote no document"; status=1; } + missing="$(jq -r '.data.rolesNotPlayed[]? | "\(.role), needs \(.needs)"' "$doc" 2>/dev/null || true)" + [ -z "$missing" ] || { echo "x drill $drill left roles out: $missing"; status=1; } done exit "$status" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 393e7cdec..49a803dcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major ## [Unreleased] -Nothing yet. +- **Many agents on one machine:** `x verify`, `x test`, every lane and `bun run coverage` reserve their measured memory in one flock-backed ledger per machine, first come first served, at most 4 cores locally; `x heavy status` and `x heavy run` expose it, and a new app's `CLAUDE.md` says a worker runs its own files and the coordinator runs the gate once ([Working with many agents](wiki/Many-Agents.md)). ## Releases diff --git a/CLAUDE.md b/CLAUDE.md index 0ae32c2d3..7a946b31d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -183,8 +183,6 @@ a list, never a count or ordinal: `PRIMITIVE_FACTORIES` in the same file, held c | which rung of the scale ladder a claim belongs to | [`docs/idea/17-scale-ladder.md`](docs/idea/17-scale-ladder.md) | | every error code | [`wiki/Error-Codes.md`](wiki/Error-Codes.md) | | every CLI flag | [`wiki/CLI-Reference.md`](wiki/CLI-Reference.md) | -| where to start reading | `packages/action/`, the load-bearing abstraction | -| what idiomatic usage looks like | [`examples/dummy/`](examples/dummy/README.md) | | tables, with no ORM | `entity()` declares; `postgresDriver()` (`packages/entity/src/pg-driver.ts`) is hand-written SQL | | realtime capacity, as measured | [`scripts/bench/results/README.md`](scripts/bench/results/README.md) | | the deploy-proof milestone | [`docs/history/milestone-11.md`](docs/history/milestone-11.md) · [`docs/idea/14-roadmap.md`](docs/idea/14-roadmap.md) | @@ -211,7 +209,7 @@ Free runners only (public repo): `ubuntu-latest`, `windows-latest`, `macos-lates Act as an independent, proactive senior engineer: do the full job. Don't ask what the code, the docs or a sensible default answers: decide, then state the decision. Finish the -work: fix, test, document, run the gate. Don't hand back a half-done step. Find the root cause; fix +work: fix, test, document, run the gate. Find the root cause; fix what you find along the way. Ask only before an irreversible or outward-facing act the owner hasn't already asked for. @@ -220,6 +218,8 @@ hasn't already asked for. No git worktrees in this checkout: a team of subagents shares it, on disjoint file sets. (Another repo the owner may be editing on a non-`main` checkout: work in a git worktree there.) +**Many agents, one machine:** a worker runs its own files, the coordinator runs the gate once; heavy runs queue in one memory ledger (`x heavy status`, [guide](wiki/Many-Agents.md)). + **Only the top-level agent spawns subagents.** A subagent works and reports back; it never delegates ([why](docs/history/feature-workflow.md)). Too large a scope is said in the report; widening the split is the top-level agent's call. diff --git a/PUBLISHING.md b/PUBLISHING.md index 0f169ba38..cb2e2192f 100644 --- a/PUBLISHING.md +++ b/PUBLISHING.md @@ -492,8 +492,9 @@ up (`X_MAINT_LANE_EXPIRED`). ### When a release is due -The maintainer bot judges it weekly (`.dz/briefs/release-due.md`), reading what would ship; a merge -alone never cuts a release, and a maintainer can still cut one by hand. A fix for a security advisory +The maintainer bot judges it weekly (`.dz/briefs/release-due.md`), reading what would ship, and when +one is needed that same run opens the `chore: release X.Y.Z` PR; a merge alone never cuts a release, +and a maintainer can still cut one by hand. A fix for a security advisory a first-tier source confirms is the one off-cycle release (`.dz/briefs/security-release.md`). `bun run scripts/release-due.ts --json` gathers the facts for that judgement, fetching `origin/main`'s tip when the checkout lacks it. The window is every commit since the highest `v*` tag HEAD contains, diff --git a/docs/architecture/guards.md b/docs/architecture/guards.md index 15ef45bc1..2c1afc947 100644 --- a/docs/architecture/guards.md +++ b/docs/architecture/guards.md @@ -70,6 +70,7 @@ The codes are in | `bun run scripts/generator-counts.ts` | Enforce, as a gate step, that every file count a doc states about a generator is a number that generator still prints. | `X_DOC_FILE_COUNT_STALE` `X_DOC_FILE_COUNT_UNSCANNED` | | `bun run guards-doc` | Generate `docs/architecture/guards.md` from the guards' own headers — command (in the form that reports, never one that writes), what it refuses, the codes it emits — and refuse a committed page that has drifted from them. | `X_GUARDS_DOC_DRIFT` `X_GUARDS_DOC_UNSCANNED` | | `bun run scripts/health.ts` | Answer whether the project is healthy: one verdict per row of `.dz/briefs/health.md`, read from git, `gh`, npm and the repo's own scripts, and refuse with `cannot tell` and no data when any row's command cannot be read — answering over a row it never read is the silent green it refuses. | `X_CI_RUN_NOT_FOUND` `X_HEALTH_ROW_UNREADABLE` `X_MAINT_LANE_ABSENT` `X_MAINT_LANE_EXPIRED` `X_TRUST_PUBLISHER_MISSING` | +| `bun run scripts/heavy-peak.ts` | The recipe behind every figure in `packages/cli/src/heavy-weights.ts`: run one command and report the peak RSS of its whole process tree, sampled from `/proc`. | `X_CLI_BAD_FLAG` | | `bun run scripts/i18n-catalog.ts` | Enforce, as a gate step, that `packages/i18n/src/catalogs/en.json` still describes the framework packages' own strings. | `X_CATALOG_KEY_UNREACHABLE` `X_CATALOG_MISSING_KEYS` | | `bun run scripts/image-contract.ts` | Enforce, as a gate step, the two properties that make `docker/Dockerfile` produce an image that can start. | `X_CORPUS_UNSCANNED` `X_IMAGE_CONTENT_UNIGNORED` `X_IMAGE_GUARD_MISSING` `X_IMAGE_LIBC_MISMATCH` `X_IMAGE_SECRET_UNIGNORED` | | `bun run index-of-order` | Refuse an ordering assertion built on `indexOf`/`findIndex` with nothing asserting the needle is PRESENT — the shape of a test that cannot fail. | `X_INDEX_ORDER_PIN_STALE` `X_INDEX_ORDER_PIN_UNEXPLAINED` `X_INDEX_ORDER_UNGUARDED` `X_INDEX_ORDER_UNSCANNED` | diff --git a/docs/ops/06-runbooks.md b/docs/ops/06-runbooks.md index 9e0d1c929..15e5399ef 100644 --- a/docs/ops/06-runbooks.md +++ b/docs/ops/06-runbooks.md @@ -163,7 +163,7 @@ deadline (at least 1 s), as `X_DB_UNAVAILABLE` if none answered — and the plat | no migrate ran: an applier that drops Helm hooks, a compose `up` without the `run --rm migrate` step, a PaaS with no release phase | run it once against the same database: `x db migrate --json`, or the image with `ROLE=migrate`. Then wire the order: [deploy contract](./07-deploy-contract.md#order-migrate-before-roles) | | the cause names a newer major as the newest applied build | refused at once, no wait: an old-major role over a database a newer release migrated. Roll forward to that release (`x deploy --image --json`), never migrate the old one over it | | a migrate Job that is merely slow, past the wait | raise `ULTIMATE_SCHEMA_WAIT_MS` on the serving roles; `0` turns the wait off and refuses at once | -| rehearsing it | `x drill roll-before-migrate --json` plays this failure on a scratch database for `web`, `worker` and `scheduler` ([deploy contract](./07-deploy-contract.md#failures-their-alert-and-their-runbook)) | +| rehearsing it | `x drill roll-before-migrate --json` plays this failure on a scratch database for every role the run's services can start ([deploy contract](./07-deploy-contract.md#failures-their-alert-and-their-runbook)) | | `ultimate framework major skew` in the logs, no refusal | a fleet across a major mid-rollout or mid-rollback; it serves. Finish the rollout: `kubectl rollout status deployment --selector app.kubernetes.io/component=web` | ## A credential leaked diff --git a/docs/ops/07-deploy-contract.md b/docs/ops/07-deploy-contract.md index 631dd8849..84f702563 100644 --- a/docs/ops/07-deploy-contract.md +++ b/docs/ops/07-deploy-contract.md @@ -134,9 +134,17 @@ first. | the replicator stops consuming its slot | WAL grows on the database | [`UltimateReplicationSlotRetention`](./06-runbooks.md#ultimatereplicationslotretention); the volume, [`UltimateVolumeNearFull`](./06-runbooks.md#ultimatevolumenearfull) | [the replication slot is accumulating WAL](./06-runbooks.md#the-replication-slot-is-accumulating-wal) | | open tabs 404 their assets after a deploy | `X_ASSET_RETAIN_UNDECLARED` at web boot | none | [Deployment](../../wiki/Deployment.md#hashed-assets-across-a-deploy--assetsretain) | -Rehearse the order before it is needed: `x drill roll-before-migrate --json` starts `web`, -`worker` and `scheduler` on a scratch database before its migrate, then runs migrate, and holds -`roles-wait`, `serves-after-migrate` and `one-wait-line`. Per role, the evidence is +Rehearse the order before it is needed: `x drill roll-before-migrate --json` starts every role on a +scratch database before its migrate, then runs migrate, and holds `roles-wait`, +`serves-after-migrate` and `one-wait-line`. Per role, the evidence is `.data.drills[].evidence.rollout.roles[]`: `{ role, exit, ready, waitLines, logTail }`; a property -that did not hold is `X_DRILL_FAILED`. `sync` and the replicator are not played — they need a bus -or a logical-replication Postgres. `x drill --list --json` names every drill. +that did not hold is `X_DRILL_FAILED`. + +| Role | Played when | +|---|---| +| `web`, `worker`, `scheduler` | always: `TEST_DATABASE_URL` or `TEST_REPLICATION_URL` | +| `sync` | `TEST_NATS_URL` is set | +| `replicator` | `TEST_REPLICATION_URL` names a `wal_level=logical` server, and the app has an entity on a table | + +A role left out is listed in `.data.rolesNotPlayed[]` as `{ drill, role, needs }`; the `fault-drills` +CI job is red on any. `x drill --list --json` names every drill. `As of 2026-11`. diff --git a/llms.txt b/llms.txt index 7e031fed3..ad55c32de 100644 --- a/llms.txt +++ b/llms.txt @@ -254,6 +254,7 @@ The single documentation source. Source of truth is `wiki/` in this repo, mirror - [Client navigation](https://raw.githubusercontent.com/developerz-ai/ultimate/main/wiki/Client-Navigation.md): soft navigation over server-rendered pages: one config line per surface, swap, prefetch, history, focus. - [MCP and AI](https://raw.githubusercontent.com/developerz-ai/ultimate/main/wiki/MCP-And-AI.md): the dev MCP server, every action as a tool, the `llm()` gateway, evals. - [Agents](https://raw.githubusercontent.com/developerz-ai/ultimate/main/wiki/Agents.md): `agent()` as an action factory, tools as real actions, `hive()`, `agentJob()`, and the at-least-once trap. +- [Working with many agents](https://raw.githubusercontent.com/developerz-ai/ultimate/main/wiki/Many-Agents.md): a worker runs its own files, the coordinator runs the gate once; the machine's memory ledger and core cap. - [Admin dashboard](https://raw.githubusercontent.com/developerz-ai/ultimate/main/wiki/Admin-Dashboard.md): `defineAdmin()` serving list, detail and form per entity with no host code; filters, scopes, sections, related rows, actions with `when` and `batch`, the audit log, the jobs dashboard; zero JavaScript; its MCP surface. - [Scraping](https://raw.githubusercontent.com/developerz-ai/ultimate/main/wiki/Scraping.md): `scrape()` as a job factory, the driver-blind page vocabulary, robots and host gates, per-run egress, a rented CDP browser, usage, sealed sessions, mid-run prompts. - [Auth](https://raw.githubusercontent.com/developerz-ai/ultimate/main/wiki/Auth.md): sessions, OAuth, MFA, API keys and service tokens — the output is an `Actor`. diff --git a/packages/action/src/session-cookie.e2e.test.ts b/packages/action/src/session-cookie.e2e.test.ts new file mode 100644 index 000000000..fb008f60f --- /dev/null +++ b/packages/action/src/session-cookie.e2e.test.ts @@ -0,0 +1,104 @@ +// An action that issues a session sets its cookie with `setCookie(sessionCookie(…))`. Proven over a +// real socket with the real auth package: what the browser is sent is what `verifySession` accepts +// on the way back, and the token never rides in the action's answer. +import { afterAll, describe, expect, test } from 'bun:test'; +import { + createSession, + defineAuth, + memoryAuthAdapter, + readSessionCookie, + sessionCookie, + verifySession, +} from '@ultimat3/auth'; +import { resetLifecycle } from '@ultimat3/core'; +import { defineHttpConfig, httpServer, setCookie } from '@ultimat3/http'; +import { allow } from '@ultimat3/policy'; +import { t } from '@ultimat3/schema'; +import { action } from './action'; +import { toRoute } from './http'; + +const USER_ID = 'user_operator_7'; +const auth = defineAuth({ adapter: memoryAuthAdapter() }); +const Answer = t.object({ userId: t.string }); + +const issueSession = action({ + input: t.object({}), + output: Answer, + policy: allow('issue-session'), + async handle() { + const issued = await createSession(auth.sessions, { userId: USER_ID }); + setCookie(sessionCookie(issued.token, auth.sessions.policy)); + setCookie('theme', 'dark'); + return { userId: issued.session.userId }; + }, +}).named('issueSession'); + +const smuggleHeader = action({ + input: t.object({}), + output: Answer, + policy: allow('smuggle-header'), + handle() { + setCookie('sid=v; Path=/; HttpOnly; Secure; SameSite=Lax\r\nX-Evil: 1'); + return { userId: USER_ID }; + }, +}).named('smuggleHeader'); + +const issueRoute = toRoute(issueSession); +const smuggleRoute = toRoute(smuggleHeader); + +resetLifecycle(); +const handle = httpServer({ + routes: [issueRoute, smuggleRoute], + role: 'web', + config: defineHttpConfig({ + rateLimit: { scope: 'process' }, + port: 0, + hostname: '127.0.0.1', + dev: false, + }), +}).start(); + +afterAll(async () => { + await handle.stop(); + resetLifecycle(); +}); + +const post = (path: string): Promise => + fetch(`${handle.url()}${path}`, { + method: 'POST', + headers: { 'content-type': 'application/json', origin: handle.url() }, + body: '{}', + }); + +describe('an action issuing a session, over a real socket', () => { + test('the browser gets the session cookie, and it verifies on the way back', async () => { + const response = await post(issueRoute.path); + const body = await response.text(); + expect(response.status).toBe(200); + expect(JSON.parse(body)).toEqual({ userId: USER_ID }); + + const [session = '', theme = ''] = response.headers.getSetCookie(); + const { cookieName, absoluteTtlMs } = auth.sessions.policy; + expect(session.startsWith(`${cookieName}=`)).toBe(true); + expect(session).toEndWith( + `; Max-Age=${absoluteTtlMs / 1000}; Path=/; HttpOnly; Secure; SameSite=Lax`, + ); + expect(theme).toStartWith('theme=dark;'); + + const returning = new Request(handle.url(), { + headers: { cookie: session.slice(0, session.indexOf(';')) }, + }); + const token = readSessionCookie(returning, auth.sessions.policy) ?? ''; + expect((await verifySession(auth.sessions, token)).userId).toBe(USER_ID); + expect(body).not.toContain(token); + }); + + test('a line carrying a second header is refused, and neither header is sent', async () => { + const response = await post(smuggleRoute.path); + const problem = (await response.json()) as { readonly code?: string }; + expect(response.status).toBeGreaterThanOrEqual(400); + expect(problem.code).toBe('X_COOKIE_INVALID'); + expect(response.headers.getSetCookie()).toEqual([]); + expect(response.headers.get('x-evil')).toBeNull(); + }); +}); diff --git a/packages/auth/README.md b/packages/auth/README.md index 4c14ccad5..1baa2bb26 100644 --- a/packages/auth/README.md +++ b/packages/auth/README.md @@ -830,6 +830,11 @@ original ceiling, so its cookie takes the time left, not the full TTL: `sessionCookie(issued.token, policy, { maxAgeSeconds: remainingMaxAgeSeconds(issued.session, clock.now()) })` — what `updatePrivileges` already does for you. +An action sets any of these lines with `@ultimat3/http`'s `setCookie(line)`, `As of 2026-11`: +`setCookie(sessionCookie(issued.token, policy))`, or `setCookie(signedIn.cookie)` with what `login()` +answered. This package cannot call it (both are tier 2), so it answers the line and the caller sends +it. No attribute is respelled at the call site. + | Attribute | Attack it closes | |---|---| | `HttpOnly` | XSS reading `document.cookie` and exfiltrating the session | diff --git a/packages/cache/CLAUDE.md b/packages/cache/CLAUDE.md index 3cfcfb446..f94c12e41 100644 --- a/packages/cache/CLAUDE.md +++ b/packages/cache/CLAUDE.md @@ -82,7 +82,11 @@ Tier 1. Tagged caching + THE invalidation graph. included (scope `'semantic'`, `jitterFraction: 0`). `X_CACHE_TTL_INVALID`, never a resolution. - **`assertTtl` also SPREADS the lease it validated.** `rng` is injected (`LruOptions.rng`, `RedisTierOptions.rng`) — never `Math.random()` at a call site; `rng: () => 0` is the full lease. - `jitterFraction` outside `[0, 1)` is `X_CACHE_JITTER_INVALID`, refused, not clamped. + `jitterFraction` outside `[0, 1)` is `X_CACHE_JITTER_INVALID`, refused, not clamped, when the + tier is built (`assertJitterFraction`) and again at each write. +- **A rung's limits are constructor options, set through `runtime.ts`'s `cacheTiers`** (`As of + 2026-11`, #1178). The boot builds `lru` and `redis` with defaults; no `app.config.ts` key exists, + because it would be a second spelling of that replacement. Every limit `fix:` names the line. - **`cacheStack` is the production read path, and `@ultimat3/query`'s `readThrough` is its caller** — it is what makes an action's `cache.invalidates` reach a `cache:` query. - **`cacheStack` shares one in-flight `load()` per key** through `@ultimat3/core`'s diff --git a/packages/cache/README.md b/packages/cache/README.md index 1298c252d..0b3f09352 100644 --- a/packages/cache/README.md +++ b/packages/cache/README.md @@ -611,7 +611,10 @@ a job boundary the class is gone and the `code` is what survives — match on th (`CacheDriverUnavailableError`, `CachePurgeFailedError`). An app hands its own to the boot in `apps/web/runtime.ts` — `cacheTiers: [myRedisTier]` replaces the rung of that name (`cache.tiers` still decides which rungs exist; a supplied rung it does not name is logged `cache.tier.unnamed` -and unused), `purge: myPurgeDriver` replaces the `CLOUDFLARE_*`/`FASTLY_*` selection. The +and unused). It is also the one way to set a built-in rung's limits: `cacheTiers: [lruTier({ +maxBytes })]`, never an `app.config.ts` key. A `redisTier` with no `client` and no `REDIS_URL` or +`VALKEY_URL` dials the local server, and the boot logs `cache.tier.default-redis` +(`usesDefaultRedisClient`). `purge: myPurgeDriver` replaces the `CLOUDFLARE_*`/`FASTLY_*` selection. The conformance suites are `@ultimat3/testing/conformance`'s `cacheTierConformance` and `purgeDriverConformance`. diff --git a/packages/cache/src/errors.test.ts b/packages/cache/src/errors.test.ts index 31329ce3a..c306ff271 100644 --- a/packages/cache/src/errors.test.ts +++ b/packages/cache/src/errors.test.ts @@ -115,7 +115,7 @@ describe('CacheTooLargeError', () => { expect(EVERY_CODE).toContain(err.code); }); - test('the fix is the constructor option that holds the cap — app.config.ts has no such key', () => { + test('the fix is the runtime.ts rung that holds the cap — app.config.ts has no such key', () => { // `cache.lru.maxBytes` is not a config field (wiki/Configuration.md): a fix naming it sent the // reader to write a key the boot refuses with X_CONFIG_INVALID. const lru = new CacheTooLargeError({ key: 'k', bytes: 5_000, tier: 'lru', maxBytes: 1_000 }); @@ -126,8 +126,9 @@ describe('CacheTooLargeError', () => { maxBytes: 1_000, }); - expect(lru.fix).toContain('lruTier({ maxBytes: 5000 })'); - expect(redis.fix).toContain('redisTier({ maxValueBytes: 5000 })'); + expect(lru.fix).toStartWith('cacheTiers: [lruTier({ maxBytes: 5000 })]'); + expect(redis.fix).toStartWith('cacheTiers: [redisTier({ maxValueBytes: 5000 })]'); + for (const err of [lru, redis]) expect(err.fix).toContain('apps/web/runtime.ts'); for (const err of [lru, redis]) expect(err.fix).not.toContain('app.config.ts'); }); }); @@ -141,12 +142,22 @@ describe('CacheJitterInvalidError', () => { expect(err.cause).toContain('1.5'); // The option of the factory that read it — `cache.redis.jitterFraction` is no config key. expect(err.fix).toStartWith('{ jitterFraction: 0.05 }'); - expect(err.fix).toContain("redis tier's lruTier({ ... }) or redisTier({ ... })"); + expect(err.fix).toContain('cacheTiers: [redisTier({ jitterFraction: 0.05 })]'); + expect(err.fix).toContain('apps/web/runtime.ts'); + expect(err.fix).not.toContain('lruTier'); expect(err.meta).toEqual({ tier: 'redis', jitterFraction: 1.5 }); expect(EVERY_CODE).toContain(err.code); }); }); +describe('a tier name that is also an Object.prototype member', () => { + test('is named as written, never replaced by the inherited function', () => { + const err = new CacheJitterInvalidError({ tier: 'constructor', jitterFraction: 2 }); + expect(err.fix).toContain('in constructor(...)'); + expect(err.fix).not.toContain('[native code]'); + }); +}); + describe('CachePurgeFailedError', () => { test('names the driver, the status and the detail, and carries the given fix', () => { const err = new CachePurgeFailedError({ diff --git a/packages/cache/src/errors.ts b/packages/cache/src/errors.ts index 1fd69e999..d1f8ee697 100644 --- a/packages/cache/src/errors.ts +++ b/packages/cache/src/errors.ts @@ -67,11 +67,17 @@ export class CacheTagUnknownError extends UltimateError { } } +/** + * Where an app tunes a rung. The boot builds `lru` and `redis` with their defaults and takes a + * replacement from `runtime.ts`'s `cacheTiers`: the one way, so every fix below names it. Limits + * are not `app.config.ts` keys — they would be a second spelling of that same replacement. + */ +const RUNG_FILE = 'apps/web/runtime.ts'; + /** A single entry cannot fit the tier's byte budget, so caching it would evict everything. */ export class CacheTooLargeError extends UltimateError { constructor(input: { key: string; bytes: number; maxBytes: number; tier: string }) { const bytes = String(input.bytes); - // The cap is a constructor option on every tier, never an `app.config.ts` key. const raise = input.tier === 'redis' ? `redisTier({ maxValueBytes: ${bytes} })` @@ -79,7 +85,7 @@ export class CacheTooLargeError extends UltimateError { super({ code: 'X_CACHE_TOO_LARGE', cause: `entry "${input.key}" is ${input.bytes}B, over the ${input.tier} budget of ${input.maxBytes}B`, - fix: `${raise} where the tier is built, or cache a projection instead of the row`, + fix: `cacheTiers: [${raise}] // in ${RUNG_FILE}, replacing the rung the boot builds — or cache a projection instead of the row`, }); } } @@ -128,10 +134,7 @@ const limitEffect = (value: number): string => ? ', and every comparison against NaN is false, so the limit stops being enforced rather than being enforced wrongly — NaN is what Number(process.env.…) answers for an unset variable, and it is not nullish, so `??` does not catch it' : ', and it is applied as given, so the tier is bounded at a value nobody chose'; -/** - * The factory each scope's limits are constructor options of — the fix names where the corrected - * option goes. None is an `app.config.ts` key: the runtime builds every tier with defaults. - */ +/** The factory each scope's limits are constructor options of: where the corrected one goes. */ const LIMIT_OWNER: Readonly> = { lru: 'lruTier', redis: 'redisTier', @@ -141,6 +144,20 @@ const LIMIT_OWNER: Readonly> = { ladder: 'cacheStack', }; +/** + * Where a corrected tier option goes. `lru` and `redis` are the rungs a boot builds, so theirs also + * names the `cacheTiers` line: an app on the defaults has no `lruTier(...)` call of its own to edit. + */ +function optionHome(tier: string, option: string, example: string): string { + // `hasOwn`: `tier` is a caller's string, and `constructor` must not read the prototype. + const owner = Object.hasOwn(LIMIT_OWNER, tier) ? LIMIT_OWNER[tier] : tier; + const rung = + tier === 'lru' || tier === 'redis' + ? `; for the rung a boot builds, cacheTiers: [${owner}({ ${option}: ${example} })] in ${RUNG_FILE}` + : ''; + return `${owner}(...)${rung} — a constructor option, never an app.config.ts key`; +} + export class CacheLimitInvalidError extends UltimateError { constructor(input: { option: string; @@ -155,9 +172,7 @@ export class CacheLimitInvalidError extends UltimateError { */ source?: string | undefined; }) { - const where = - input.source ?? - `${LIMIT_OWNER[input.tier] ?? input.tier}(...) — a constructor option, never an app.config.ts key`; + const where = input.source ?? optionHome(input.tier, input.option, input.example); super({ code: 'X_CACHE_LIMIT_INVALID', cause: `the ${input.tier} tier was given ${input.option}=${String(input.value)}; it must be ${input.expected}${limitEffect(input.value)}`, @@ -205,9 +220,7 @@ export class CacheJitterInvalidError extends UltimateError { cause: `the ${input.tier} tier was configured with jitterFraction=${String( input.jitterFraction, )}; a jitter fraction is a finite number in [0, 1)`, - // A constructor option of the tier, never an `app.config.ts` key: the runtime builds both - // tiers with none (`runtime-cache.ts`), so the config path this used to name did not exist. - fix: `{ jitterFraction: 0.05 } // in the ${input.tier} tier's lruTier({ ... }) or redisTier({ ... }): a value in [0, 1) — 0.05 is the default, 0 disables jitter`, + fix: `{ jitterFraction: 0.05 } // a value in [0, 1) — 0.05 is the default, 0 disables jitter — in ${optionHome(input.tier, 'jitterFraction', '0.05')}`, meta: { tier: input.tier, jitterFraction: input.jitterFraction }, }); } diff --git a/packages/cache/src/index.ts b/packages/cache/src/index.ts index 03f9caaf8..4f0ac0bce 100644 --- a/packages/cache/src/index.ts +++ b/packages/cache/src/index.ts @@ -68,7 +68,7 @@ export type { FastlyPurgeOptions } from './purge-fastly'; export { fastlyPurgeDriver } from './purge-fastly'; export type { PurgeFetch } from './purge-http'; export type { RedisLike, RedisTierOptions } from './redis'; -export { namespaceFor, redisTier } from './redis'; +export { namespaceFor, redisTier, usesDefaultRedisClient } from './redis'; export type { InvalidationBus, InvalidationSubscription, diff --git a/packages/cache/src/lru.test.ts b/packages/cache/src/lru.test.ts index b94915e46..31c2e20a7 100644 --- a/packages/cache/src/lru.test.ts +++ b/packages/cache/src/lru.test.ts @@ -266,6 +266,10 @@ describe('LruCache refuses a byte budget that is not a budget', () => { expect((caught as UltimateError).cause).toContain('NaN'); expect((caught as UltimateError).fix).toStartWith('{ maxBytes: 10_000 }'); expect((caught as UltimateError).fix).toContain('lruTier(...)'); + // The rung a boot builds has no call of the app's to edit: the fix names where one goes. + expect((caught as UltimateError).fix).toContain( + 'cacheTiers: [lruTier({ maxBytes: 10_000 })] in apps/web/runtime.ts', + ); expect((caught as UltimateError).fix).not.toContain('lru.maxBytes in app.config.ts'); }); diff --git a/packages/cache/src/lru.ts b/packages/cache/src/lru.ts index 502e60090..87f35bb39 100644 --- a/packages/cache/src/lru.ts +++ b/packages/cache/src/lru.ts @@ -17,7 +17,13 @@ import type { TierInvalidation, TtlJitter, } from './tiers'; -import { assertFiniteCapacity, assertFiniteDurationMs, assertTtl, nowMs } from './tiers'; +import { + assertFiniteCapacity, + assertFiniteDurationMs, + assertJitterFraction, + assertTtl, + nowMs, +} from './tiers'; import { decodeCacheValue, encodeCacheValue } from './value-codec'; export interface LruOptions { @@ -130,6 +136,9 @@ export class LruCache { options.defaultTtlMs ?? 60_000, ); this.clock = options.clock ?? systemClock; + if (options.jitterFraction !== undefined) { + assertJitterFraction('lru', options.jitterFraction); + } this.jitter = { ...(options.jitterFraction === undefined ? {} : { jitterFraction: options.jitterFraction }), ...(options.rng === undefined ? {} : { rng: options.rng }), diff --git a/packages/cache/src/redis.test.ts b/packages/cache/src/redis.test.ts index 0f2aaa9d7..6a97559e1 100644 --- a/packages/cache/src/redis.test.ts +++ b/packages/cache/src/redis.test.ts @@ -8,7 +8,7 @@ import { describe, expect, test } from 'bun:test'; import { appVersion, frozenClock } from '@ultimat3/core'; import { lruTier } from './lru'; -import { namespaceFor, redisTier } from './redis'; +import { namespaceFor, redisTier, usesDefaultRedisClient } from './redis'; import { bucketsJoined, bucketsRead, fakeRedis, keysOf, tierFor } from './redis-fake-fixture'; import { tag } from './tags'; import { cacheStack } from './tiers'; @@ -278,3 +278,17 @@ describe('the redis lease is spread', () => { expect(client.sent.find((entry) => entry[0] === 'SET')?.[4]).toBe('300000'); }); }); + +// The boot cannot read a tier's options, and it owes an operator one line when a rung it was handed +// will dial the local default because no Redis URL is set. +describe('usesDefaultRedisClient', () => { + test('true for a tier built with no client, false for one handed its own', () => { + expect(usesDefaultRedisClient(redisTier({ maxValueBytes: 2_048 }))).toBe(true); + expect(usesDefaultRedisClient(redisTier({ client: fakeRedis() }))).toBe(false); + }); + + test('false for a tier that is not this factory, whatever its name', () => { + expect(usesDefaultRedisClient(lruTier())).toBe(false); + expect(usesDefaultRedisClient({ ...redisTier(), name: 'redis' })).toBe(false); + }); +}); diff --git a/packages/cache/src/redis.ts b/packages/cache/src/redis.ts index 1a7329f8b..3c272cbeb 100644 --- a/packages/cache/src/redis.ts +++ b/packages/cache/src/redis.ts @@ -30,7 +30,13 @@ import type { TierInvalidation, TtlJitter, } from './tiers'; -import { assertFiniteCapacity, assertFiniteDurationMs, assertTtl, nowMs } from './tiers'; +import { + assertFiniteCapacity, + assertFiniteDurationMs, + assertJitterFraction, + assertTtl, + nowMs, +} from './tiers'; import { decodeCacheValue, encodeCacheValue } from './value-codec'; /** The slice of Bun's Redis client this tier uses. Narrow on purpose: easy to fake in tests. */ @@ -144,6 +150,16 @@ function raiseSweepFailure(failures: readonly unknown[], attempted: number): nev }); } +/** The tiers `redisTier` built with no `client`: the ones core's `redisClient()` will dial for. */ +const onDefaultClient = new WeakSet(); + +/** + * Whether `tier` is a `redisTier` left to core's default client, which reads `REDIS_URL`, then + * `VALKEY_URL`, then the local server. A tier's options are closed over, so the boot asks here: a + * rung it did not build still owes an operator a word when no URL is set (`runtime-cache.ts`). + */ +export const usesDefaultRedisClient = (tier: CacheTier): boolean => onDefaultClient.has(tier); + export function redisTier(options: RedisTierOptions = {}): CacheTier { const ns = namespaceFor(options.prefix ?? 'x', options.buildId); const defaultTtlMs = assertFiniteDurationMs( @@ -157,6 +173,9 @@ export function redisTier(options: RedisTierOptions = {}): CacheTier { 'maxValueBytes', options.maxValueBytes ?? DEFAULT_REDIS_MAX_VALUE_BYTES, ); + if (options.jitterFraction !== undefined) { + assertJitterFraction('redis', options.jitterFraction); + } const jitter: TtlJitter = { ...(options.jitterFraction === undefined ? {} : { jitterFraction: options.jitterFraction }), ...(options.rng === undefined ? {} : { rng: options.rng }), @@ -185,7 +204,7 @@ export function redisTier(options: RedisTierOptions = {}): CacheTier { const { valueKey, writeBucketsFor, bustBucketsFor, sweepBucketsFor } = bucketLayout(ns); - return { + const tier: CacheTier = { name: 'redis', /** @@ -315,4 +334,6 @@ export function redisTier(options: RedisTierOptions = {}): CacheTier { return { tier: 'redis', keys: stripped }; }, }; + if (options.client === undefined) onDefaultClient.add(tier); + return tier; } diff --git a/packages/cache/src/tiers.ts b/packages/cache/src/tiers.ts index 3fab1ed6d..03583e57b 100644 --- a/packages/cache/src/tiers.ts +++ b/packages/cache/src/tiers.ts @@ -103,7 +103,7 @@ export type TtlScope = TierName | 'semantic'; /** * A tier's ceiling — bytes or entries — screened where it is still nameable. The same rule as - * `assertTtl` beside it, over the other knob an `app.config.ts` carries. A SEPARATE refusal from + * `assertTtl` beside it, over the other knob a tier's constructor takes. A SEPARATE refusal from * `X_CACHE_TOO_LARGE`, which is one entry against a valid ceiling: this one is the ceiling itself, * and it needs a screen because a bad ceiling refuses nothing and evicts nothing rather than * refusing everything. `NaN > x` and `x > NaN` are both false. @@ -165,6 +165,15 @@ export function assertFiniteSimilarityFloor( }); } +/** + * A tier's TTL spread, screened when the tier is BUILT as well as at each write: found only at the + * first `set`, a bad fraction refused every write of a tier that had booted healthy. + */ +export function assertJitterFraction(tier: TtlScope, fraction: number): number { + if (Number.isFinite(fraction) && fraction >= 0 && fraction < 1) return fraction; + throw new CacheJitterInvalidError({ tier, jitterFraction: fraction }); +} + export function assertTtl( key: string, ttlMs: number, @@ -174,10 +183,10 @@ export function assertTtl( if (!Number.isFinite(ttlMs) || ttlMs <= 0) { throw new CacheTtlInvalidError({ key, ttlMs, tier }); } - const fraction = jitter.jitterFraction ?? DEFAULT_TTL_JITTER_FRACTION; - if (!Number.isFinite(fraction) || fraction < 0 || fraction >= 1) { - throw new CacheJitterInvalidError({ tier, jitterFraction: fraction }); - } + const fraction = + jitter.jitterFraction === undefined + ? DEFAULT_TTL_JITTER_FRACTION + : assertJitterFraction(tier, jitter.jitterFraction); if (fraction === 0) return ttlMs; // Clamped rather than trusted: an `rng` outside [0, 1) would EXTEND the lease past what the // caller asked for, which is a stale read no reader can explain. diff --git a/packages/cache/src/ttl-jitter.test.ts b/packages/cache/src/ttl-jitter.test.ts index da6f2c802..b68821b61 100644 --- a/packages/cache/src/ttl-jitter.test.ts +++ b/packages/cache/src/ttl-jitter.test.ts @@ -4,6 +4,8 @@ import { describe, expect, test } from 'bun:test'; import { CacheJitterInvalidError, CacheTtlInvalidError } from './errors'; +import { lruTier } from './lru'; +import { redisTier } from './redis'; import { assertTtl, DEFAULT_TTL_JITTER_FRACTION } from './tiers'; describe('assertTtl spreads the lease it validates', () => { @@ -32,6 +34,15 @@ describe('assertTtl spreads the lease it validates', () => { } }); + // At first write a bad fraction refused every `set`, and the stack reads a refused write as a + // tier failure: a boot that came up healthy and cached nothing. + test('a tier refuses a bad fraction when it is built, before any write', () => { + for (const jitterFraction of [1, 1.5, -0.1, Number.NaN]) { + expect(() => lruTier({ jitterFraction })).toThrow(CacheJitterInvalidError); + expect(() => redisTier({ jitterFraction })).toThrow(CacheJitterInvalidError); + } + }); + test('an rng outside [0, 1) can never EXTEND the lease past what the caller asked for', () => { expect(assertTtl('k', 1_000, 'lru', { rng: () => -5 })).toBe(1_000); expect(assertTtl('k', 1_000, 'lru', { rng: () => 99 })).toBe(950); diff --git a/packages/cli/src/cmd-drill.ts b/packages/cli/src/cmd-drill.ts index 0c9cda110..291c9c51a 100644 --- a/packages/cli/src/cmd-drill.ts +++ b/packages/cli/src/cmd-drill.ts @@ -11,6 +11,7 @@ import type { DrillDeclarations, DrillFleet, DrillReport, + DrillRoleNotPlayed, } from '@ultimat3/testing'; import { bunDrillSql, @@ -21,6 +22,7 @@ import { drillBudgets, drillNamed, drillRoles, + drillRolesNotPlayed, drillServices, runDrill, runRollout, @@ -38,6 +40,7 @@ import { readIntFlag } from './flag-number'; import { msg } from './messages'; import type { CommandResult, Finding, JsonValue } from './output'; import { flagBool } from './parse'; +import { replicatedRelations } from './role-replicator'; /** Read through `readIntFlag`, which refuses `5s`, `NaN` and a fraction with the flag's name. */ const BUDGET_FLAG = 'budget-ms'; @@ -177,9 +180,20 @@ export const drillCommand: CliCommand = { } const budgets = drillBudgets(drillConfigOf(config), override); const { targets, skipped } = await drillTargets(root); + // After `drillTargets`: loading the app is what fills the entity registry read here. + const publishable = replicatedRelations().length > 0; const reports: DrillReport[] = []; + const rolesNotPlayed: (DrillRoleNotPlayed & { readonly drill: string })[] = []; for (const { drill, services } of planned) { const { fault } = drill; + const found = { + bus: services.nats !== undefined, + logicalPostgres: services.logicalPostgres === true, + publishable, + }; + for (const gap of drillRolesNotPlayed(drill, found)) { + rolesNotPlayed.push({ drill: drill.name, ...gap }); + } // The rollout order: the stamp is this package's to write, so the rewind is handed in. const order = fault.kind === 'order' @@ -188,7 +202,7 @@ export const drillCommand: CliCommand = { const fleet = await startDrillFleet({ root, services, - roles: drillRoles(drill), + roles: drillRoles(drill, found), readyTimeoutMs: READY_TIMEOUT_MS, beforeMigrate: order, }); @@ -209,6 +223,9 @@ export const drillCommand: CliCommand = { lines: [ ...reports.map((report) => ` ${report.verdict.ok ? 'held' : 'FAILED'} ${report.name}`), ...notRun.map((drill) => ` not run ${drill}`), + ...rolesNotPlayed.map( + (gap) => ` not played ${gap.drill}: ${gap.role}, needs ${gap.needs}`, + ), ], findings, data: { @@ -218,6 +235,8 @@ export const drillCommand: CliCommand = { skippedWrites: skipped.map((entry) => ({ ...entry })), drills: reports.map(reportJson), notRun, + // A role the rollout order left out, with what it needs: CI refuses a non-empty list. + rolesNotPlayed: rolesNotPlayed.map((gap) => ({ ...gap })), }, }; }, diff --git a/packages/cli/src/cmd-heavy-spec.ts b/packages/cli/src/cmd-heavy-spec.ts new file mode 100644 index 000000000..67f003bb6 --- /dev/null +++ b/packages/cli/src/cmd-heavy-spec.ts @@ -0,0 +1,28 @@ +// `x heavy`'s declaration, apart from its body: the parser, `x help` and the `errors` step read it +// without loading `cmd-heavy.ts`, which `registry.ts` imports only when the command runs. + +import type { CommandSpec } from './parse'; + +export const heavySpec: CommandSpec = { + name: 'heavy', + summary: + "the machine's shared memory ledger: who is running, who is queued, and any command run inside it", + usage: 'x heavy [status] [--json] · x heavy run [--label s] [--mb n] [--json] -- ', + subcommands: ['status', 'run'], + defaultSubcommand: 'status', + passthrough: true, + flags: [ + { + name: 'label', + type: 'string', + summary: 'what other waiters see this run called (default: the command)', + subcommands: ['run'], + }, + { + name: 'mb', + type: 'string', + summary: 'the memory to reserve, its measured peak (default: a whole-repo typecheck)', + subcommands: ['run'], + }, + ], +}; diff --git a/packages/cli/src/cmd-heavy.test.ts b/packages/cli/src/cmd-heavy.test.ts new file mode 100644 index 000000000..c1fc691f5 --- /dev/null +++ b/packages/cli/src/cmd-heavy.test.ts @@ -0,0 +1,170 @@ +// `x heavy` end to end, through the real bin: what a caller depends on is its exit code, its streams +// and what becomes of the slot when it is killed. + +import { afterEach, describe, expect, test } from 'bun:test'; +// why: a temp ledger per test; Bun ships no mkdtemp. +import { mkdtempSync, rmSync } from 'node:fs'; +// why: the test's scratch root. +import { tmpdir } from 'node:os'; +// why: Bun ships no path-join primitive. +import { join } from 'node:path'; + +// flock(2) does not exist on Windows, where admission is off (`heavy-flock.ts`). +const NO_FLOCK = process.platform === 'win32'; +const BIN = join(import.meta.dir, 'bin.ts'); +const dirs: string[] = []; +const started: { kill(signal: 'SIGKILL'): void }[] = []; +afterEach(() => { + for (const child of started.splice(0)) child.kill('SIGKILL'); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function ledgerEnv(memory = '100m') { + const dir = mkdtempSync(join(tmpdir(), 'heavy-cmd-')); + dirs.push(dir); + const { CI: _ci, ULTIMATE_HEAVY_HOLD: _hold, ...rest } = Bun.env; + return { ...rest, ULTIMATE_HEAVY_DIR: dir, ULTIMATE_HEAVY_MEMORY: memory }; +} + +const x = (args: readonly string[], env: Record) => { + const child = Bun.spawn(['bun', BIN, ...args], { + env, + stdout: 'pipe', + stderr: 'pipe', + stdin: 'ignore', + }); + started.push(child); + return child; +}; + +async function finished(child: ReturnType) { + const [out, err, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + return { out, err, code }; +} + +async function statusOf(env: Record) { + const { out } = await finished(x(['heavy', 'status', '--json'], env)); + return JSON.parse(out).data as { + enabled: boolean; + capacityMb: number; + holds: { label: string }[]; + queue: { label: string }[]; + }; +} + +/** Starts a holder and returns once the shell inside it has run, so the hold is real. */ +async function holding(env: Record, label: string) { + const child = x( + ['heavy', 'run', '--label', label, '--mb', '100', '--', 'sh', '-c', 'echo up; exec sleep 60'], + env, + ); + const reader = child.stdout.getReader(); + await reader.read(); + return child; +} + +describe.skipIf(NO_FLOCK)('x heavy run', () => { + test('passes the exit code through and keeps stdout the child’s', async () => { + const run = await finished( + x(['heavy', 'run', '--mb', '1', '--', 'sh', '-c', 'echo out; exit 7'], ledgerEnv()), + ); + expect(run.code).toBe(7); + expect(run.out).toBe('out\n'); + }, 30_000); + + test('--json leaves one document on stdout and moves the child to stderr', async () => { + const run = await finished( + x(['heavy', 'run', '--json', '--mb', '1', '--', 'sh', '-c', 'echo out'], ledgerEnv()), + ); + expect(JSON.parse(run.out).data.exitCode).toBe(0); + expect(run.err).toContain('out'); + }, 30_000); + + test('a nested run inherits the hold instead of waiting on its own parent', async () => { + const inner = `bun ${BIN} heavy run --mb 100 -- sh -c 'echo inner'`; + const run = await finished( + x(['heavy', 'run', '--mb', '100', '--', 'sh', '-c', inner], ledgerEnv()), + ); + expect(run.code).toBe(0); + expect(run.out).toContain('inner'); + }, 30_000); + + test('a waiter says who holds the memory, on stderr only, and SIGTERM frees the holder', async () => { + const env = ledgerEnv(); + const holder = await holding(env, 'first'); + const waiter = x(['heavy', 'run', '--mb', '100', '--', 'sh', '-c', 'echo second'], env); + const reader = waiter.stderr.getReader(); + const { value } = await reader.read(); + const line = new TextDecoder().decode(value); + expect(line).toContain('position 1 of 1'); + expect(line).toContain('first'); + holder.kill('SIGTERM'); + expect(await holder.exited).toBe(143); + expect(await waiter.exited).toBe(0); + expect(await new Response(waiter.stdout).text()).toBe('second\n'); + }, 30_000); + + test('kill -9 of the wrapper keeps the reservation until the workload is gone', async () => { + const env = ledgerEnv(); + const holder = x( + [ + 'heavy', + 'run', + '--label', + 'doomed', + '--mb', + '100', + '--', + 'sh', + '-c', + 'echo $$; exec sleep 60', + ], + env, + ); + const { value } = await holder.stdout.getReader().read(); + const workload = Number(new TextDecoder().decode(value).trim()); + holder.kill('SIGKILL'); + await holder.exited; + expect((await statusOf(env)).holds.map((hold) => hold.label)).toEqual(['doomed']); + process.kill(workload, 'SIGKILL'); + // Each read is a whole process, so this loop paces itself. + while ((await statusOf(env)).holds.length > 0); + }, 30_000); + + test('a command that does not exist is 127 with a coded finding', async () => { + const run = await finished( + x(['heavy', 'run', '--json', '--', 'no-such-tool-here'], ledgerEnv()), + ); + expect(run.code).toBe(127); + expect(JSON.parse(run.out).findings[0].code).toBe('X_HEAVY_COMMAND_NOT_FOUND'); + }, 30_000); + + test('no command is refused with a runnable example', async () => { + const run = await finished(x(['heavy', 'run', '--json'], ledgerEnv())); + expect(run.code).toBe(1); + expect(JSON.parse(run.out).findings[0].code).toBe('X_CLI_BAD_FLAG'); + }, 30_000); +}); + +describe.skipIf(NO_FLOCK)('x heavy status', () => { + test('reports capacity and the machine', async () => { + const data = await statusOf(ledgerEnv('2g')); + expect(data.enabled).toBe(true); + expect(data.capacityMb).toBe(2048); + }, 30_000); + + test('says why admission is off under CI', async () => { + const { out } = await finished(x(['heavy', '--json'], { ...ledgerEnv(), CI: 'true' })); + expect(JSON.parse(out).data).toMatchObject({ enabled: false, off: 'ci' }); + }, 30_000); + + test('a bad memory override is a coded failure, not a crash', async () => { + const run = await finished(x(['heavy', '--json'], { ...ledgerEnv('lots') })); + expect(run.code).toBe(1); + expect(JSON.parse(run.out).findings[0].code).toBe('X_HEAVY_MEMORY_INVALID'); + }, 30_000); +}); diff --git a/packages/cli/src/cmd-heavy.ts b/packages/cli/src/cmd-heavy.ts new file mode 100644 index 000000000..16f86a5c7 --- /dev/null +++ b/packages/cli/src/cmd-heavy.ts @@ -0,0 +1,160 @@ +// `x heavy`: the machine ledger as a command. `status` is a read that changes nothing; `run` puts any +// command inside one reservation, passes its exit code through and forwards stop signals, which is +// how a repository with no Ultimate in it shares the same queue. + +// why: Bun exposes no load average, core count or total-memory primitive. +import { availableParallelism, loadavg, totalmem } from 'node:os'; +import { heavySpec } from './cmd-heavy-spec'; +import type { CliCommand, CommandContext } from './command'; +import { MissingPositionalError } from './errors'; +import { readIntFlag } from './flag-number'; +import { describeEntry, heavyState, withHeavy } from './heavy'; +import { coreCap, runnableTasks } from './heavy-cores'; +import { HeavyCommandNotFoundError } from './heavy-errors'; +import { readLedger } from './heavy-ledger'; +import { availableMemoryBytes, usableMemoryBytes } from './heavy-memory'; +import { DEFAULT_RUN_MB } from './heavy-weights'; +import type { HandOffChild } from './local-cli-handoff'; +import { handOff } from './local-cli-handoff'; +import { msg } from './messages'; +import type { CommandResult, JsonValue } from './output'; +import { findingFrom } from './output'; +import { flagString } from './parse'; + +const gb = (bytes: number): string => (bytes / 2 ** 30).toFixed(1); +const MAX_RUN_MB = 1024 * 1024; + +async function status(ctx: CommandContext): Promise { + const state = await heavyState({ ...ctx.env, ULTIMATE_HEAVY_HOLD: undefined }); + const now = Date.now(); + const view = state.on ? readLedger(state.dir, state.tryLock) : { holds: [], queue: [] }; + const cores = coreCap(ctx.env); + const coreText = cores === undefined ? String(availableParallelism()) : String(cores); + const usedMb = view.holds.reduce((sum, entry) => sum + entry.mb, 0); + const [one = 0, five = 0, fifteen = 0] = loadavg(); + const machine = { + cores: availableParallelism(), + runnable: runnableTasks(), + load: { one, five, fifteen }, + totalBytes: usableMemoryBytes(totalmem()), + availableBytes: availableMemoryBytes(), + }; + const entryJson = (entry: (typeof view.holds)[number]): JsonValue => ({ + pid: entry.pid, + mb: entry.mb, + label: entry.label, + cwd: entry.cwd, + ageMs: Math.max(0, now - entry.at), + }); + return { + ok: true, + command: 'heavy', + summary: state.on + ? msg('cli.heavy.status.summary', { + used: usedMb, + capacity: state.capacityMb, + holds: view.holds.length, + queued: view.queue.length, + cores: coreText, + }) + : msg('cli.heavy.status.off', { reason: state.detail ?? state.off, cores: coreText }), + lines: [ + msg('cli.heavy.status.machine', { + load: one.toFixed(1), + cpus: machine.cores, + runnable: machine.runnable, + free: gb(machine.availableBytes), + total: gb(machine.totalBytes), + }), + ...view.holds.map((entry) => + msg('cli.heavy.status.hold', { entry: describeEntry(entry, now) }), + ), + ...view.queue.map((entry, at) => + msg('cli.heavy.status.queued', { position: at + 1, entry: describeEntry(entry, now) }), + ), + ], + data: { + enabled: state.on, + ...(state.on + ? { dir: state.dir, capacityMb: state.capacityMb, usedMb } + : { off: state.off, ...(state.detail === undefined ? {} : { detail: state.detail }) }), + coreCap: cores ?? null, + machine, + holds: view.holds.map(entryJson), + queue: view.queue.map(entryJson), + }, + }; +} + +const seconds = (ms: number): string => `${(ms / 1000).toFixed(1)}s`; + +async function run(ctx: CommandContext): Promise { + const command = ctx.args.passthrough; + const first = command[0]; + if (first === undefined || ctx.args.positionals.length > 0) { + throw new MissingPositionalError({ + command: 'heavy run', + positional: '-- command', + example: 'x heavy run --label tests -- bun test', + }); + } + // Under `--json` fd 1 is this command's one document, so the child's stdout moves to fd 2. + const stream = ctx.args.json ? undefined : ('stderr' as const); + const envelope = { command: 'heavy', ...(stream === undefined ? {} : { stream }) }; + const path = ctx.env['PATH']; + if (Bun.which(first, { cwd: ctx.cwd, ...(path === undefined ? {} : { PATH: path }) }) === null) { + const finding = findingFrom(new HeavyCommandNotFoundError(first)); + return { ...envelope, ok: false, summary: finding.cause, findings: [finding], exitCode: 127 }; + } + const mb = readIntFlag(ctx.args, { + name: 'mb', + command: 'heavy', + min: 1, + max: MAX_RUN_MB, + example: 'x heavy run --mb 4096 -- bun test', + }); + const label = flagString(ctx.args, 'label') ?? command.slice(0, 3).join(' '); + const queued = performance.now(); + let startedAt = queued; + const code = await withHeavy( + { label, cwd: ctx.cwd, env: ctx.env, unitMb: mb ?? DEFAULT_RUN_MB }, + (child) => { + startedAt = performance.now(); + return handOff( + command, + { ...ctx.env, ...child.env }, + { + spawn: (argv, env): HandOffChild => + // Untimed: the child is the command the caller asked to run, and its stop signals are + // forwarded by `handOff`. Counted under `cli`'s spawn-timeout pin with that hand-off. + Bun.spawn([...argv], { + cwd: ctx.cwd, + env, + // The reservation's lock goes with it: killing this wrapper frees nothing the + // command is still using. + stdio: ['inherit', ctx.args.json ? 2 : 'inherit', 'inherit', ...child.fds], + }), + }, + ); + }, + ); + const ranMs = performance.now() - startedAt; + const waitedMs = startedAt - queued; + return { + ...envelope, + ok: code === 0, + summary: msg('cli.heavy.run.done', { + command: label, + code, + ran: seconds(ranMs), + waited: seconds(waitedMs), + }), + exitCode: code, + data: { exitCode: code, ranMs: Math.round(ranMs), waitedMs: Math.round(waitedMs) }, + }; +} + +export const heavyCommand: CliCommand = { + spec: heavySpec, + run: (ctx) => (ctx.args.subcommand === 'run' ? run(ctx) : status(ctx)), +}; diff --git a/packages/cli/src/cmd-test.ts b/packages/cli/src/cmd-test.ts index 089a0543c..a5ceaa5f3 100644 --- a/packages/cli/src/cmd-test.ts +++ b/packages/cli/src/cmd-test.ts @@ -19,6 +19,7 @@ import { ok } from './command'; import { BadFlagError, NoTestFilesError } from './errors'; import type { Runner } from './exec'; import { readIntFlag } from './flag-number'; +import { FREE_RUN_FILES } from './heavy-weights'; import { msg } from './messages'; import type { CommandResult, JsonValue } from './output'; import type { ParsedArgs } from './parse'; @@ -245,9 +246,18 @@ export const testCommand: CliCommand = { workers, ...(isolate ? { isolate } : {}), ...(seed === undefined ? {} : { seed }), - // A default-width run leases its workers from the machine pool, so a second `x test` or - // gate on the same box runs narrower instead of doubling its memory. - ...(plan === undefined ? {} : withLease(machineLease(plan.workers, ctx.env))), + // Every run but a worker's own few files enters the machine ledger, so a second `x test` or + // gate on the same box narrows or queues instead of doubling its memory. + ...(files.length <= FREE_RUN_FILES + ? {} + : withLease( + machineLease(ctx.env, { + cwd: ctx.cwd, + label: type === undefined ? 'x test' : `x test ${type}`, + ...(type === undefined ? {} : { type }), + ...(plan === undefined ? { exact: true } : {}), + }), + )), ...(filter === undefined ? {} : { filter }), ...(type === undefined ? {} : { type }), // `kept` is the corpus the run saw. `selected`, not `discovered`: with `--affected` the sample was taken from the narrowed diff --git a/packages/cli/src/codemods/docs-many-agents.test.ts b/packages/cli/src/codemods/docs-many-agents.test.ts new file mode 100644 index 000000000..c1b4bc076 --- /dev/null +++ b/packages/cli/src/codemods/docs-many-agents.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from 'bun:test'; +import { names } from '../templates/naming'; +import { docsFiles } from '../templates/scaffold-docs'; +import { memoryCodemodApp } from './codemod'; +import { codemod, MANY_AGENTS_ROW } from './docs-many-agents'; + +const TABLE = + '| Coverage | c | d |\n| One way | generators, not hand-rolled files: `x` | review |\n'; + +describe('docs-many-agents', () => { + test('adds the row above the One way row, once', async () => { + const app = memoryCodemodApp({ 'CLAUDE.md': TABLE }); + expect((await codemod.run(app)).changed).toEqual(['CLAUDE.md']); + expect(app.files['CLAUDE.md']).toContain(`${MANY_AGENTS_ROW}\n| One way |`); + expect((await codemod.run(app)).changed).toEqual([]); + }); + + test('leaves an app that reworded the table alone', async () => { + const app = memoryCodemodApp({ 'CLAUDE.md': '| Rules | mine |\n' }); + expect(await codemod.run(app)).toEqual({ changed: [], skipped: [] }); + }); + + test('says what a new app is scaffolded with', () => { + const written = docsFiles(names('shop')).find((file) => file.path === 'CLAUDE.md'); + expect(written?.contents).toContain(MANY_AGENTS_ROW); + }); +}); diff --git a/packages/cli/src/codemods/docs-many-agents.ts b/packages/cli/src/codemods/docs-many-agents.ts new file mode 100644 index 000000000..243fafd27 --- /dev/null +++ b/packages/cli/src/codemods/docs-many-agents.ts @@ -0,0 +1,30 @@ +// 31.48.0: CLAUDE.md states who runs what when several agents share a machine, the rule the machine +// ledger exists to back. Prose: it lands where the table stands as `x new` wrote it, and is skipped +// silently where the app reworded that row or already says it. + +import type { Codemod } from './codemod'; +import type { SnippetEdit } from './engine-edits'; +import { editSnippets } from './engine-edits'; + +const CLAUDE = 'CLAUDE.md'; +const NEXT_ROW = '| One way | generators, not hand-rolled files:'; + +/** Terse on purpose: a shaped app's CLAUDE.md sits within bytes of `AGENTS_MD_MAX_BYTES`. */ +export const MANY_AGENTS_ROW = + '| Many agents | a worker runs its own files; the coordinator runs the gate once | `bunx x heavy status` |'; + +export const DOCS_MANY_AGENTS_EDITS: readonly SnippetEdit[] = [ + { from: NEXT_ROW, to: `${MANY_AGENTS_ROW}\n${NEXT_ROW}`, optional: true }, +]; + +export const codemod: Codemod = { + id: 'docs-many-agents', + since: '31.48.0', + summary: "CLAUDE.md's table says a worker runs its own files and the coordinator runs the gate", + async run(app) { + if (((await app.read(CLAUDE)) ?? '').includes('x heavy status')) { + return { changed: [], skipped: [] }; + } + return editSnippets(app, CLAUDE, DOCS_MANY_AGENTS_EDITS); + }, +}; diff --git a/packages/cli/src/codemods/index.ts b/packages/cli/src/codemods/index.ts index 740688b28..6c83a45ca 100644 --- a/packages/cli/src/codemods/index.ts +++ b/packages/cli/src/codemods/index.ts @@ -18,6 +18,7 @@ import { codemod as docsCiParts } from './docs-ci-parts'; import { codemod as docsEnvGuard } from './docs-env-guard'; import { codemod as docsGateProse } from './docs-gate-prose'; import { codemod as docsIslandRows } from './docs-island-rows'; +import { codemod as docsManyAgents } from './docs-many-agents'; import { codemod as docsShippedGuards } from './docs-shipped-guards'; import { codemod as docsTestServices } from './docs-test-services'; import { codemod as e2eSuiteFloor } from './e2e-suite-floor'; @@ -71,4 +72,5 @@ export const CODEMODS: readonly Codemod[] = [ ciArtifactRetention, ciWeeklySchedule, e2eSuiteFloor, + docsManyAgents, ]; diff --git a/packages/cli/src/error-codes-tooling.ts b/packages/cli/src/error-codes-tooling.ts index 5d6cdf447..d2e91ce76 100644 --- a/packages/cli/src/error-codes-tooling.ts +++ b/packages/cli/src/error-codes-tooling.ts @@ -1,8 +1,8 @@ -// The X_* codes of the app tooling commands — `x client`, `x graph` / `x impact`, `x repro` and -// `x explain` — split out of `error-codes.ts` at its 500-line ceiling, as `error-codes-shot.ts` +// The X_* codes of the tooling commands — `x client`, `x graph` / `x impact`, `x repro`, `x explain` +// and `x heavy` — split out of `error-codes.ts` at its 500-line ceiling, as `error-codes-shot.ts` // was. That file folds these into the CLI's one owned union and registers them in its one call. -/** Codes `x client`, `x graph`, `x impact`, `x repro` and `x explain` declare and own. */ +/** Codes `x client`, `x graph`, `x impact`, `x repro`, `x explain` and `x heavy` own. */ export const CLI_TOOLING_ERROR_CODES = [ 'X_CLIENT_SPEC_UNSUPPORTED', 'X_CLIENT_STALE', @@ -13,6 +13,9 @@ export const CLI_TOOLING_ERROR_CODES = [ 'X_REPRO_NOT_REPRODUCED', 'X_REPRO_CAPTURE_OFF', 'X_EXPLAIN_PRODUCTION_DATABASE', + 'X_HEAVY_LEDGER_UNAVAILABLE', + 'X_HEAVY_MEMORY_INVALID', + 'X_HEAVY_COMMAND_NOT_FOUND', ] as const; export type CliToolingErrorCode = (typeof CLI_TOOLING_ERROR_CODES)[number]; @@ -28,4 +31,7 @@ export const CLI_TOOLING_ERROR_TITLES: Readonly>; readonly stdin?: string; + /** + * Descriptors the child inherits, from fd 3 up: the machine ledger's locks (`heavy-ledger.ts`), + * which must outlive this process when it is killed and the child is not. + */ + readonly inheritFds?: readonly number[]; /** * Told the child's pid the moment it exists. A gate step's deadline records it here * (`verify-deadline.ts`): on Windows no process's environment is readable, so the tag sweep @@ -91,9 +96,12 @@ function spawnOrRefuse(command: readonly string[], options: ExecOptions) { return Bun.spawn([head, ...rest], { cwd: options.cwd, env: options.env === undefined ? Bun.env : mergedEnv(options.env), - stdin: options.stdin === undefined ? 'ignore' : new TextEncoder().encode(options.stdin), - stdout: 'pipe', - stderr: 'pipe', + stdio: [ + options.stdin === undefined ? 'ignore' : new TextEncoder().encode(options.stdin), + 'pipe', + 'pipe', + ...(options.inheritFds ?? []), + ], }); } catch (error) { throw new UltimateError({ diff --git a/packages/cli/src/heavy-cores.ts b/packages/cli/src/heavy-cores.ts new file mode 100644 index 000000000..d32d149d1 --- /dev/null +++ b/packages/cli/src/heavy-cores.ts @@ -0,0 +1,51 @@ +// How many cores a local run may take: every one in CI, at most `LOCAL_CORE_CAP` on a shared box, and +// fewer while other sessions' runnable tasks already fill the machine. Oversubscribed cores do not +// finish work sooner; they turn timing-sensitive tests red. + +// why: Bun ships no synchronous file read for `/proc/loadavg`. +import { readFileSync } from 'node:fs'; +// why: Bun exposes no available-parallelism primitive. +import { availableParallelism } from 'node:os'; +import type { HeavyEnv } from './heavy-env'; +import { HEAVY_CORES_ENV, isCi } from './heavy-env'; + +export const LOCAL_CORE_CAP = 4; +/** A busy machine still gets this much, or one run behind a burst never makes progress. */ +export const BUSY_CORE_FLOOR = 2; + +/** The instantaneous runnable count: `/proc/loadavg`'s `running/total`. The load average lags. */ +export function parseRunnable(loadavg: string): number | undefined { + const match = /^\S+\s+\S+\s+\S+\s+(\d+)\/\d+\s/.exec(loadavg); + return match === null ? undefined : Number(match[1]); +} + +/** Runnable tasks now, this reader excluded; 0 where the kernel does not report them. */ +export function runnableTasks(read: () => string = () => readFileSync('/proc/loadavg', 'utf8')) { + try { + return Math.max(0, (parseRunnable(read()) ?? 1) - 1); + } catch { + return 0; + } +} + +/** `undefined` means no ceiling: a CI runner, whose every core is this run's. */ +export function coreCap( + env: HeavyEnv, + cores: number = availableParallelism(), + runnable: number = runnableTasks(), +): number | undefined { + const raw = Number(env[HEAVY_CORES_ENV]); + if (Number.isInteger(raw) && raw > 0) return raw; + if (isCi(env)) return undefined; + const ceiling = Math.min(LOCAL_CORE_CAP, Math.max(1, cores)); + return Math.min(ceiling, Math.max(BUSY_CORE_FLOOR, cores - runnable)); +} + +/** Thread caps for the native tools: tsc is Go, biome is rayon. An exported value wins. */ +export function nativeToolEnv(env: HeavyEnv, cap: number | undefined): Record { + if (cap === undefined) return {}; + return { + ...(env['GOMAXPROCS'] === undefined ? { GOMAXPROCS: String(cap) } : {}), + ...(env['RAYON_NUM_THREADS'] === undefined ? { RAYON_NUM_THREADS: String(cap) } : {}), + }; +} diff --git a/packages/cli/src/heavy-env.ts b/packages/cli/src/heavy-env.ts new file mode 100644 index 000000000..356480cf8 --- /dev/null +++ b/packages/cli/src/heavy-env.ts @@ -0,0 +1,19 @@ +// Every environment variable the machine ledger reads or sets, and the one reading of `CI`. A leaf: +// the worker planner and the error classes need these names without loading the ledger. + +export type HeavyEnv = Readonly>; + +/** The ledger's directory. Set by a test or an unusual machine; also opts a test run back in. */ +export const HEAVY_DIR_ENV = 'ULTIMATE_HEAVY_DIR'; +/** What the ledger admits in total (`8g`), replacing half of RAM. `0` turns it off. */ +export const HEAVY_MEMORY_ENV = 'ULTIMATE_HEAVY_MEMORY'; +/** The local core ceiling, replacing `LOCAL_CORE_CAP`. */ +export const HEAVY_CORES_ENV = 'ULTIMATE_HEAVY_CORES'; +/** Set on a holder's children: the reservation file they run inside. A protocol, not a knob. */ +export const HEAVY_HOLD_ENV = 'ULTIMATE_HEAVY_HOLD'; + +/** A CI runner is single-tenant: nothing to queue behind, and every core is the run's. */ +export const isCi = (env: HeavyEnv): boolean => { + const value = env['CI']; + return value !== undefined && value !== '' && value !== '0' && value !== 'false'; +}; diff --git a/packages/cli/src/heavy-errors.ts b/packages/cli/src/heavy-errors.ts new file mode 100644 index 000000000..fb0668430 --- /dev/null +++ b/packages/cli/src/heavy-errors.ts @@ -0,0 +1,36 @@ +// The failures of the machine memory ledger. Admission is a courtesy between processes, so the two +// environment failures are reported and the work goes ahead; only `x heavy run` refuses anything. + +import { UltimateError } from '@ultimat3/core'; +import { HEAVY_MEMORY_ENV } from './heavy-env'; +import { quoteArg } from './shell-quote'; + +export class HeavyLedgerUnavailableError extends UltimateError { + constructor(reason: string) { + super({ + code: 'X_HEAVY_LEDGER_UNAVAILABLE', + cause: `the machine memory ledger cannot be used: ${reason}; this run continues unadmitted`, + fix: 'x heavy status --json', + }); + } +} + +export class HeavyMemoryInvalidError extends UltimateError { + constructor(raw: string) { + super({ + code: 'X_HEAVY_MEMORY_INVALID', + cause: `${HEAVY_MEMORY_ENV}=${JSON.stringify(raw)} is not a byte size`, + fix: 'export ULTIMATE_HEAVY_MEMORY=8g # or 0 to turn the ledger off', + }); + } +} + +export class HeavyCommandNotFoundError extends UltimateError { + constructor(command: string) { + super({ + code: 'X_HEAVY_COMMAND_NOT_FOUND', + cause: `${JSON.stringify(command)} is not an executable on PATH`, + fix: `bun --print ${quoteArg(`Bun.which(${JSON.stringify(command)})`)} # null: install it or correct the name, then: x heavy run -- `, + }); + } +} diff --git a/packages/cli/src/heavy-flock.ts b/packages/cli/src/heavy-flock.ts new file mode 100644 index 000000000..282b076c6 --- /dev/null +++ b/packages/cli/src/heavy-flock.ts @@ -0,0 +1,73 @@ +// flock(2) through `bun:ffi`: the one primitive the ledger stands on. The kernel drops the lock when +// the last descriptor closes, which includes the holder dying by any signal, so "the holder died" +// and "the lock is free" are one event and nothing has to detect a stale file. +// +// WINDOWS has no flock, and a Node descriptor there is not the HANDLE `LockFileEx` wants, so the +// ledger is OFF on Windows: `x heavy status` says so and the core cap still applies. An honest gap +// beats a pid file that reintroduces the stale-lock race this module exists to remove. + +import { HeavyLedgerUnavailableError } from './heavy-errors'; + +const LOCK_EX = 2; +const LOCK_NB = 4; +const EINTR = 4; + +const LIBC: Readonly> = { + // glibc first; the two musl names are Alpine's, the base of many app images. + linux: { + libs: ['libc.so.6', 'libc.musl-x86_64.so.1', 'libc.musl-aarch64.so.1'], + errno: '__errno_location', + busy: 11, + }, + darwin: { libs: ['libSystem.B.dylib'], errno: '__error', busy: 35 }, +}; + +/** True when this descriptor now holds the lock, false when another one does. */ +export type TryLock = (fd: number) => boolean; + +export type FlockBinding = + | { readonly ok: true; readonly tryLock: TryLock } + | { readonly ok: false; readonly reason: string }; + +const bound = new Map>(); + +async function bind(platform: string): Promise { + const spec = Object.hasOwn(LIBC, platform) ? LIBC[platform] : undefined; + if (spec === undefined) return { ok: false, reason: `flock(2) does not exist on ${platform}` }; + // Imported here so a command that never reaches the ledger never loads FFI. + const { dlopen, FFIType, read } = await import('bun:ffi'); + for (const lib of spec.libs) { + try { + const { symbols } = dlopen(lib, { + flock: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, + [spec.errno]: { args: [], returns: FFIType.ptr }, + }); + const errnoAt = symbols[spec.errno] as unknown as () => Parameters[0]; + const flock = symbols.flock as unknown as (fd: number, operation: number) => number; + return { + ok: true, + tryLock: (fd) => { + if (flock(fd, LOCK_EX | LOCK_NB) === 0) return true; + const errno = read.i32(errnoAt(), 0); + if (errno === spec.busy || errno === EINTR) return false; + // Reading this as "busy" would wait forever on a filesystem that cannot lock at all. + throw new HeavyLedgerUnavailableError( + `flock(2) failed with errno ${String(errno)}, which is not contention`, + ); + }, + }; + } catch { + // Not this libc: the next name, and "no libc" below when none loads. + } + } + return { ok: false, reason: `no libc with flock(2) loaded (${spec.libs.join(', ')})` }; +} + +/** The process's one binding, loaded on first use. */ +export const flockBinding = (platform: string = process.platform): Promise => { + const known = bound.get(platform); + if (known !== undefined) return known; + const binding = bind(platform); + bound.set(platform, binding); + return binding; +}; diff --git a/packages/cli/src/heavy-ledger.test.ts b/packages/cli/src/heavy-ledger.test.ts new file mode 100644 index 000000000..d496cd08b --- /dev/null +++ b/packages/cli/src/heavy-ledger.test.ts @@ -0,0 +1,234 @@ +// The ledger's guarantees are the kernel's, so these run real processes on real files: a fake flock +// would prove only the fake. Every wait is on a line a child printed, never on a guessed delay. + +import { afterEach, describe, expect, test } from 'bun:test'; +// why: temp directories and an append-only event log; Bun ships neither primitive. +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +// why: the test's scratch root. +import { tmpdir } from 'node:os'; +// why: Bun ships no path-join primitive. +import { join } from 'node:path'; +import { flockBinding } from './heavy-flock'; +import { readLedger, unitsThatFit } from './heavy-ledger'; + +// flock(2) does not exist on Windows, where admission is off (`heavy-flock.ts`). +const NO_FLOCK = process.platform === 'win32'; +const FLOCK = join(import.meta.dir, 'heavy-flock.ts'); +const LEDGER = join(import.meta.dir, 'heavy-ledger.ts'); +const dirs: string[] = []; +const scratch = (): string => { + const dir = mkdtempSync(join(tmpdir(), 'heavy-ledger-')); + dirs.push(dir); + return dir; +}; +const started: { kill(signal: 'SIGKILL'): void }[] = []; +afterEach(() => { + // A failed assertion must not leave a holder waiting forever behind the run's own pipes. + for (const child of started.splice(0)) child.kill('SIGKILL'); + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +const tryLock = async () => { + const binding = await flockBinding(); + if (!binding.ok) expect.unreachable(binding.reason); + return binding.tryLock; +}; + +interface ChildPlan { + readonly dir: string; + readonly capacityMb: number; + readonly mb: number; + readonly label: string; + /** Runs once admitted, as source text; the child then prints `held` and awaits it. */ + readonly body?: string; +} + +function reserver(plan: ChildPlan) { + const source = ` + import { flockBinding } from ${JSON.stringify(FLOCK)}; + import { reserve } from ${JSON.stringify(LEDGER)}; + import { appendFileSync } from 'node:fs'; + const { tryLock } = await flockBinding(); + const log = (line) => appendFileSync(${JSON.stringify(join(plan.dir, 'events'))}, line + '\\n'); + console.log('joining'); + const held = await reserve({ dir: ${JSON.stringify(plan.dir)}, capacityMb: ${plan.capacityMb}, + unitMb: ${plan.mb}, units: 1, label: ${JSON.stringify(plan.label)}, cwd: '/w', tryLock, + pollMs: 5, onWait: () => console.log('waiting') }); + log('+ ${plan.mb} ${plan.label}'); + console.log('held'); + ${plan.body ?? 'await new Promise(() => {});'} + log('- ${plan.mb} ${plan.label}'); + held.release(); + process.exit(0);`; + const child = Bun.spawn(['bun', '-e', source], { stdout: 'pipe', stderr: 'inherit' }); + started.push(child); + const reader = child.stdout.getReader(); + let seen = ''; + const until = async (word: string): Promise => { + while (!seen.includes(word)) { + const { value, done } = await reader.read(); + if (done) expect.unreachable(`the child exited before printing ${word}: ${seen}`); + seen += new TextDecoder().decode(value); + } + }; + const untilAny = async (words: readonly string[]): Promise => { + for (;;) { + const found = words.find((word) => seen.includes(word)); + if (found !== undefined) return found; + const { value, done } = await reader.read(); + if (done) expect.unreachable(`the child exited before printing any of ${words.join()}`); + seen += new TextDecoder().decode(value); + } + }; + return { child, until, untilAny, output: () => seen }; +} + +const events = (dir: string): string[] => + readFileSync(join(dir, 'events'), 'utf8').trim().split('\n'); + +describe.skipIf(NO_FLOCK)('reserve', () => { + test('concurrent processes never sum past the capacity', async () => { + const dir = scratch(); + const go = join(dir, 'go'); + // Every holder stays until all twelve have either been admitted or queued, so an admission + // past the capacity cannot hide behind holders that happened to finish first. + const children = Array.from({ length: 12 }, (_, at) => + reserver({ + dir, + capacityMb: 100, + mb: 40, + label: `c${String(at)}`, + body: `while (!(await Bun.file(${JSON.stringify(go)}).exists())) await Bun.sleep(5);`, + }), + ); + const said = await Promise.all(children.map((one) => one.untilAny(['held', 'waiting']))); + expect(said.filter((word) => word === 'held')).toHaveLength(2); + await Bun.write(go, ''); + await Promise.all(children.map((one) => one.child.exited)); + let used = 0; + let peak = 0; + for (const line of events(dir)) { + used += line.startsWith('+') ? 40 : -40; + peak = Math.max(peak, used); + } + expect(events(dir)).toHaveLength(24); + expect(peak).toBe(80); + expect(readdirSync(join(dir, 'holds'))).toEqual([]); + }, 60_000); + + test('kill -9 of a holder frees its share at once, with its file still on disk', async () => { + const dir = scratch(); + const holder = reserver({ dir, capacityMb: 10, mb: 10, label: 'holder' }); + await holder.until('held'); + const waiter = reserver({ dir, capacityMb: 10, mb: 10, label: 'waiter', body: '' }); + await waiter.until('waiting'); + holder.child.kill('SIGKILL'); + expect(await waiter.child.exited).toBe(0); + expect(events(dir)).toEqual(['+ 10 holder', '+ 10 waiter', '- 10 waiter']); + }, 30_000); + + test('kill -9 of the waiter at the head lets the one behind it through', async () => { + const dir = scratch(); + const holder = reserver({ dir, capacityMb: 10, mb: 10, label: 'holder' }); + await holder.until('held'); + const head = reserver({ dir, capacityMb: 10, mb: 10, label: 'head' }); + await head.until('waiting'); + const next = reserver({ dir, capacityMb: 10, mb: 10, label: 'next', body: '' }); + await next.until('waiting'); + head.child.kill('SIGKILL'); + await head.child.exited; + holder.child.kill('SIGKILL'); + expect(await next.child.exited).toBe(0); + expect(events(dir)).not.toContain('+ 10 head'); + }, 30_000); + + test('first come, first served: a small request does not pass a big one', async () => { + const dir = scratch(); + const holder = reserver({ dir, capacityMb: 100, mb: 60, label: 'holder' }); + await holder.until('held'); + const big = reserver({ dir, capacityMb: 100, mb: 80, label: 'big', body: '' }); + await big.until('waiting'); + // 30 MB fits beside the holder right now; only the queue keeps it behind `big`. + const small = reserver({ dir, capacityMb: 100, mb: 30, label: 'small', body: '' }); + await small.until('waiting'); + expect(events(dir)).toEqual(['+ 60 holder']); + holder.child.kill('SIGKILL'); + await Promise.all([big.child.exited, small.child.exited]); + expect(events(dir).slice(1, 3)).toEqual(['+ 80 big', '- 80 big']); + expect(events(dir)[3]).toBe('+ 30 small'); + }, 30_000); + + test('a child of the holder does not keep its lock alive', async () => { + const dir = scratch(); + const holder = reserver({ + dir, + capacityMb: 10, + mb: 10, + label: 'holder', + body: `const kid = Bun.spawn(['sleep', '30'], { stdio: ['ignore', 'ignore', 'ignore'] }); + console.log('kid ' + kid.pid + ' .'); await new Promise(() => {});`, + }); + await holder.until(' .'); + holder.child.kill('SIGKILL'); + await holder.child.exited; + expect(readLedger(dir, await tryLock()).holds).toEqual([]); + }, 30_000); + + test('a child handed the locked descriptor keeps the reservation after kill -9', async () => { + const dir = scratch(); + const holder = reserver({ + dir, + capacityMb: 10, + mb: 10, + label: 'holder', + body: `const kid = Bun.spawn(['sleep', '30'], { + stdio: ['ignore', 'ignore', 'ignore', held.fd] }); + console.log('kid ' + kid.pid + ' .'); await new Promise(() => {});`, + }); + await holder.until(' .'); + const pid = Number(/kid (\d+) \./.exec(holder.output())?.[1]); + holder.child.kill('SIGKILL'); + await holder.child.exited; + const lock = await tryLock(); + expect(readLedger(dir, lock).holds.map((entry) => entry.label)).toEqual(['holder']); + process.kill(pid, 'SIGKILL'); + while (readLedger(dir, lock).holds.length > 0) await new Promise(setImmediate); + }, 30_000); + + test('a request over the capacity runs alone instead of being refused', async () => { + const dir = scratch(); + const huge = reserver({ dir, capacityMb: 10, mb: 50, label: 'huge', body: '' }); + expect(await huge.child.exited).toBe(0); + expect(events(dir)[0]).toBe('+ 50 huge'); + }, 30_000); + + test('the view shows a live holder and a live waiter, oldest first', async () => { + const dir = scratch(); + const holder = reserver({ dir, capacityMb: 10, mb: 10, label: 'holder' }); + await holder.until('held'); + const waiter = reserver({ dir, capacityMb: 10, mb: 5, label: 'waiter' }); + await waiter.until('waiting'); + const view = readLedger(dir, await tryLock()); + expect(view.holds.map((entry) => entry.label)).toEqual(['holder']); + expect(view.queue.map((entry) => entry.label)).toEqual(['waiter']); + holder.child.kill('SIGKILL'); + waiter.child.kill('SIGKILL'); + await Promise.all([holder.child.exited, waiter.child.exited]); + }, 30_000); +}); + +describe('unitsThatFit', () => { + const base = { capacityMb: 100, unitMb: 30, units: 3, minUnits: 1 }; + test('narrows to what is free, down to the minimum', () => { + expect(unitsThatFit({ ...base, usedMb: 0 })).toBe(3); + expect(unitsThatFit({ ...base, usedMb: 50 })).toBe(1); + expect(unitsThatFit({ ...base, usedMb: 80 })).toBe(0); + }); + test('an exact request takes all of it or waits', () => { + expect(unitsThatFit({ ...base, minUnits: 3, usedMb: 50 })).toBe(0); + }); + test('a unit over the capacity is granted once the ledger is empty', () => { + expect(unitsThatFit({ ...base, unitMb: 500, usedMb: 0 })).toBe(1); + expect(unitsThatFit({ ...base, unitMb: 500, usedMb: 1 })).toBe(0); + }); +}); diff --git a/packages/cli/src/heavy-ledger.ts b/packages/cli/src/heavy-ledger.ts new file mode 100644 index 000000000..c02dd53b8 --- /dev/null +++ b/packages/cli/src/heavy-ledger.ts @@ -0,0 +1,358 @@ +// The machine's memory ledger: reservations weighted in MB, admitted first come, first served, every +// one a file its owner holds an flock on. A file nobody holds is dead by definition, so a holder or +// waiter killed by any signal stops counting at that instant and no take-over exists to race. +// +// queue/