refactor(global): shipped files by package and path, the sandbox as a module, exports a shell needs

capabilities/computation/measure/common.ts

@@ -1,10 +1,12 @@
-import { handedWorkersWithChecks } from '@on.auto/definitions/check';
+import { handedWorkersWithChecks, checkShippedRoots } from '@on.auto/definitions/check';
 import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk';
 import { programPool, type Json, type ProgramPool, type ProgramRequest } from '@on.auto/workflow-engine/dsl';
 
 import { computationBounds } from '../src/run/run-bounds.ts';
 
-export const handed = await handedWorkersWithChecks(shippedFilesOnDisk);
+const checkFiles = shippedFilesOnDisk(checkShippedRoots);
+
+export const handed = await handedWorkersWithChecks(checkFiles);
 
 export function millisecondsOf(work: () => void): number {
   const started = performance.now();

capabilities/coordination/src/testing/test-host.ts

@@ -1,5 +1,5 @@
 import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk';
-import { freshInstances, machineSandboxOf } from '@on.auto/workflow-engine/dsl';
+import { freshInstances, machineSandboxOf, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl';
 import { openWorkflowHost, type WorkflowHost } from '@on.auto/workflow-host';
 import { recordedReactions, recordedWaiting } from '@on.auto/workflow-host/testing';
 import { Effect, Function } from 'effect';
@@ -9,10 +9,14 @@ import { workflowMachineOptionsOf } from '../runs/workflow-machine-options.ts';
 import type { Playbook } from './playbook.ts';
 import { testPlaybookNames } from './test-sandbox.ts';
 
+const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots);
+
+const sandboxModule = await sandboxModuleIn(sandboxFiles);
+
 export function testHost(nested: Playbook): Promise<WorkflowHost> {
   return openWorkflowHost({
     database: { store: 'sqlite', file: ':memory:' },
-    machine: workflowMachineOptionsOf(machineSandboxOf(freshInstances(shippedFilesOnDisk)), testPlaybookNames),
+    machine: workflowMachineOptionsOf(machineSandboxOf(freshInstances(sandboxModule)), testPlaybookNames),
     perform: definitionCalls(nested.executeNested),
     settle: nested.settle,
     reports: {

capabilities/coordination/src/testing/test-sandbox.ts

@@ -7,14 +7,16 @@ import {
   type MachineSandbox,
   type MachineSettings,
 } from '@on.auto/workflow-engine';
-import { freshInstances, unitMemoryBytes } from '@on.auto/workflow-engine/dsl';
+import { freshInstances, sandboxModuleIn, unitMemoryBytes, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl';
 
 import { workflowFacts } from '../capability/workflow.ts';
 import { workflowPolicyOf, type WorkflowPolicy } from '../document/workflow-functions.ts';
 import { workflowMachineOptionsOf, workflowMachineSettingsOf } from '../runs/workflow-machine-options.ts';
 
+const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots);
+
 export const testSandbox: MachineSandbox = reusedSandbox(
-  await freshInstances(shippedFilesOnDisk)(unitMemoryBytes),
+  await freshInstances(await sandboxModuleIn(sandboxFiles))(unitMemoryBytes),
   () => 0,
 );
 

capabilities/prediction/measure.ts

@@ -1,11 +1,13 @@
 import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk';
-import { handedWorkersOf, programPool } from '@on.auto/workflow-engine/dsl';
+import { handedWorkersOf, programPool, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl';
 
 import { rebuiltOn, type Folding, type History } from './measure/dataset-rebuild.ts';
 import { dailyOrders, seriesMeasured, testedDailyOrders } from './measure/order-history.ts';
 import { measuredStores, type MeasuredStore } from './measure/stores.ts';
 import { writtenOn, type FileRead } from './measure/written-files.ts';
 
+const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots);
+
 const postgresqlServer = process.env['LEDGER_MEASURE_POSTGRESQL_URL'] ?? '';
 
 const days = 200;
@@ -71,7 +73,7 @@ async function testedOn({ store, aDatabase }: MeasuredStore, history: History):
 
 async function measuredOn(measured: MeasuredStore): Promise<void> {
   const pool = programPool({
-    handed: await handedWorkersOf(shippedFilesOnDisk),
+    handed: await handedWorkersOf(sandboxFiles),
     instances: 'fresh',
     workers: 1,
     heapMegabytes: 256,

capabilities/prediction/src/dataset-testing/projected-datasets.ts

@@ -1,5 +1,5 @@
 import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk';
-import { filterSandboxOf, freshInstances } from '@on.auto/workflow-engine/dsl';
+import { filterSandboxOf, freshInstances, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl';
 import { someFolder, type DatabaseSettings } from '@on.auto/workflow-host';
 import { projectedFolders, type ProjectedFolders } from '@on.auto/workflow-host/testing';
 import { Effect, Option, Result, type Schema } from 'effect';
@@ -10,6 +10,10 @@ import { datasetFolder, datasetsOf, type DatasetFolding } from '../dataset/datas
 import type { DatasetRow } from '../dataset/dataset-rows.ts';
 import { parsePredictionDocument } from '../document/document-parsing.ts';
 
+const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots);
+
+const sandboxModule = await sandboxModuleIn(sandboxFiles);
+
 export const alphaKey = 'playbook/acme/alpha/';
 
 const savedAt = '2026-10-06T09:00:00.000Z';
@@ -86,7 +90,7 @@ type Folding = Partial<Omit<DatasetFolding, 'filters'>>;
 
 export function testDatasetFolder(folding: Folding = {}): ReturnType<typeof datasetFolder> {
   return datasetFolder({
-    filters: filterSandboxOf(freshInstances(shippedFilesOnDisk)),
+    filters: filterSandboxOf(freshInstances(sandboxModule)),
     triesBeforeStall: Number.POSITIVE_INFINITY,
     ...folding,
   });

capabilities/prediction/src/dataset/dataset-matching.test.ts

@@ -1,12 +1,14 @@
 import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk';
-import { filterSandboxOf, freshInstances } from '@on.auto/workflow-engine/dsl';
+import { filterSandboxOf, freshInstances, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl';
 import type { PageEvent } from '@on.auto/workflow-host';
 import { Effect } from 'effect';
 import { describe, expect, it } from 'vitest';
 
 import type { DatasetData } from './dataset-data.ts';
 import { datasetMatchingOf } from './dataset-matching.ts';
 
+const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots);
+
 const dataNamedFirst: DatasetData = {
   period: 'week',
   events: [
@@ -37,7 +39,7 @@ describe('the matching of a dataset whose filters name their data before their t
   it('tests an event against the data tests of the filters of its own type alone, matching each type first', async () => {
     const matching = datasetMatchingOf(
       dataNamedFirst,
-      filterSandboxOf(freshInstances(shippedFilesOnDisk), () => 0),
+      filterSandboxOf(freshInstances(await sandboxModuleIn(sandboxFiles)), () => 0),
     );
 
     const match = await Effect.runPromise(matching.matchOf(placed));

capabilities/prediction/src/files/series-rows.ts

@@ -111,5 +111,5 @@ export function seriesRowsOfStream(
   stream: ReadableStream<Uint8Array>,
   reading: SeriesReading,
 ): Promise<readonly SeriesRows[]> {
-  return seriesRowsFrom(() => Readable.fromWeb(stream), reading);
+  return seriesRowsFrom(() => Readable.from(stream, { objectMode: false }), reading);
 }

capabilities/prediction/src/trainer/candidate-staging.ts

@@ -1,5 +1,4 @@
-import { createReadStream } from 'node:fs';
-import { Readable } from 'node:stream';
+import { open, type FileHandle } from 'node:fs/promises';
 
 import type { Conflict } from '@on.auto/operations';
 import { Effect } from 'effect';
@@ -8,14 +7,39 @@ import type { BuildState } from '../builds/build-state.ts';
 import type { PredictionPorts } from '../capability/prediction-ports.ts';
 import { buildEnded } from './waiting-builds.ts';
 
+const chunkBytes = 1_048_576;
+
+function chunksOf(path: string): ReadableStream<Uint8Array> {
+  const reading: { file?: Promise<FileHandle> } = {};
+  return new ReadableStream<Uint8Array>(
+    {
+      pull: async (controller) => {
+        reading.file ??= open(path);
+        const file = await reading.file;
+        const { bytesRead, buffer } = await file.read({ buffer: new Uint8Array(chunkBytes) });
+        if (bytesRead > 0) {
+          controller.enqueue(buffer.subarray(0, bytesRead));
+          return;
+        }
+        await file.close();
+        controller.close();
+      },
+      cancel: async () => {
+        await (await reading.file)?.close();
+      },
+    },
+    { highWaterMark: 0 },
+  );
+}
+
 export function staged(
   ports: Pick<PredictionPorts, 'ledger'>,
   { address }: Pick<BuildState, 'address'>,
   path: string,
   sha256: string,
 ): Effect.Effect<void, Conflict> {
   const { org, playbook, id } = address;
-  const stream = Readable.toWeb(createReadStream(path));
+  const stream = chunksOf(path);
   return ports.ledger.content
     .stage({ org, playbook }, id, { stream, mediaType: 'application/octet-stream', expected: { sha256 } })
     .pipe(

capabilities/prediction/src/upload/utf8-text.ts

@@ -67,7 +67,7 @@ function validPrefixOf(bytes: Readonly<Uint8Array>): string {
     reached = scannedAt(bytes, at, sequence);
     at += reached === 'whole' ? sequence.length : 0;
   }
-  return new TextDecoder('utf-8', { ignoreBOM: true }).decode(bytes.subarray(0, at));
+  return new TextDecoder('utf-8', { fatal: false, ignoreBOM: true }).decode(bytes.subarray(0, at));
 }
 
 function isContinuation(byte: number | undefined): boolean {

capabilities/recall/measure/folds.ts

@@ -1,4 +1,4 @@
-import { handedWorkersWithChecks } from '@on.auto/definitions/check';
+import { handedWorkersWithChecks, checkShippedRoots } from '@on.auto/definitions/check';
 import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk';
 import { programPool, type FoldRequest, type Json, type JsonObject } from '@on.auto/workflow-engine/dsl';
 import { Effect, Result } from 'effect';
@@ -8,7 +8,9 @@ import { parseRecallDocument } from '../src/document/document-parsing.ts';
 import { recallBounds, recallFolding } from '../src/run/recall-bounds.ts';
 import { campaignReviews } from '../src/testing/campaign-reviews.ts';
 
-const handed = await handedWorkersWithChecks(shippedFilesOnDisk);
+const checkFiles = shippedFilesOnDisk(checkShippedRoots);
+
+const handed = await handedWorkersWithChecks(checkFiles);
 
 export interface Folded {
   readonly events: number;

packages/api/README.md

@@ -13,7 +13,7 @@ Each request passes through the same chain before it reaches an operation, in th
 
 **Signed-in persons.** With sign-in on, the authenticator carries the protected resource, `PUBLIC_URL` and the Auto identity service's issuer, and the API serves `GET /.well-known/oauth-protected-resource` and the same path followed by `/mcp`, `/orgs/{org}/mcp` and `/orgs/{org}/playbooks/{playbook}/mcp` before the chain checks a credential, as public paths matching that prefix, answering [RFC 9728](https://datatracker.ietf.org/doc/html/rfc9728) metadata: `resource` is `PUBLIC_URL` followed by the path after the prefix, `authorization_servers` the identity service's issuer, `bearer_methods_supported` `["header"]`, `scopes_supported` the scopes a sign-in asks for, `signInScopes` of `@on.auto/identity`: `user:org:read`, which asks the identity service for a token bound to the organization the person signs in to, then the four permissions, which the identity service grants as scopes of its own, and `resource_documentation` the public MCP reference; a path under the prefix that names no endpoint, or an org or playbook id that is not one, is `404`. A `401` then carries `Bearer resource_metadata="<PUBLIC_URL>/.well-known/oauth-protected-resource", scope="<the sign-in scopes>"`, the five separated by spaces, after `error="invalid_token"` when a credential was refused, and its detail says a key or a token. With sign-in off none of this is served, the metadata's path needs a key, and the challenges are `Bearer` and `Bearer error="invalid_token"`. The challenges are built in `src/authentication/challenges.ts` and the metadata in `src/authentication/protected-resource.ts`.
 
-Before that chain, a browser opening `/` (a `GET` that accepts `text/html`) gets a page saying the server is running, with a button that opens Studio on this server: the Studio a person can reach, `studioBaseOf` of `@on.auto/identity`, followed by `?server=<this server's origin>`, which is `<this server's origin>/studio/?server=…` where the runtime serves Studio and `https://studio.on.auto/?server=…` for the default Studio host otherwise. The page needs no key and loads nothing from any server: its styles, its icon and its typefaces, DM Mono and DM Sans, are inside it. The two font files and their SIL Open Font Licenses are in `src/landing/fonts`; `readPageStyle(shippedFiles)` reads them through the `ShippedFiles` of `@on.auto/operations` the server composes, once, and answers the page's styles, which its content security policy hashes, as a `PageStyle`, which the server reads at its start and gives the API as `ApiOptions.pageStyle`, so a typeface that cannot be read stops the start (`src/landing/page-style.ts`, `src/landing/fonts.ts`). Any other request to `/` has no route.
+Before that chain, a browser opening `/` (a `GET` that accepts `text/html`) gets a page saying the server is running, with a button that opens Studio on this server: the Studio a person can reach, `studioBaseOf` of `@on.auto/identity`, followed by `?server=<this server's origin>`, which is `<this server's origin>/studio/?server=…` where the runtime serves Studio and `https://studio.on.auto/?server=…` for the default Studio host otherwise. The page needs no key and loads nothing from any server: its styles, its icon and its typefaces, DM Mono and DM Sans, are inside it. The two font files and their SIL Open Font Licenses are in `src/landing/fonts`; `readPageStyle(shippedFiles)` reads them, as files of `@on.auto/api` at the root `apiShippedRoots` declares, through the `ShippedFiles` of `@on.auto/operations` the server composes, once, and answers the page's styles, which its content security policy hashes, as a `PageStyle`, which the server reads at its start and gives the API as `ApiOptions.pageStyle`, so a typeface that cannot be read stops the start (`src/landing/page-style.ts`, `src/landing/fonts.ts`). Any other request to `/` has no route.
 
 **The served Studio.** A page on a public origin may not reach an address on the person's machine, so a runtime may serve Studio itself, at `/studio/` on its own origin. `ApiOptions.studio` says whether it does: a `HostedStudio`, `served: false`, serves none, and `/studio/` has no route; a `ServedStudio`, `served: true`, carries `host`, a `StudioHost`, which answers a path asked of the Studio host with a `StudioAnswer`, its status, content type, body, the seconds a browser may keep it, the host's validators, `ETag` and `Last-Modified`, and, for a redirect, the path on the host it sends the browser to, or with nothing when the host cannot be reached and nothing was kept. `GET` and `HEAD` under `/studio/` answer what the host answers for the same path under its root, without the query, with its status, its content type, its body, its `ETag` and `Last-Modified`, a redirect's `Location` under `/studio/`, and `Cache-Control` `public, max-age=<seconds>, immutable` for what the host keeps fresh or `no-cache` for the rest; every other method is `405` `method_not_allowed` with `Allow: GET, HEAD`, `/studio` is sent on to `/studio/` with its query by `308`, since the page resolves its files against the folder it was served from, and a path that holds `..`, written or with a dot percent-encoded, is `404` `not_found` before the host is asked (`src/studio/served-studio.ts`). When the host answers nothing, the runtime answers `503` with a page of its own, `src/studio/unreachable-studio.ts`, which runs no script and says that Studio cannot be fetched now and from where. A request at `/studio` or under `/studio/` needs no key, since a browser opening a page sends none, and passes the `Origin` check and, in local mode, the `Host` check as every other request does (`src/middleware/public-paths.ts`). Its answers carry a content security policy of their own, `servedStudioPolicyOf` of `src/studio/studio-policy.ts`, written once from what Studio's build loads: `default-src 'none'; script-src 'self'; worker-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; img-src 'self' data:; connect-src 'self'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'`, for its module scripts and the worker of its editor, its style sheets and the style elements it writes as it runs, its fonts and its images, the `data:` images its style sheets hold, and its calls to the runtime, every one from the runtime's origin. The page the host answers holds an inline script, so a `text/html` answer's `script-src` also allows each inline script of that page by its SHA-256, read from the page as the host answered it; every other answer under `/studio/` carries the policy with no hash, set by `src/middleware/response-headers.ts`. Every other path keeps `default-src 'none'; frame-ancestors 'none'`, and the other security headers are the same under `/studio/` as elsewhere.
 

packages/api/src/index.ts

@@ -1,6 +1,7 @@
 export { createApiHandler } from './api-handler.ts';
 export type { ApiHandler } from './api-handler.ts';
 export type { ApiOptions } from './api-options.ts';
+export { apiShippedRoots } from './landing/fonts.ts';
 export { readPageStyle, type PageStyle } from './landing/page-style.ts';
 export type { ApiStudio, HostedStudio, ServedStudio, StudioAnswer, StudioHost } from './studio/served-studio.ts';
 export {

packages/api/src/landing/fonts.ts

@@ -1,8 +1,12 @@
-import type { ShippedFiles } from '@on.auto/operations';
+import type { ShippedFile, ShippedFiles, ShippedRoots } from '@on.auto/operations';
 
-const monoFile = new URL('./fonts/dm-mono-latin-500.woff2', import.meta.url);
+const apiPackage = '@on.auto/api';
 
-const sansFile = new URL('./fonts/dm-sans-latin-variable.woff2', import.meta.url);
+export const apiShippedRoots: ShippedRoots = { [apiPackage]: new URL('../../', import.meta.url) };
+
+const monoFile: ShippedFile = { package: apiPackage, path: 'src/landing/fonts/dm-mono-latin-500.woff2' };
+
+const sansFile: ShippedFile = { package: apiPackage, path: 'src/landing/fonts/dm-sans-latin-variable.woff2' };
 
 function embedded(font: Readonly<Uint8Array>): string {
   return `url(data:font/woff2;base64,${Buffer.from(font).toString('base64')}) format('woff2')`;

packages/api/src/landing/landing-page.test.ts

@@ -5,11 +5,13 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk';
 import { shippedFilesInMemory } from '@on.auto/operations/testing';
 import { describe, expect, it } from 'vitest';
 
-import { readPageStyle, type ApiHandler } from '../index.ts';
+import { readPageStyle, type ApiHandler, apiShippedRoots } from '../index.ts';
 import { call, createTestHandler } from '../testing/api-calls.ts';
 import { operationServer } from '../testing/operation-server.ts';
 import { servedStudio } from '../testing/studios.ts';
 
+const apiFilesOnDisk = shippedFilesOnDisk(apiShippedRoots);
+
 const fromABrowser = { accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' };
 
 function open(handler: ApiHandler, address = 'http://localhost:5555/'): Promise<Response> {
@@ -162,10 +164,10 @@ describe('what the page at the root of the server loads', () => {
   it('has its typefaces read through the files the server ships once, before the server answers, and none read again for a page', async () => {
     const read: string[] = [];
     const shippedFiles: ShippedFiles = {
-      ...shippedFilesOnDisk,
+      ...apiFilesOnDisk,
       bytes: (file) => {
-        read.push(file.pathname.slice(file.pathname.lastIndexOf('/') + 1));
-        return shippedFilesOnDisk.bytes(file);
+        read.push(file.path.slice(file.path.lastIndexOf('/') + 1));
+        return apiFilesOnDisk.bytes(file);
       },
     };
     const { handler } = createTestHandler({ pageStyle: await readPageStyle(shippedFiles) });