diff --git a/capabilities/computation/measure/common.ts b/capabilities/computation/measure/common.ts index c3c24cbd3..872959ab3 100644 --- a/capabilities/computation/measure/common.ts +++ b/capabilities/computation/measure/common.ts @@ -1,10 +1,12 @@ -import { handedWorkersWithChecks } from '@on.auto/definitions/check'; +import { handedWorkersWithChecks, checkShippedRoots } from '@on.auto/definitions/check'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { programPool, type Json, type ProgramPool, type ProgramRequest } from '@on.auto/workflow-engine/dsl'; import { computationBounds } from '../src/run/run-bounds.ts'; -export const handed = await handedWorkersWithChecks(shippedFilesOnDisk); +const checkFiles = shippedFilesOnDisk(checkShippedRoots); + +export const handed = await handedWorkersWithChecks(checkFiles); export function millisecondsOf(work: () => void): number { const started = performance.now(); diff --git a/capabilities/coordination/src/testing/test-host.ts b/capabilities/coordination/src/testing/test-host.ts index 6698de1ab..a2f0577d8 100644 --- a/capabilities/coordination/src/testing/test-host.ts +++ b/capabilities/coordination/src/testing/test-host.ts @@ -1,5 +1,5 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { freshInstances, machineSandboxOf } from '@on.auto/workflow-engine/dsl'; +import { freshInstances, machineSandboxOf, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { openWorkflowHost, type WorkflowHost } from '@on.auto/workflow-host'; import { recordedReactions, recordedWaiting } from '@on.auto/workflow-host/testing'; import { Effect, Function } from 'effect'; @@ -9,10 +9,14 @@ import { workflowMachineOptionsOf } from '../runs/workflow-machine-options.ts'; import type { Playbook } from './playbook.ts'; import { testPlaybookNames } from './test-sandbox.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const sandboxModule = await sandboxModuleIn(sandboxFiles); + export function testHost(nested: Playbook): Promise { return openWorkflowHost({ database: { store: 'sqlite', file: ':memory:' }, - machine: workflowMachineOptionsOf(machineSandboxOf(freshInstances(shippedFilesOnDisk)), testPlaybookNames), + machine: workflowMachineOptionsOf(machineSandboxOf(freshInstances(sandboxModule)), testPlaybookNames), perform: definitionCalls(nested.executeNested), settle: nested.settle, reports: { diff --git a/capabilities/coordination/src/testing/test-sandbox.ts b/capabilities/coordination/src/testing/test-sandbox.ts index b4c03675b..b724190bd 100644 --- a/capabilities/coordination/src/testing/test-sandbox.ts +++ b/capabilities/coordination/src/testing/test-sandbox.ts @@ -7,14 +7,16 @@ import { type MachineSandbox, type MachineSettings, } from '@on.auto/workflow-engine'; -import { freshInstances, unitMemoryBytes } from '@on.auto/workflow-engine/dsl'; +import { freshInstances, sandboxModuleIn, unitMemoryBytes, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { workflowFacts } from '../capability/workflow.ts'; import { workflowPolicyOf, type WorkflowPolicy } from '../document/workflow-functions.ts'; import { workflowMachineOptionsOf, workflowMachineSettingsOf } from '../runs/workflow-machine-options.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + export const testSandbox: MachineSandbox = reusedSandbox( - await freshInstances(shippedFilesOnDisk)(unitMemoryBytes), + await freshInstances(await sandboxModuleIn(sandboxFiles))(unitMemoryBytes), () => 0, ); diff --git a/capabilities/prediction/measure.ts b/capabilities/prediction/measure.ts index 7e6e6d3cd..48eb35613 100644 --- a/capabilities/prediction/measure.ts +++ b/capabilities/prediction/measure.ts @@ -1,11 +1,13 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { handedWorkersOf, programPool } from '@on.auto/workflow-engine/dsl'; +import { handedWorkersOf, programPool, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { rebuiltOn, type Folding, type History } from './measure/dataset-rebuild.ts'; import { dailyOrders, seriesMeasured, testedDailyOrders } from './measure/order-history.ts'; import { measuredStores, type MeasuredStore } from './measure/stores.ts'; import { writtenOn, type FileRead } from './measure/written-files.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + const postgresqlServer = process.env['LEDGER_MEASURE_POSTGRESQL_URL'] ?? ''; const days = 200; @@ -71,7 +73,7 @@ async function testedOn({ store, aDatabase }: MeasuredStore, history: History): async function measuredOn(measured: MeasuredStore): Promise { const pool = programPool({ - handed: await handedWorkersOf(shippedFilesOnDisk), + handed: await handedWorkersOf(sandboxFiles), instances: 'fresh', workers: 1, heapMegabytes: 256, diff --git a/capabilities/prediction/src/dataset-testing/projected-datasets.ts b/capabilities/prediction/src/dataset-testing/projected-datasets.ts index ec6b3cbc7..9f4121437 100644 --- a/capabilities/prediction/src/dataset-testing/projected-datasets.ts +++ b/capabilities/prediction/src/dataset-testing/projected-datasets.ts @@ -1,5 +1,5 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { filterSandboxOf, freshInstances } from '@on.auto/workflow-engine/dsl'; +import { filterSandboxOf, freshInstances, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { someFolder, type DatabaseSettings } from '@on.auto/workflow-host'; import { projectedFolders, type ProjectedFolders } from '@on.auto/workflow-host/testing'; import { Effect, Option, Result, type Schema } from 'effect'; @@ -10,6 +10,10 @@ import { datasetFolder, datasetsOf, type DatasetFolding } from '../dataset/datas import type { DatasetRow } from '../dataset/dataset-rows.ts'; import { parsePredictionDocument } from '../document/document-parsing.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const sandboxModule = await sandboxModuleIn(sandboxFiles); + export const alphaKey = 'playbook/acme/alpha/'; const savedAt = '2026-10-06T09:00:00.000Z'; @@ -86,7 +90,7 @@ type Folding = Partial>; export function testDatasetFolder(folding: Folding = {}): ReturnType { return datasetFolder({ - filters: filterSandboxOf(freshInstances(shippedFilesOnDisk)), + filters: filterSandboxOf(freshInstances(sandboxModule)), triesBeforeStall: Number.POSITIVE_INFINITY, ...folding, }); diff --git a/capabilities/prediction/src/dataset/dataset-matching.test.ts b/capabilities/prediction/src/dataset/dataset-matching.test.ts index 7db9b72a4..297050333 100644 --- a/capabilities/prediction/src/dataset/dataset-matching.test.ts +++ b/capabilities/prediction/src/dataset/dataset-matching.test.ts @@ -1,5 +1,5 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { filterSandboxOf, freshInstances } from '@on.auto/workflow-engine/dsl'; +import { filterSandboxOf, freshInstances, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import type { PageEvent } from '@on.auto/workflow-host'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -7,6 +7,8 @@ import { describe, expect, it } from 'vitest'; import type { DatasetData } from './dataset-data.ts'; import { datasetMatchingOf } from './dataset-matching.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + const dataNamedFirst: DatasetData = { period: 'week', events: [ @@ -37,7 +39,7 @@ describe('the matching of a dataset whose filters name their data before their t it('tests an event against the data tests of the filters of its own type alone, matching each type first', async () => { const matching = datasetMatchingOf( dataNamedFirst, - filterSandboxOf(freshInstances(shippedFilesOnDisk), () => 0), + filterSandboxOf(freshInstances(await sandboxModuleIn(sandboxFiles)), () => 0), ); const match = await Effect.runPromise(matching.matchOf(placed)); diff --git a/capabilities/prediction/src/files/series-rows.ts b/capabilities/prediction/src/files/series-rows.ts index ef3128948..cdc2a11ea 100644 --- a/capabilities/prediction/src/files/series-rows.ts +++ b/capabilities/prediction/src/files/series-rows.ts @@ -111,5 +111,5 @@ export function seriesRowsOfStream( stream: ReadableStream, reading: SeriesReading, ): Promise { - return seriesRowsFrom(() => Readable.fromWeb(stream), reading); + return seriesRowsFrom(() => Readable.from(stream, { objectMode: false }), reading); } diff --git a/capabilities/prediction/src/trainer/candidate-staging.ts b/capabilities/prediction/src/trainer/candidate-staging.ts index 8346c9e0c..d5fc1e4a6 100644 --- a/capabilities/prediction/src/trainer/candidate-staging.ts +++ b/capabilities/prediction/src/trainer/candidate-staging.ts @@ -1,5 +1,4 @@ -import { createReadStream } from 'node:fs'; -import { Readable } from 'node:stream'; +import { open, type FileHandle } from 'node:fs/promises'; import type { Conflict } from '@on.auto/operations'; import { Effect } from 'effect'; @@ -8,6 +7,31 @@ import type { BuildState } from '../builds/build-state.ts'; import type { PredictionPorts } from '../capability/prediction-ports.ts'; import { buildEnded } from './waiting-builds.ts'; +const chunkBytes = 1_048_576; + +function chunksOf(path: string): ReadableStream { + const reading: { file?: Promise } = {}; + return new ReadableStream( + { + pull: async (controller) => { + reading.file ??= open(path); + const file = await reading.file; + const { bytesRead, buffer } = await file.read({ buffer: new Uint8Array(chunkBytes) }); + if (bytesRead > 0) { + controller.enqueue(buffer.subarray(0, bytesRead)); + return; + } + await file.close(); + controller.close(); + }, + cancel: async () => { + await (await reading.file)?.close(); + }, + }, + { highWaterMark: 0 }, + ); +} + export function staged( ports: Pick, { address }: Pick, @@ -15,7 +39,7 @@ export function staged( sha256: string, ): Effect.Effect { const { org, playbook, id } = address; - const stream = Readable.toWeb(createReadStream(path)); + const stream = chunksOf(path); return ports.ledger.content .stage({ org, playbook }, id, { stream, mediaType: 'application/octet-stream', expected: { sha256 } }) .pipe( diff --git a/capabilities/prediction/src/upload/utf8-text.ts b/capabilities/prediction/src/upload/utf8-text.ts index b0a249d46..fcb34b7a1 100644 --- a/capabilities/prediction/src/upload/utf8-text.ts +++ b/capabilities/prediction/src/upload/utf8-text.ts @@ -67,7 +67,7 @@ function validPrefixOf(bytes: Readonly): string { reached = scannedAt(bytes, at, sequence); at += reached === 'whole' ? sequence.length : 0; } - return new TextDecoder('utf-8', { ignoreBOM: true }).decode(bytes.subarray(0, at)); + return new TextDecoder('utf-8', { fatal: false, ignoreBOM: true }).decode(bytes.subarray(0, at)); } function isContinuation(byte: number | undefined): boolean { diff --git a/capabilities/recall/measure/folds.ts b/capabilities/recall/measure/folds.ts index 4384829eb..abf23df7f 100644 --- a/capabilities/recall/measure/folds.ts +++ b/capabilities/recall/measure/folds.ts @@ -1,4 +1,4 @@ -import { handedWorkersWithChecks } from '@on.auto/definitions/check'; +import { handedWorkersWithChecks, checkShippedRoots } from '@on.auto/definitions/check'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { programPool, type FoldRequest, type Json, type JsonObject } from '@on.auto/workflow-engine/dsl'; import { Effect, Result } from 'effect'; @@ -8,7 +8,9 @@ import { parseRecallDocument } from '../src/document/document-parsing.ts'; import { recallBounds, recallFolding } from '../src/run/recall-bounds.ts'; import { campaignReviews } from '../src/testing/campaign-reviews.ts'; -const handed = await handedWorkersWithChecks(shippedFilesOnDisk); +const checkFiles = shippedFilesOnDisk(checkShippedRoots); + +const handed = await handedWorkersWithChecks(checkFiles); export interface Folded { readonly events: number; diff --git a/packages/api/README.md b/packages/api/README.md index 4fec4c499..7a249e5d9 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -13,7 +13,7 @@ Each request passes through the same chain before it reaches an operation, in th **Signed-in persons.** With sign-in on, the authenticator carries the protected resource, `PUBLIC_URL` and the Auto identity service's issuer, and the API serves `GET /.well-known/oauth-protected-resource` and the same path followed by `/mcp`, `/orgs/{org}/mcp` and `/orgs/{org}/playbooks/{playbook}/mcp` before the chain checks a credential, as public paths matching that prefix, answering [RFC 9728](https://datatracker.ietf.org/doc/html/rfc9728) metadata: `resource` is `PUBLIC_URL` followed by the path after the prefix, `authorization_servers` the identity service's issuer, `bearer_methods_supported` `["header"]`, `scopes_supported` the scopes a sign-in asks for, `signInScopes` of `@on.auto/identity`: `user:org:read`, which asks the identity service for a token bound to the organization the person signs in to, then the four permissions, which the identity service grants as scopes of its own, and `resource_documentation` the public MCP reference; a path under the prefix that names no endpoint, or an org or playbook id that is not one, is `404`. A `401` then carries `Bearer resource_metadata="/.well-known/oauth-protected-resource", scope=""`, the five separated by spaces, after `error="invalid_token"` when a credential was refused, and its detail says a key or a token. With sign-in off none of this is served, the metadata's path needs a key, and the challenges are `Bearer` and `Bearer error="invalid_token"`. The challenges are built in `src/authentication/challenges.ts` and the metadata in `src/authentication/protected-resource.ts`. -Before that chain, a browser opening `/` (a `GET` that accepts `text/html`) gets a page saying the server is running, with a button that opens Studio on this server: the Studio a person can reach, `studioBaseOf` of `@on.auto/identity`, followed by `?server=`, which is `/studio/?server=…` where the runtime serves Studio and `https://studio.on.auto/?server=…` for the default Studio host otherwise. The page needs no key and loads nothing from any server: its styles, its icon and its typefaces, DM Mono and DM Sans, are inside it. The two font files and their SIL Open Font Licenses are in `src/landing/fonts`; `readPageStyle(shippedFiles)` reads them through the `ShippedFiles` of `@on.auto/operations` the server composes, once, and answers the page's styles, which its content security policy hashes, as a `PageStyle`, which the server reads at its start and gives the API as `ApiOptions.pageStyle`, so a typeface that cannot be read stops the start (`src/landing/page-style.ts`, `src/landing/fonts.ts`). Any other request to `/` has no route. +Before that chain, a browser opening `/` (a `GET` that accepts `text/html`) gets a page saying the server is running, with a button that opens Studio on this server: the Studio a person can reach, `studioBaseOf` of `@on.auto/identity`, followed by `?server=`, which is `/studio/?server=…` where the runtime serves Studio and `https://studio.on.auto/?server=…` for the default Studio host otherwise. The page needs no key and loads nothing from any server: its styles, its icon and its typefaces, DM Mono and DM Sans, are inside it. The two font files and their SIL Open Font Licenses are in `src/landing/fonts`; `readPageStyle(shippedFiles)` reads them, as files of `@on.auto/api` at the root `apiShippedRoots` declares, through the `ShippedFiles` of `@on.auto/operations` the server composes, once, and answers the page's styles, which its content security policy hashes, as a `PageStyle`, which the server reads at its start and gives the API as `ApiOptions.pageStyle`, so a typeface that cannot be read stops the start (`src/landing/page-style.ts`, `src/landing/fonts.ts`). Any other request to `/` has no route. **The served Studio.** A page on a public origin may not reach an address on the person's machine, so a runtime may serve Studio itself, at `/studio/` on its own origin. `ApiOptions.studio` says whether it does: a `HostedStudio`, `served: false`, serves none, and `/studio/` has no route; a `ServedStudio`, `served: true`, carries `host`, a `StudioHost`, which answers a path asked of the Studio host with a `StudioAnswer`, its status, content type, body, the seconds a browser may keep it, the host's validators, `ETag` and `Last-Modified`, and, for a redirect, the path on the host it sends the browser to, or with nothing when the host cannot be reached and nothing was kept. `GET` and `HEAD` under `/studio/` answer what the host answers for the same path under its root, without the query, with its status, its content type, its body, its `ETag` and `Last-Modified`, a redirect's `Location` under `/studio/`, and `Cache-Control` `public, max-age=, immutable` for what the host keeps fresh or `no-cache` for the rest; every other method is `405` `method_not_allowed` with `Allow: GET, HEAD`, `/studio` is sent on to `/studio/` with its query by `308`, since the page resolves its files against the folder it was served from, and a path that holds `..`, written or with a dot percent-encoded, is `404` `not_found` before the host is asked (`src/studio/served-studio.ts`). When the host answers nothing, the runtime answers `503` with a page of its own, `src/studio/unreachable-studio.ts`, which runs no script and says that Studio cannot be fetched now and from where. A request at `/studio` or under `/studio/` needs no key, since a browser opening a page sends none, and passes the `Origin` check and, in local mode, the `Host` check as every other request does (`src/middleware/public-paths.ts`). Its answers carry a content security policy of their own, `servedStudioPolicyOf` of `src/studio/studio-policy.ts`, written once from what Studio's build loads: `default-src 'none'; script-src 'self'; worker-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; img-src 'self' data:; connect-src 'self'; base-uri 'self'; form-action 'self'; frame-ancestors 'none'`, for its module scripts and the worker of its editor, its style sheets and the style elements it writes as it runs, its fonts and its images, the `data:` images its style sheets hold, and its calls to the runtime, every one from the runtime's origin. The page the host answers holds an inline script, so a `text/html` answer's `script-src` also allows each inline script of that page by its SHA-256, read from the page as the host answered it; every other answer under `/studio/` carries the policy with no hash, set by `src/middleware/response-headers.ts`. Every other path keeps `default-src 'none'; frame-ancestors 'none'`, and the other security headers are the same under `/studio/` as elsewhere. diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index e1c9db364..57ba2cc0c 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -1,6 +1,7 @@ export { createApiHandler } from './api-handler.ts'; export type { ApiHandler } from './api-handler.ts'; export type { ApiOptions } from './api-options.ts'; +export { apiShippedRoots } from './landing/fonts.ts'; export { readPageStyle, type PageStyle } from './landing/page-style.ts'; export type { ApiStudio, HostedStudio, ServedStudio, StudioAnswer, StudioHost } from './studio/served-studio.ts'; export { diff --git a/packages/api/src/landing/fonts.ts b/packages/api/src/landing/fonts.ts index d8f846645..fefe8356a 100644 --- a/packages/api/src/landing/fonts.ts +++ b/packages/api/src/landing/fonts.ts @@ -1,8 +1,12 @@ -import type { ShippedFiles } from '@on.auto/operations'; +import type { ShippedFile, ShippedFiles, ShippedRoots } from '@on.auto/operations'; -const monoFile = new URL('./fonts/dm-mono-latin-500.woff2', import.meta.url); +const apiPackage = '@on.auto/api'; -const sansFile = new URL('./fonts/dm-sans-latin-variable.woff2', import.meta.url); +export const apiShippedRoots: ShippedRoots = { [apiPackage]: new URL('../../', import.meta.url) }; + +const monoFile: ShippedFile = { package: apiPackage, path: 'src/landing/fonts/dm-mono-latin-500.woff2' }; + +const sansFile: ShippedFile = { package: apiPackage, path: 'src/landing/fonts/dm-sans-latin-variable.woff2' }; function embedded(font: Readonly): string { return `url(data:font/woff2;base64,${Buffer.from(font).toString('base64')}) format('woff2')`; diff --git a/packages/api/src/landing/landing-page.test.ts b/packages/api/src/landing/landing-page.test.ts index 75df0aab5..5f197ef8c 100644 --- a/packages/api/src/landing/landing-page.test.ts +++ b/packages/api/src/landing/landing-page.test.ts @@ -5,11 +5,13 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { shippedFilesInMemory } from '@on.auto/operations/testing'; import { describe, expect, it } from 'vitest'; -import { readPageStyle, type ApiHandler } from '../index.ts'; +import { readPageStyle, type ApiHandler, apiShippedRoots } from '../index.ts'; import { call, createTestHandler } from '../testing/api-calls.ts'; import { operationServer } from '../testing/operation-server.ts'; import { servedStudio } from '../testing/studios.ts'; +const apiFilesOnDisk = shippedFilesOnDisk(apiShippedRoots); + const fromABrowser = { accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' }; function open(handler: ApiHandler, address = 'http://localhost:5555/'): Promise { @@ -162,10 +164,10 @@ describe('what the page at the root of the server loads', () => { it('has its typefaces read through the files the server ships once, before the server answers, and none read again for a page', async () => { const read: string[] = []; const shippedFiles: ShippedFiles = { - ...shippedFilesOnDisk, + ...apiFilesOnDisk, bytes: (file) => { - read.push(file.pathname.slice(file.pathname.lastIndexOf('/') + 1)); - return shippedFilesOnDisk.bytes(file); + read.push(file.path.slice(file.path.lastIndexOf('/') + 1)); + return apiFilesOnDisk.bytes(file); }, }; const { handler } = createTestHandler({ pageStyle: await readPageStyle(shippedFiles) }); diff --git a/packages/api/src/testing/api-calls.ts b/packages/api/src/testing/api-calls.ts index 91f479acc..4ea395bcb 100644 --- a/packages/api/src/testing/api-calls.ts +++ b/packages/api/src/testing/api-calls.ts @@ -3,9 +3,12 @@ import { allPermissions, type CallSummary } from '@on.auto/operations'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { createApiHandler, type ApiHandler, type ApiOptions, type RegisterRoutes } from '../index.ts'; +import { apiShippedRoots } from '../landing/fonts.ts'; import { readPageStyle } from '../landing/page-style.ts'; import { hostedStudio } from './studios.ts'; +const apiFilesOnDisk = shippedFilesOnDisk(apiShippedRoots); + export interface Call { readonly method?: string; readonly headers?: Readonly>; @@ -37,7 +40,7 @@ const admitEveryone: Authenticator = { }), }; -const pageStyle = await readPageStyle(shippedFilesOnDisk); +const pageStyle = await readPageStyle(apiFilesOnDisk); export function createTestHandler(options: Partial = {}): { handler: ApiHandler; diff --git a/packages/connections/src/providers/shipped-consents.test.ts b/packages/connections/src/providers/shipped-consents.test.ts index 5fe822e16..166a59d88 100644 --- a/packages/connections/src/providers/shipped-consents.test.ts +++ b/packages/connections/src/providers/shipped-consents.test.ts @@ -1,12 +1,14 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { readProviders, type Placeholder, type ProviderEntry } from '@on.auto/providers'; +import { readProviders, type Placeholder, type ProviderEntry, providersShippedRoots } from '@on.auto/providers'; import { Option, Result, Schema } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; import { vendorsOfTheShippedWays, type VendorOfTheWay } from '../testing/shipped-vendors.ts'; import { consentedAtTheVendor, harnessAtTheVendor, servedAsTheVendor } from '../testing/vendor-fakes.ts'; -const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); +const providersOnDisk = shippedFilesOnDisk(providersShippedRoots); + +const shipped = Result.getOrThrow(await readProviders(providersOnDisk)); const closing: (() => Promise)[] = []; diff --git a/packages/connections/src/providers/shipped-providers.test.ts b/packages/connections/src/providers/shipped-providers.test.ts index 71b4d27e9..03d39f13e 100644 --- a/packages/connections/src/providers/shipped-providers.test.ts +++ b/packages/connections/src/providers/shipped-providers.test.ts @@ -6,6 +6,7 @@ import { type Placeholder, type Provider, type ProviderAddition, + providersShippedRoots, } from '@on.auto/providers'; import { describedToolServers, @@ -20,11 +21,14 @@ import { describe, expect, it } from 'vitest'; import { additionIssuesOf, readAddition, type AdditionInput } from '../entries/entry-checks.ts'; import { credentialFieldIssuesOf, placesNoValue } from '../registry/credential.ts'; -const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); +const providersOnDisk = shippedFilesOnDisk(providersShippedRoots); -const filesBesideTheEntries = await shippedFilesOnDisk.list( - new URL('entries/documents/', import.meta.resolve('@on.auto/providers')), -); +const shipped = Result.getOrThrow(await readProviders(providersOnDisk)); + +const filesBesideTheEntries = await providersOnDisk.list({ + package: '@on.auto/providers', + path: 'src/entries/documents', +}); const checking = { org: 'acme', diff --git a/packages/definitions/README.md b/packages/definitions/README.md index 2055434b8..d4abedd95 100644 --- a/packages/definitions/README.md +++ b/packages/definitions/README.md @@ -178,9 +178,9 @@ A run rejected with `conflict` records the kind the capability gave, so `get_run ## The check at save -`checkedAtSave(pool, job, deadlineMs)` of `@on.auto/definitions/check`, a subpath of its own because it names the engine's pool, which nothing else of the main entry needs, checks the programs and the expressions of a document as a job of the engine's pool, `check`, on the permit the pool keeps for checks beside the workers of runs, so a save never waits behind the runs (`src/program-check/save-check.ts`). The computation, recall, prediction and workflow capabilities call it from their `check`, with the document's module, its place, `computation` or `recall`, the schemas of its types, and every expression it holds with the names each may read; a workflow's expressions go as one job, whatever their number, and so do a prediction function's, the `data` expressions of its filters, which read `$data`, with no module. It answers the compiler's issues, each with the expression it belongs to, or `module`, and its line there, which the capability places in the document; or, when there are none, the document's `module` and the expressions stripping changed, by their source, which the capability answers from its `check` and the save keeps (`src/program-check/type-stripping.ts`), so no run strips anything and no runtime holds a compiler; or `InvalidInput` when the job does not answer within `deadlineMs`, 2,000 ms of the server's wall-clock time, counted from when its worker is ready, since the compiler took longer than a save allows over this document: its types should be simpler or its program split, and, since a busy server may also have slowed the check, the words say a later save may succeed; or `Unavailable` when the job stops otherwise: `busy` when no checker came free within that time or was ready within the start the pool allows it, the worker's memory, a cancelled save or the server stopping. The assignment bound stays the fast refusal of the one shape measured to grow the compiler's work with its square; others, such as hundreds of narrowings, meet the deadline. The compiler loads in the check worker and nowhere else: no entry of this package reaches `typescript6` or `ts-blank-space`, the `check` entry among them, whose bound of the walk, `mostAssignmentsInAFunction`, is kept apart from the walk (`src/program-check/compiler-entries.test.ts`), so the thread that serves requests never loads the compiler. The pool starts the worker of the checks with the first check and keeps it however long it idles, so a server that checks nothing never loads the compiler, and the first check pays its load as the first job of any worker pays the worker's start, within the start the pool allows a checker rather than within the deadline, which counts from when the worker is ready: 240.7 ms for the example's first job, the worker's start included (see [Measured](#measured)), which no later check pays again. A worker that crashes or cannot read the job is a defect. +`checkedAtSave(pool, job, deadlineMs)` of `@on.auto/definitions/check`, a subpath of its own because it names the engine's pool, which nothing else of the main entry needs, checks the programs and the expressions of a document as a job of the engine's pool, `check`, on the permit the pool keeps for checks beside the workers of runs, so a save never waits behind the runs (`src/program-check/save-check.ts`). The computation, recall, prediction and workflow capabilities call it from their `check`, with the document's module, its place, `computation` or `recall`, the schemas of its types, and every expression it holds with the names each may read; a workflow's expressions go as one job, whatever their number, and so do a prediction function's, the `data` expressions of its filters, which read `$data`, with no module. It answers the compiler's issues, each with the expression it belongs to, or `module`, and its line there, which the capability places in the document; or, when there are none, the document's `module` and the expressions stripping changed, by their source, which the capability answers from its `check` and the save keeps (`src/program-check/type-stripping.ts`), so no run strips anything and no runtime holds a compiler; or `InvalidInput` when the job does not answer within `deadlineMs`, 2,000 ms of the server's wall-clock time, counted from when its worker is ready, since the compiler took longer than a save allows over this document: its types should be simpler or its program split, and, since a busy server may also have slowed the check, the words say a later save may succeed; or `Unavailable` when the job stops otherwise: `busy` when no checker came free within that time or was ready within the start the pool allows it, the worker's memory, a cancelled save or the server stopping. The assignment bound stays the fast refusal of the one shape measured to grow the compiler's work with its square; others, such as hundreds of narrowings, meet the deadline. The compiler loads in the check worker and nowhere else: no entry of this package but the check handler's reaches `typescript6` or `ts-blank-space`, the `check` entry among them, whose bound of the walk, `mostAssignmentsInAFunction`, is kept apart from the walk (`src/program-check/compiler-entries.test.ts`), so the thread that serves requests never loads the compiler. The pool starts the worker of the checks with the first check and keeps it however long it idles, so a server that checks nothing never loads the compiler, and the first check pays its load as the first job of any worker pays the worker's start, within the start the pool allows a checker rather than within the deadline, which counts from when the worker is ready: 240.7 ms for the example's first job, the worker's start included (see [Measured](#measured)), which no later check pays again. A worker that crashes or cannot read the job is a defect. -The job runs in its own worker module, `src/program-check/check-worker.ts`, which loads the compiler once and keeps it, with the sandbox's lib, between checks, so a warm check pays only for the document. A thread cannot be handed a reader, so the files of the kept lib are read once, before the pool is made, through the `ShippedFiles` the server composes: `handedWorkersWithChecks(shippedFiles)` of the `check` entry answers the files the pool hands its workers, the module of the sandbox, and for the check worker the kept lib in its place, which the server gives the one pool it makes (`src/program-check/save-check.ts`). The worker makes its checker of what it was handed with `serveJobsWith` of the engine's job loop, through `sandboxLibFrom(handed)` of `src/program-check/kept-lib.ts` (`handedChecks`, `src/program-check/handed-checks.ts`), and a test makes its checker of the same lib read on disk (`keptLibOnDisk`, `src/testing/read-on-disk.ts`). The compiler is TypeScript 6, `typescript` 6.0.3 installed under the alias `typescript6`, since TypeScript 7 has no compiler API to call and the repository's own toolchain keeps the name `typescript`; it is a dependency of this package alone. Its options are `strict`, `exactOptionalPropertyTypes`, `isolatedModules`, `erasableSyntaxOnly`, `noEmit` and `noLib`, with the sandbox's lib as the only lib and `types: []` (`src/program-check/checking.ts`). One program holds, each as a file of its own, so a line of the document's program is a line of its file and no offset is subtracted: +The job runs in its own worker module, `src/program-check/check-worker.ts`, which loads the compiler once and keeps it, with the sandbox's lib, between checks, so a warm check pays only for the document. A thread cannot be handed a reader, so the files of the kept lib are read once, before the pool is made, through the `ShippedFiles` the server composes over `checkShippedRoots` of the `check` entry, the root of this package, where the kept lib is `lib/sandbox.d.ts.txt` and `lib/sandbox-iterator.d.ts.txt`, with the sandbox's: `handedWorkersWithChecks(shippedFiles)` of the `check` entry answers the files the pool hands its workers, the module of the sandbox, and for the check worker the kept lib in its place, which the server gives the one pool it makes (`src/program-check/save-check.ts`). The worker makes its checker of what it was handed with `serveJobsWith` of the engine's job loop, through `sandboxLibFrom(handed)` of `src/program-check/kept-lib.ts` (`handedChecks`, `src/program-check/handed-checks.ts`), which is an entry of its own, `@on.auto/definitions/check-handler`, for a host that serves the checks in a worker of its own, apart from the main and `check` entries since it loads the compiler, and a test makes its checker of the same lib read on disk (`keptLibOnDisk`, `src/testing/read-on-disk.ts`). The compiler is TypeScript 6, `typescript` 6.0.3 installed under the alias `typescript6`, since TypeScript 7 has no compiler API to call and the repository's own toolchain keeps the name `typescript`; it is a dependency of this package alone. Its options are `strict`, `exactOptionalPropertyTypes`, `isolatedModules`, `erasableSyntaxOnly`, `noEmit` and `noLib`, with the sandbox's lib as the only lib and `types: []` (`src/program-check/checking.ts`). One program holds, each as a file of its own, so a line of the document's program is a line of its file and no offset is subtracted: - the sandbox's lib and its iterator lib; - the declarations of the document, `Json`, the types its schemas give, `Input`, `Output`, each definition of a schema's `$defs` or `definitions` as a type of its own named after the schema and the definition, `Input_address`, a character a name cannot hold written as its code point between two `$`, `Input_street$45$address`, so that a `$ref` may recur, and, for a recall function, `View`, each `Json` without its schema, and, for a recall function, `Event`, the playbook's event with its attributes and no extension, so that a misspelt attribute is refused rather than read as an extension (`src/program-check/schema-types.ts`); @@ -192,7 +192,7 @@ Beside the compiler's diagnostics, a walk of the module (`src/program-check/modu ### The sandbox's lib -The lib is the sandbox's own, so a name the sandbox lacks is a name the check lacks. `sandboxLibOf(shippedFiles, probe)` (`src/program-check/sandbox-lib.ts`) makes it from the compiler's ES2025 libs, read through the `ShippedFiles` it is given, without `intl` and `weakref`, less the globals, `Math` members and `Date` methods of the engine's `sandboxRemovals`, the list the prelude removes them by, less the call signature of `DateConstructor` and its constructors of several numbers, which read the server's time zone, less the values of `Intl`, and less every global and member the sandbox answers it does not have, each asked of a fresh instance through the engine's `sandboxAnswers`, the members of an interface asked of a sample of it, which `src/program-check/instance-samples.ts` lists, a strict function among them, so the members that throw in a module, `arguments` and `caller`, go too. The comments go, and the licence notice of the compiler's libs is kept once at the top. The iterator lib stays a module of its own, as the compiler ships it, since its abstract class `Iterator` would collide with the global interface `Iterator` in a script (`lib.es2025.iterator.d.ts`, lines 19 to 22). +The lib is the sandbox's own, so a name the sandbox lacks is a name the check lacks. `sandboxLibOf(shippedFiles, probe)` (`src/program-check/sandbox-lib.ts`) makes it from the compiler's ES2025 libs, read as `lib` of `typescript6` through the `ShippedFiles` it is given, at the root of that package, `compilerShippedRoots`, which this package resolves with `import.meta.resolve` since it depends on it, without `intl` and `weakref`, less the globals, `Math` members and `Date` methods of the engine's `sandboxRemovals`, the list the prelude removes them by, less the call signature of `DateConstructor` and its constructors of several numbers, which read the server's time zone, less the values of `Intl`, and less every global and member the sandbox answers it does not have, each asked of a fresh instance through the engine's `sandboxAnswers`, the members of an interface asked of a sample of it, which `src/program-check/instance-samples.ts` lists, a strict function among them, so the members that throw in a module, `arguments` and `caller`, go too. The comments go, and the licence notice of the compiler's libs is kept once at the top. The iterator lib stays a module of its own, as the compiler ships it, since its abstract class `Iterator` would collide with the global interface `Iterator` in a script (`lib.es2025.iterator.d.ts`, lines 19 to 22). The repository has no build step, so the lib is generated by a command, `pnpm --filter @on.auto/definitions sandbox-lib` (`write-sandbox-lib.ts`), and kept under `lib/`, beside `src`, as `sandbox.d.ts.txt` and `sandbox-iterator.d.ts.txt`: text files, which the repository's lint and format do not read as its own declarations. `src/program-check/sandbox-lib.test.ts` generates it again and fails when the kept lib differs, so a new build of the engine, a change to the prelude's list or a new compiler changes the kept lib in the same change. The server's image copies `lib/`. @@ -439,4 +439,4 @@ A run that finishes later records facts after the call that started it returned, ## Source -`src/shapes` holds the output shapes and their checks. `src/used-by` holds the shape of what a definition calls and of who uses it, and the reader of who uses a definition, and `src/schema-words` the words of a schema that a refused value is told in. `src/index.ts` is the main entry point, `src/document.ts` the entry point of the reader of function documents, held in `src/document`, the checked worker and the words of a refused value in `src/checking`, `src/template.ts` the entry point of the template engine of the capabilities, held in `src/template`, and `src/testing/index.ts` the entry point of the test support. `src/capability` holds the declaration of a capability and the list of known capabilities. `src/registry` holds the definitions of a capability in a playbook: a definition, the events and commands of its stream, and the decider and its rules. `src/runs` holds a run: its events, commands, state, decider and rules, its size limits, and `runSettler`. `src/operations` holds the eight operations that change and read definitions and runs, `start_definition_version` among them, and how they load and record them. `src/reading` holds `list_runs` and `get_run_history`, `src/analytics` `get_playbook_analytics` and `runOutcomeMapping`, and `src/presenting` the presenters of the four stream kinds. `src/events` holds the events of a playbook: the shape of a CloudEvent, `publish_event` and the stream of a published event, and the playbook's own facts as events. `src/tool-calls` holds the journal of a run's tool calls, and `src/run-work` the work a run records after its start: the decisions on its calls, the recorder of its outbound calls, the words and accounts of its deliveries and deferral, and a run as it was recorded. `src/testing` holds what the tests share, with the capabilities of the tests in `src/testing/capabilities` and their clocks and patient checks in `src/testing/time`. `operations` depends on `capability`, `plain-language`, `reading`, `registry`, `run-work`, `runs`, `shapes`, `tool-calls` and `used-by`, and not on `events`; `reading` on `analytics`, `cancellation`, `capability`, `plain-language`, `presenting`, `run-work`, `runs` and the fields and `get_run` of `operations`; `analytics` on `capability` and the fields of `operations`; `events` on `capability`, `registry`, `run-work`, `runs` and the command metadata of `operations`; `presenting` on `capability`, `events`, `plain-language`, `registry`, `run-work` and `runs`; `used-by` on `events`, `registry` and the words of `capability`; `checking` on `schema-words`, and `schema-words` on `plain-language`; `registry` on the words and the stripped forms of `capability` and the shape of a user, `user-relations.ts` of `used-by`; and `runs` on `registry`, `shapes` and the work decisions of `run-work`. +`src/shapes` holds the output shapes and their checks. `src/used-by` holds the shape of what a definition calls and of who uses it, and the reader of who uses a definition, and `src/schema-words` the words of a schema that a refused value is told in. `src/index.ts` is the main entry point, `src/check.ts` the entry point of the check at save and `src/program-check/handed-checks.ts` that of the handler its worker serves, `src/document.ts` the entry point of the reader of function documents, held in `src/document`, the checked worker and the words of a refused value in `src/checking`, `src/template.ts` the entry point of the template engine of the capabilities, held in `src/template`, and `src/testing/index.ts` the entry point of the test support. `src/capability` holds the declaration of a capability and the list of known capabilities. `src/registry` holds the definitions of a capability in a playbook: a definition, the events and commands of its stream, and the decider and its rules. `src/runs` holds a run: its events, commands, state, decider and rules, its size limits, and `runSettler`. `src/operations` holds the eight operations that change and read definitions and runs, `start_definition_version` among them, and how they load and record them. `src/reading` holds `list_runs` and `get_run_history`, `src/analytics` `get_playbook_analytics` and `runOutcomeMapping`, and `src/presenting` the presenters of the four stream kinds. `src/events` holds the events of a playbook: the shape of a CloudEvent, `publish_event` and the stream of a published event, and the playbook's own facts as events. `src/tool-calls` holds the journal of a run's tool calls, and `src/run-work` the work a run records after its start: the decisions on its calls, the recorder of its outbound calls, the words and accounts of its deliveries and deferral, and a run as it was recorded. `src/testing` holds what the tests share, with the capabilities of the tests in `src/testing/capabilities` and their clocks and patient checks in `src/testing/time`. `operations` depends on `capability`, `plain-language`, `reading`, `registry`, `run-work`, `runs`, `shapes`, `tool-calls` and `used-by`, and not on `events`; `reading` on `analytics`, `cancellation`, `capability`, `plain-language`, `presenting`, `run-work`, `runs` and the fields and `get_run` of `operations`; `analytics` on `capability` and the fields of `operations`; `events` on `capability`, `registry`, `run-work`, `runs` and the command metadata of `operations`; `presenting` on `capability`, `events`, `plain-language`, `registry`, `run-work` and `runs`; `used-by` on `events`, `registry` and the words of `capability`; `checking` on `schema-words`, and `schema-words` on `plain-language`; `registry` on the words and the stripped forms of `capability` and the shape of a user, `user-relations.ts` of `used-by`; and `runs` on `registry`, `shapes` and the work decisions of `run-work`. diff --git a/packages/definitions/measure/measure-checks.ts b/packages/definitions/measure/measure-checks.ts index 5d6241fcc..0c6f31f48 100644 --- a/packages/definitions/measure/measure-checks.ts +++ b/packages/definitions/measure/measure-checks.ts @@ -1,14 +1,22 @@ import { cpus } from 'node:os'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { freshInstances, programPool, sandboxAnswers, type CheckJob } from '@on.auto/workflow-engine/dsl'; +import { + freshInstances, + programPool, + sandboxAnswers, + sandboxModuleIn, + type CheckJob, +} from '@on.auto/workflow-engine/dsl'; import { checkerOf } from '../src/program-check/checking.ts'; import { sandboxLibFrom } from '../src/program-check/kept-lib.ts'; -import { sandboxLibOf } from '../src/program-check/sandbox-lib.ts'; -import { checkWorker, handedWorkersWithChecks } from '../src/program-check/save-check.ts'; +import { sandboxLibOf, compilerShippedRoots } from '../src/program-check/sandbox-lib.ts'; +import { checkWorker, handedWorkersWithChecks, checkShippedRoots } from '../src/program-check/save-check.ts'; import { keptLibOnDisk } from '../src/testing/read-on-disk.ts'; +const shippedFiles = shippedFilesOnDisk({ ...compilerShippedRoots, ...checkShippedRoots }); + const schemas = { input: { type: 'object', @@ -111,10 +119,10 @@ function write(line: string): void { const [cpu] = cpus(); write(`Node ${process.version} on ${cpu?.model ?? 'an unknown processor'}, ${cpus().length} cores`); +const sandboxModule = await sandboxModuleIn(shippedFiles); + const generating = performance.now(); -const generated = await sandboxLibOf(shippedFilesOnDisk, (probed) => - sandboxAnswers(freshInstances(shippedFilesOnDisk), probed), -); +const generated = await sandboxLibOf(shippedFiles, (probed) => sandboxAnswers(freshInstances(sandboxModule), probed)); write( `the lib generated in ${(performance.now() - generating).toFixed(0)} ms: ${generated.script.length} and ${generated.iterator.length} characters`, ); @@ -142,7 +150,7 @@ for (const [name, job] of cases) { ); } -const handed = await handedWorkersWithChecks(shippedFilesOnDisk); +const handed = await handedWorkersWithChecks(shippedFiles); const pool = programPool({ workers: 1, heapMegabytes: 256, handed, instances: 'fresh' }); const checkedBy = { worker: checkWorker }; const cold = await pool.check({ ...computation(pace), deadlineMs: 60_000, ...checkedBy }); diff --git a/packages/definitions/package.json b/packages/definitions/package.json index d79343cab..ff9fad2f5 100644 --- a/packages/definitions/package.json +++ b/packages/definitions/package.json @@ -7,6 +7,7 @@ "exports": { ".": "./src/index.ts", "./check": "./src/check.ts", + "./check-handler": "./src/program-check/handed-checks.ts", "./document": "./src/document.ts", "./template": "./src/template.ts", "./testing": "./src/testing/index.ts" diff --git a/packages/definitions/src/check.ts b/packages/definitions/src/check.ts index 1f7356e87..1cc904fea 100644 --- a/packages/definitions/src/check.ts +++ b/packages/definitions/src/check.ts @@ -1,2 +1,7 @@ export { mostAssignmentsInAFunction } from './program-check/assignment-bound.ts'; -export { checkedAtSave, handedWorkersWithChecks, type CheckedDocument } from './program-check/save-check.ts'; +export { + checkShippedRoots, + checkedAtSave, + handedWorkersWithChecks, + type CheckedDocument, +} from './program-check/save-check.ts'; diff --git a/packages/definitions/src/program-check/compiler-entries.test.ts b/packages/definitions/src/program-check/compiler-entries.test.ts index f53de4484..a9902cc48 100644 --- a/packages/definitions/src/program-check/compiler-entries.test.ts +++ b/packages/definitions/src/program-check/compiler-entries.test.ts @@ -19,10 +19,15 @@ function compilersReachedFrom(entry: string): readonly string[] { return modulesReachedFrom(sources, entry).packages.filter((name) => compilers.has(name)); } +const checkHandler = 'program-check/handed-checks.ts'; + describe('the compiler', () => { - it('is reached by no entry of the package, the check entry among them, and by the check worker, which the pool starts with the first check', () => { - expect(entries).toEqual(['index.ts', 'check.ts', 'document.ts', 'template.ts', 'testing/index.ts']); - expect(entries.flatMap((entry) => compilersReachedFrom(entry))).toEqual([]); + it('is reached by the check worker, which the pool starts with the first check, and by the entry of the handler it serves, and by no other entry of the package, the check entry among them', () => { + expect(entries).toEqual(['index.ts', 'check.ts', checkHandler, 'document.ts', 'template.ts', 'testing/index.ts']); + expect(entries.filter((entry) => entry !== checkHandler).flatMap((entry) => compilersReachedFrom(entry))).toEqual( + [], + ); + expect(compilersReachedFrom(checkHandler)).toEqual(['ts-blank-space', 'typescript6']); expect(compilersReachedFrom('program-check/check-worker.ts')).toEqual(['ts-blank-space', 'typescript6']); }); }); diff --git a/packages/definitions/src/program-check/kept-lib.ts b/packages/definitions/src/program-check/kept-lib.ts index e9219d243..bea30958f 100644 --- a/packages/definitions/src/program-check/kept-lib.ts +++ b/packages/definitions/src/program-check/kept-lib.ts @@ -1,10 +1,15 @@ +import type { ShippedFile, ShippedRoots } from '@on.auto/operations'; import type { HandedFile } from '@on.auto/workflow-engine/job-loop'; import type { SandboxLib } from './sandbox-lib.ts'; -export const keptLibFiles: Readonly> = { - script: new URL('../../lib/sandbox.d.ts.txt', import.meta.url), - iterator: new URL('../../lib/sandbox-iterator.d.ts.txt', import.meta.url), +const definitionsPackage = '@on.auto/definitions'; + +export const definitionsShippedRoots: ShippedRoots = { [definitionsPackage]: new URL('../../', import.meta.url) }; + +export const keptLibFiles: Readonly> = { + script: { package: definitionsPackage, path: 'lib/sandbox.d.ts.txt' }, + iterator: { package: definitionsPackage, path: 'lib/sandbox-iterator.d.ts.txt' }, }; export function sandboxLibFrom(handed: HandedFile): SandboxLib { diff --git a/packages/definitions/src/program-check/sandbox-lib.test.ts b/packages/definitions/src/program-check/sandbox-lib.test.ts index 6c69e26a9..60144341a 100644 --- a/packages/definitions/src/program-check/sandbox-lib.test.ts +++ b/packages/definitions/src/program-check/sandbox-lib.test.ts @@ -1,17 +1,22 @@ +import { shippedFileName } from '@on.auto/operations'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { freshInstances, sandboxAnswers } from '@on.auto/workflow-engine/dsl'; +import { freshInstances, sandboxAnswers, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { handedFileIn } from '@on.auto/workflow-engine/job-loop'; import type { CheckJob } from '@on.auto/workflow-engine/worker'; import { describe, expect, it } from 'vitest'; import { keptLibOnDisk } from '../testing/read-on-disk.ts'; import { handedChecks } from './handed-checks.ts'; -import { keptLibFiles, sandboxLibFrom } from './kept-lib.ts'; -import { sandboxLibOf } from './sandbox-lib.ts'; +import { definitionsShippedRoots, keptLibFiles, sandboxLibFrom } from './kept-lib.ts'; +import { compilerShippedRoots, sandboxLibOf } from './sandbox-lib.ts'; const kept = sandboxLibFrom(keptLibOnDisk); -const probe = (expressions: readonly string[]) => sandboxAnswers(freshInstances(shippedFilesOnDisk), expressions); +const compilerFilesOnDisk = shippedFilesOnDisk(compilerShippedRoots); + +const sandboxModule = await sandboxModuleIn(shippedFilesOnDisk(sandboxShippedRoots)); + +const probe = (expressions: readonly string[]) => sandboxAnswers(freshInstances(sandboxModule), expressions); function computationReturning(output: string): CheckJob { return { @@ -30,9 +35,13 @@ function declares(text: string): (fragment: string) => boolean { describe('the lib of the sandbox', () => { it('is what the generator makes from the compiler’s libs and the sandbox’s own answers, kept beside the package', async () => { - expect(await sandboxLibOf(shippedFilesOnDisk, probe)).toEqual(kept); - expect(keptLibFiles.script.pathname).toMatch(/\/definitions\/lib\/sandbox\.d\.ts\.txt$/u); - expect(keptLibFiles.iterator.pathname).toMatch(/\/definitions\/lib\/sandbox-iterator\.d\.ts\.txt$/u); + expect(await sandboxLibOf(compilerFilesOnDisk, probe)).toEqual(kept); + expect( + Object.values(keptLibFiles).map((file) => new URL(file.path, definitionsShippedRoots[file.package]).href), + ).toEqual([ + new URL('../../lib/sandbox.d.ts.txt', import.meta.url).href, + new URL('../../lib/sandbox-iterator.d.ts.txt', import.meta.url).href, + ]); }); it('declares nothing the prelude removes or the engine lacks, and what the engine has', () => { @@ -88,7 +97,7 @@ describe('the checks of the worker the pool hands the kept lib', () => { it('cannot be made when the pool handed the worker no kept lib', () => { expect(() => handedChecks(handedFileIn({}))).toThrow( - `The pool handed this worker no file at ${keptLibFiles.script.href}`, + `The pool handed this worker no file at ${shippedFileName(keptLibFiles.script)}`, ); }); }); diff --git a/packages/definitions/src/program-check/sandbox-lib.ts b/packages/definitions/src/program-check/sandbox-lib.ts index 6dd8393af..f2459ef16 100644 --- a/packages/definitions/src/program-check/sandbox-lib.ts +++ b/packages/definitions/src/program-check/sandbox-lib.ts @@ -1,4 +1,4 @@ -import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFileIn, type ShippedFile, type ShippedFiles, type ShippedRoots } from '@on.auto/operations'; import { sandboxRemovals } from '@on.auto/workflow-engine/worker'; import ts from 'typescript6'; @@ -37,7 +37,13 @@ interface LibFile { readonly candidates: Candidates; } -const libFolder = new URL(import.meta.resolve('typescript6').replace(/typescript\.js$/u, '')); +const compilerPackage = 'typescript6'; + +export const compilerShippedRoots: ShippedRoots = { + [compilerPackage]: new URL('.', import.meta.resolve(`${compilerPackage}/package.json`)), +}; + +const compilerLib: ShippedFile = { package: compilerPackage, path: 'lib' }; const skipped = /intl|weakref/u; @@ -49,7 +55,7 @@ const iteratorLib = 'lib.es2025.iterator.d.ts'; async function libSource(shippedFiles: ShippedFiles, name: string): Promise { const fileName = `lib.${name}.d.ts`; - const text = await shippedFiles.text(new URL(fileName, libFolder)); + const text = await shippedFiles.text(shippedFileIn(compilerLib, fileName)); return ts.createSourceFile(fileName, text, ts.ScriptTarget.ES2025, true); } diff --git a/packages/definitions/src/program-check/save-check.ts b/packages/definitions/src/program-check/save-check.ts index a46723dda..ad5c69910 100644 --- a/packages/definitions/src/program-check/save-check.ts +++ b/packages/definitions/src/program-check/save-check.ts @@ -1,8 +1,9 @@ -import { InvalidInput, Unavailable, type ShippedFiles } from '@on.auto/operations'; +import { InvalidInput, Unavailable, type ShippedFiles, type ShippedRoots } from '@on.auto/operations'; import { handedFilesOf, handedWorkersOf, pooledJob, + sandboxShippedRoots, type CheckIssue, type CheckJob, type CheckOutcome, @@ -15,10 +16,12 @@ import { import { Effect } from 'effect'; import { changedExpressions, type StrippedForms } from '../capability/stripped-forms.ts'; -import { keptLibFiles } from './kept-lib.ts'; +import { definitionsShippedRoots, keptLibFiles } from './kept-lib.ts'; export const checkWorker = new URL('./check-worker.ts', import.meta.url); +export const checkShippedRoots: ShippedRoots = { ...sandboxShippedRoots, ...definitionsShippedRoots }; + export function keptLibOf(shippedFiles: ShippedFiles): Promise { return handedFilesOf(shippedFiles, Object.values(keptLibFiles)); } diff --git a/packages/definitions/src/testing/read-on-disk.ts b/packages/definitions/src/testing/read-on-disk.ts index 23fa37a9c..8488c33c5 100644 --- a/packages/definitions/src/testing/read-on-disk.ts +++ b/packages/definitions/src/testing/read-on-disk.ts @@ -3,12 +3,14 @@ import { measuredEnvironment } from '@on.auto/operations/testing'; import type { PoolSettings } from '@on.auto/workflow-engine/dsl'; import { handedFileIn, type HandedFile } from '@on.auto/workflow-engine/job-loop'; -import { handedWorkersWithChecks, keptLibOf } from '../program-check/save-check.ts'; +import { checkShippedRoots, handedWorkersWithChecks, keptLibOf } from '../program-check/save-check.ts'; + +const checkFilesOnDisk = shippedFilesOnDisk(checkShippedRoots); export const measuredPoolSettings: Pick = { - handed: await handedWorkersWithChecks(shippedFilesOnDisk), + handed: await handedWorkersWithChecks(checkFilesOnDisk), instances: 'fresh', environment: measuredEnvironment, }; -export const keptLibOnDisk: HandedFile = handedFileIn(await keptLibOf(shippedFilesOnDisk)); +export const keptLibOnDisk: HandedFile = handedFileIn(await keptLibOf(checkFilesOnDisk)); diff --git a/packages/definitions/write-sandbox-lib.ts b/packages/definitions/write-sandbox-lib.ts index ca0bb8f4b..51e66ccc1 100644 --- a/packages/definitions/write-sandbox-lib.ts +++ b/packages/definitions/write-sandbox-lib.ts @@ -1,17 +1,21 @@ import { writeFileSync } from 'node:fs'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { freshInstances, sandboxAnswers } from '@on.auto/workflow-engine/dsl'; +import { freshInstances, sandboxAnswers, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { keptLibFiles } from './src/program-check/kept-lib.ts'; -import { sandboxLibOf } from './src/program-check/sandbox-lib.ts'; +import { compilerShippedRoots, sandboxLibOf } from './src/program-check/sandbox-lib.ts'; -const lib = await sandboxLibOf(shippedFilesOnDisk, (expressions) => - sandboxAnswers(freshInstances(shippedFilesOnDisk), expressions), +const sandboxModule = await sandboxModuleIn(shippedFilesOnDisk(sandboxShippedRoots)); + +const lib = await sandboxLibOf(shippedFilesOnDisk(compilerShippedRoots), (expressions) => + sandboxAnswers(freshInstances(sandboxModule), expressions), ); -writeFileSync(keptLibFiles.script, lib.script); -writeFileSync(keptLibFiles.iterator, lib.iterator); +const script = new URL(keptLibFiles.script.path, import.meta.url); +const iterator = new URL(keptLibFiles.iterator.path, import.meta.url); +writeFileSync(script, lib.script); +writeFileSync(iterator, lib.iterator); process.stdout.write( - `Wrote the sandbox's lib, ${lib.script.length} and ${lib.iterator.length} characters, to ${keptLibFiles.script.pathname} and ${keptLibFiles.iterator.pathname}\n`, + `Wrote the sandbox's lib, ${lib.script.length} and ${lib.iterator.length} characters, to ${script.pathname} and ${iterator.pathname}\n`, ); diff --git a/packages/identity/src/identity-service/issuer-requests.ts b/packages/identity/src/identity-service/issuer-requests.ts index 81e3966ba..ff175c54f 100644 --- a/packages/identity/src/identity-service/issuer-requests.ts +++ b/packages/identity/src/identity-service/issuer-requests.ts @@ -14,7 +14,7 @@ export const issuerAnswerMs = 10_000; export class UnreadAnswer extends Data.TaggedError('UnreadAnswer')<{ readonly message: string }> {} -const utf8 = new TextDecoder('utf-8', { fatal: true }); +const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: false }); async function bytesUpTo(body: ReadableStream | null, mostBytes: number): Promise { if (body === null) { diff --git a/packages/ledger/README.md b/packages/ledger/README.md index 5dd7a21cc..390016ab0 100644 --- a/packages/ledger/README.md +++ b/packages/ledger/README.md @@ -298,7 +298,7 @@ Measured on 2026-10-12 on an Apple M4 Max with 16 cores, with Node 26.10.0, SQLi ## Kept secrets -The ledger keeps the rows of the secret store of [`@on.auto/secrets`](../secrets/README.md) apart from its facts and from its content: what the runtime must hold to act for someone, such as a key a person pasted or a token minted for them, which no fact of a stream may carry, since every reader of a stream reads its facts. It keeps the rows alone and seals nothing: a value reaches it as its keeper holds it, sealed under a data key of its own that the store's custodian wraps, as that package's README sets out under [Sealing](../secrets/README.md#sealing), or not at all, where a secret manager keeps it and the row holds the name and the version of the value there, as its README sets out under [The keeper](../secrets/README.md#the-keeper), so the ledger knows no key and neither its database nor a backup of it holds a value anyone can read without the custodian's or the manager's. The rows are not part of the `Ledger` port: `sqliteSecretRows({ fileName })` of `@on.auto/ledger/sqlite3` and `postgresqlSecretRows({ pools })` of `@on.auto/ledger/postgresql` each answer an `Effect` of the `SecretRows` port of `@on.auto/secrets`, opened on connections of their own from the settings the ledger is opened from, as the workflow host opens its own, once the ledger is open, as the server opens them. On SQLite the rows migrate the ledger's schema on their own pool first, which finds the layout the ledger wrote and everything it made when it opened, and creates nothing, since a pool of its own on `:memory:` is a database of its own; on PostgreSQL they open a pool of two connections, from `pools` or any `(bounds) => SecretConnections`, on the database the ledger migrated, and log a connection they lose as the ledger logs one of its own. One table holds them, `kept_secrets`, created with `CREATE TABLE IF NOT EXISTS` by the same schema step that creates the content tables, on SQLite and on PostgreSQL, through the same dialect, and looked up first in the catalog as they are, so a start that finds it creates nothing: +The ledger keeps the rows of the secret store of [`@on.auto/secrets`](../secrets/README.md) apart from its facts and from its content: what the runtime must hold to act for someone, such as a key a person pasted or a token minted for them, which no fact of a stream may carry, since every reader of a stream reads its facts. It keeps the rows alone and seals nothing: a value reaches it as its keeper holds it, sealed under a data key of its own that the store's custodian wraps, as that package's README sets out under [Sealing](../secrets/README.md#sealing), or not at all, where a secret manager keeps it and the row holds the name and the version of the value there, as its README sets out under [The keeper](../secrets/README.md#the-keeper), so the ledger knows no key and neither its database nor a backup of it holds a value anyone can read without the custodian's or the manager's. The rows are not part of the `Ledger` port: `sqliteSecretRows({ fileName })` of `@on.auto/ledger/sqlite3` and `postgresqlSecretRows({ pools })` of `@on.auto/ledger/postgresql` each answer an `Effect` of the `SecretRows` port of `@on.auto/secrets`, opened on connections of their own from the settings the ledger is opened from, as the workflow host opens its own, once the ledger is open, as the server opens them. `sqliteSecretRowsOn(optionsOf)` of the main entry answers the same rows on any of Emmett's SQLite drivers, as `sqliteEventStore` opens the store on one, for code that opens the ledger on a driver of its own; `sqliteSecretRows` is it on `sqlite3`. On SQLite the rows migrate the ledger's schema on their own pool first, which finds the layout the ledger wrote and everything it made when it opened, and creates nothing, since a pool of its own on `:memory:` is a database of its own; on PostgreSQL they open a pool of two connections, from `pools` or any `(bounds) => SecretConnections`, on the database the ledger migrated, and log a connection they lose as the ledger logs one of its own. One table holds them, `kept_secrets`, created with `CREATE TABLE IF NOT EXISTS` by the same schema step that creates the content tables, on SQLite and on PostgreSQL, through the same dialect, and looked up first in the catalog as they are, so a start that finds it creates nothing: | Column | What it holds | | --------------- | ---------------------------------------------------------------------------------------- | @@ -496,7 +496,7 @@ Opened without the run outcome mapping and the projections, the ledger neither c ## Testing -The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. Its part on the layout of the ledger, `src/layout/ledger-layout-behaviour.ts`, runs on both stores: a database that holds no table given the layout and opened again in it, a database in another layout refused with the layout found and the layout kept, in the words of the refusal, and left as it was, and a database that holds a table but no layout number refused the same way and left as it was; `src/layout/ledger-layout.test.ts` pins the layout on SQLite (see [The layout of the ledger](#the-layout-of-the-ledger)), `src/layout/sqlite-schema.test.ts` checks without a database that SQLite lists the tables and writes the layout in one transaction, and `src/layout/sqlite-file-in-layout.test.ts` that `aSQLiteFileInLayout` of `@on.auto/ledger/testing`, with which the server's tests make a ledger of another version, puts a new file in the layout it is given. Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, and the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, a read of a chunk its store answers as a `Uint8Array` or as an `ArrayBuffer`, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing), and the rows opened on connections of their own on the database the ledger made, which on SQLite migrate it again, and which leave it in the layout the ledger wrote. `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, and that the migration takes its lock and lists the tables of the schema, then writes the layout into a database that holds none, or reads it from one that holds `ledger_layout`, before anything else. The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. +The ledger's behaviour is one suite, in `src/testing/ledger-behaviour.ts`, that `src/ledger.test.ts` runs on SQLite and `src/postgresql/ledger-on-postgresql.test.ts` runs on PostgreSQL. `@on.auto/ledger/testing` exports it, `ledgerBehaviour(entry)`, with `LedgerEntry`, what a store hands the suite, so a ledger opened on a driver of its own runs the same suite. The parts of the SQLite entry that hang on SQLite's dialect rather than on its driver are exported from the main entry, which loads no driver: `sqliteLayout`, the `LayoutDialect` that lists the tables of the schema, `sqlitePlanOf(execute, read)`, the plans SQLite makes of the statements a read of a `RecordedStore` asks, and `sqliteDefinitionStreamsIndexed(execute)`, whether the read of the definition streams goes through its index, each over a `SQLExecutor` of dumbo that the caller opens on its driver (`src/recorded/sqlite-plans.ts`), beside the types the entry names, `LedgerStore`, `RecordedStore` and `StatementExecutor`. This package's own entry opens them on `sqlite3` (`src/recorded/sqlite-checks.ts`), and `src/ledger.test.ts` lists the indexes and the tables it expects SQLite to name. Its part on the layout of the ledger, `src/layout/ledger-layout-behaviour.ts`, runs on both stores: a database that holds no table given the layout and opened again in it, a database in another layout refused with the layout found and the layout kept, in the words of the refusal, and left as it was, and a database that holds a table but no layout number refused the same way and left as it was; `src/layout/ledger-layout.test.ts` pins the layout on SQLite (see [The layout of the ledger](#the-layout-of-the-ledger)), `src/layout/sqlite-schema.test.ts` checks without a database that SQLite lists the tables and writes the layout in one transaction, and `src/layout/sqlite-file-in-layout.test.ts` that `aSQLiteFileInLayout` of `@on.auto/ledger/testing`, with which the server's tests make a ledger of another version, puts a new file in the layout it is given. Its part on the outcomes of runs, `src/outcomes/run-outcomes-behaviour.ts`, also runs on the in-memory ledger, in `src/outcomes/run-outcomes-in-memory.test.ts`, with the `runTallies` mapping of `@on.auto/operations/testing`; `src/outcomes/run-outcome-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a ledger opened without the projection, a table filled at open, an interrupted fill done again, and a table found and not filled again. That ledgers that start together fill the table once is tested on PostgreSQL alone, in `src/outcomes/run-outcomes-on-postgresql.test.ts`. That a server waits past the migrator's 10 s for a migration lock another server holds is tested with a fake lock and a fake clock in `src/postgresql/postgresql-run-outcomes.test.ts`, and on PostgreSQL in `src/postgresql/connections-on-postgresql.test.ts`, where a client holds the lock for 12 s. The part on the keyed projections, `src/projections/projections-behaviour.ts`, the rows kept, the reads by columns, in order and from where a page ended, the counts, the rows due across playbooks and the next due time, the rows a mapping keys over two stream kinds and the reader's advance, which a fold leaves as it stands unless it sets the columns, runs on both stores and on the in-memory ledger, in `src/projections/projections-in-memory.test.ts`, with the `runTallyRows` and `topicRows` projections of `@on.auto/operations/testing`, and `src/projections/projection-table-behaviour.ts` holds what only a store does: an append whose projection throws, of which nothing is kept, a table made with its indexes and filled at open with the version before it dropped, a table found and not filled again, and a table keyed by its mapping filled in the order its facts were appended, over more than one batch. `src/postgresql/postgresql-projections.test.ts` checks the statements of the reads, the advance, the ordered fill and the fold's upsert on PostgreSQL without a database, and `src/postgresql/advances-on-postgresql.test.ts`, on PostgreSQL alone, a fold that read a row before an advance committed. The server tests the mapping of `@on.auto/definitions` on both stores, through the ledger it composes. Its part on reading what a playbook recorded, `src/testing/recorded-behaviour.ts`, `src/testing/runs-behaviour.ts`, `src/run-filters/run-filters-behaviour.ts`, `src/lineage/lineage-behaviour.ts` and `src/heads/heads-behaviour.ts`, the ids, causes and correlations a read gives, the runs of one capability or name, the read by correlation, the read from inside a record, and the versions and the read of heads, also runs on the in-memory ledger of `@on.auto/operations`, in `src/recorded/recorded-in-memory.test.ts`, so the three agree. A read behind an append whose transaction is still open, the same read by correlation, the same read newest first, a list of runs oldest first behind such an append, a write open in another database that hides nothing, and a start that finds the indexes while an append is open are tested on PostgreSQL alone, since SQLite serialises appends. Before each read of what a playbook recorded, the PostgreSQL entry of the suite waits until every committed message of its own database is older than every write open on the server, polling every 20 ms for at most 10 s and failing the test past that; the tests that wait have a timeout of 30 s, above vitest's 5 s. Waiting for the ledger's own horizon is not enough when other tests write on the same server: a write in another database that began before a message and ends while a read runs is in doubt to that read, which then stays behind it and may answer nothing, as a probe that ended such a write during a read confirmed. The test of a write open in another database waits for every write but that one. The SQLite entry and the in-memory ledger do not wait. `src/testing` holds 15 files, the most a folder holds, so the suite of lineage opened `src/lineage`, the suite of heads `src/heads`, the suite of the runs of one definition `src/run-filters`, the suite of the context of each message, the place of each record and the plans of the reads by definition and by id `src/context`, and the suite of recorded content `src/content`, which runs the behaviour suite of the port, `recordedContentBehaviour` of `@on.auto/operations/testing`, on both stores, reading through a second ledger on the same database and counting the chunks no head names, and adds what only chunks in a database show: content of no bytes, 100 bytes and 5 MB kept in chunks of 1 MiB under a head written last and read back a chunk at a time, a read whose chunk is missing failing after the chunks before it, no last chunk handed over of bytes that are not their digest's, a read of a staged content failing when its run is unstaged during it, and the chunks no head names swept on the ledger's schedule and before a close (`unnamed-chunks-behaviour.ts`, with `sweepsWatched` of `watched-sweeps.ts`, which lets a test wait for a sweep or hold it); its order of writes, the chunks first and the head last, the removal of a put's own chunks when another head stands, a read of a chunk its store answers as a `Uint8Array` or as an `ArrayBuffer`, and an unstage's record and then its one transaction, the heads first, `src/content/content-store.test.ts` checks without a database, and `src/content/postgresql-content.test.ts` the hex a chunk travels as and the transaction of an unstage on one PostgreSQL connection, rolled back when a delete fails. The suites of kept secrets, `rowsBehaviour` and `storeBehaviour` of `@on.auto/secrets/testing`, run beside the ledger's on both stores, in the same two files, through `src/secrets/kept-secrets-behaviour.ts`, each on a database whose ledger opened first: every claim on the rows, and every claim on the store over them with the ledger's keeper and a key in hand, each moment set with `TestClock`, as that package's README lists them under [Testing](../secrets/README.md#testing), and the rows opened on connections of their own on the database the ledger made, which on SQLite migrate it again, and which leave it in the layout the ledger wrote. `src/secrets/postgresql-secret-rows.test.ts` checks the rows on PostgreSQL without a database, through fake connections: their pool, a lost connection logged, and each statement whole with the values it binds, the insert, the read with the end of its open claim, the forget and the holding it answers, the replacement, the claim with its condition of no open claim, the read of a keeper's rows under a kid, and the re-holding with its condition on the version and the holding read; `src/postgresql/postgresql-schema.test.ts` checks that the schema step creates its table only when it is missing, and that the migration takes its lock and lists the tables of the schema, then writes the layout into a database that holds none, or reads it from one that holds `ledger_layout`, before anything else. The read of one event by its id, with its data, context and place, no page cursor, and nothing of another playbook, is part of the suite of lineage, so it runs on both stores and on the in-memory ledger. `src/postgresql-reads/postgresql-runs.test.ts` checks the statement of that read on PostgreSQL without a database. `src/signal/append-signal.test.ts` tests the signal of an append on SQLite. ```bash docker run --detach --name ledger-pg-test --publish 127.0.0.1:19632:5432 --env POSTGRES_PASSWORD=ledger-test postgres:18.6-alpine diff --git a/packages/ledger/src/index.ts b/packages/ledger/src/index.ts index 4a0f29db5..c18daea4b 100644 --- a/packages/ledger/src/index.ts +++ b/packages/ledger/src/index.ts @@ -14,12 +14,16 @@ export type { DefinitionStream, EncodedEvent, EventStore, + LedgerStore, RecordedPoint, + RecordedStore, RecordedStream, + StatementExecutor, StoredPage, StoredPlace, StoredRecord, } from './event-store.ts'; +export { sqliteLayout, type LayoutDialect } from './layout/layout-dialects.ts'; export { LedgerLayoutRefused } from './layout/ledger-layout-refused.ts'; export { cursorOf as recordedCursorOf } from './recorded/cursor.ts'; export { @@ -31,6 +35,8 @@ export { type StreamLoad, } from './ledger-service.ts'; export { sqliteEventStore, sqliteLedgerLayer, type SQLiteStoreOptions } from './sqlite-event-store.ts'; +export { sqliteSecretRowsOn } from './secrets/sqlite-secret-rows.ts'; export { recordedReaderOf } from './recorded/recorded-reader.ts'; +export { sqliteDefinitionStreamsIndexed, sqlitePlanOf } from './recorded/sqlite-plans.ts'; export { retriedOnVersionConflict, VersionConflict } from './version-conflict.ts'; export { streamAppends, streamSignalOf, type StreamAppended, type StreamSignal } from './signal/append-signal.ts'; diff --git a/packages/ledger/src/ledger.test.ts b/packages/ledger/src/ledger.test.ts index 66b21e59f..5c188f790 100644 --- a/packages/ledger/src/ledger.test.ts +++ b/packages/ledger/src/ledger.test.ts @@ -1,8 +1,7 @@ import { sqlite3EventStoreDriver } from '@event-driven-io/emmett-sqlite/sqlite3'; import { describe, onTestFinished } from 'vitest'; -import { sqliteEventStore } from './index.ts'; -import { sqliteLayout } from './layout/layout-dialects.ts'; +import { sqliteEventStore, sqliteLayout } from './index.ts'; import { definitionStreamsIndexed, executedOn, planOf, queried } from './recorded/sqlite-checks.ts'; import { keptSecretsBehaviour } from './secrets/kept-secrets-behaviour.ts'; import { ledgerLayer, sqliteSecretRows } from './sqlite3.ts'; diff --git a/packages/ledger/src/recorded/sqlite-checks.ts b/packages/ledger/src/recorded/sqlite-checks.ts index b58a2314d..fd6e70154 100644 --- a/packages/ledger/src/recorded/sqlite-checks.ts +++ b/packages/ledger/src/recorded/sqlite-checks.ts @@ -1,9 +1,8 @@ import { SQL, dumbo, type SQLExecutor } from '@event-driven-io/dumbo'; import { sqlite3EventStoreDriver } from '@event-driven-io/emmett-sqlite/sqlite3'; -import { definitionStreamsQuery } from '../definitions/sqlite-definition-streams.ts'; import type { RecordedStore } from '../event-store.ts'; -import { sqliteRecordedStore } from './sqlite-recorded.ts'; +import { sqliteDefinitionStreamsIndexed, sqlitePlanOf } from './sqlite-plans.ts'; export async function executedOn(fileName: string, work: (execute: SQLExecutor) => Promise): Promise { const pool = dumbo(sqlite3EventStoreDriver.mapToDumboOptions({ fileName })); @@ -19,25 +18,9 @@ export function queried(fileName: string, statement: string): Promise { - return executedOn(fileName, async (execute) => { - const { rows } = await execute.query(SQL`EXPLAIN QUERY PLAN ${definitionStreamsQuery('recall')}`); - return JSON.stringify(rows).includes('USING INDEX ledger_definition_streams'); - }); + return executedOn(fileName, sqliteDefinitionStreamsIndexed); } export function planOf(fileName: string, read: (store: RecordedStore) => Promise): Promise { - const asked: SQL[] = []; - return executedOn(fileName, async (execute) => { - await read( - sqliteRecordedStore({ - ...execute, - query: (sql) => { - asked.push(sql); - return execute.query(sql); - }, - }), - ); - const plans = await execute.batchQuery(asked.map((sql) => SQL`EXPLAIN QUERY PLAN ${sql}`)); - return JSON.stringify(plans); - }); + return executedOn(fileName, (execute) => sqlitePlanOf(execute, read)); } diff --git a/packages/ledger/src/recorded/sqlite-plans.ts b/packages/ledger/src/recorded/sqlite-plans.ts new file mode 100644 index 000000000..e2bc7715a --- /dev/null +++ b/packages/ledger/src/recorded/sqlite-plans.ts @@ -0,0 +1,28 @@ +import { SQL, type SQLExecutor } from '@event-driven-io/dumbo'; + +import { definitionStreamsQuery } from '../definitions/sqlite-definition-streams.ts'; +import type { RecordedStore } from '../event-store.ts'; +import { sqliteRecordedStore } from './sqlite-recorded.ts'; + +export async function sqliteDefinitionStreamsIndexed(execute: SQLExecutor): Promise { + const { rows } = await execute.query(SQL`EXPLAIN QUERY PLAN ${definitionStreamsQuery('recall')}`); + return JSON.stringify(rows).includes('USING INDEX ledger_definition_streams'); +} + +export async function sqlitePlanOf( + execute: SQLExecutor, + read: (store: RecordedStore) => Promise, +): Promise { + const asked: SQL[] = []; + await read( + sqliteRecordedStore({ + ...execute, + query: (sql) => { + asked.push(sql); + return execute.query(sql); + }, + }), + ); + const plans = await execute.batchQuery(asked.map((sql) => SQL`EXPLAIN QUERY PLAN ${sql}`)); + return JSON.stringify(plans); +} diff --git a/packages/ledger/src/testing/index.ts b/packages/ledger/src/testing/index.ts index 98ec7b2e8..072b432f8 100644 --- a/packages/ledger/src/testing/index.ts +++ b/packages/ledger/src/testing/index.ts @@ -1,3 +1,5 @@ export { ledgerLayout } from '../layout/ledger-layout.ts'; export { aSQLiteFileInLayout } from '../layout/sqlite-file-in-layout.ts'; +export { ledgerBehaviour } from './ledger-behaviour.ts'; +export type { LedgerEntry } from './ledger-entry.ts'; export { unreachableDatabase, type UnreachableDatabase } from './unreachable-database.ts'; diff --git a/packages/operations/README.md b/packages/operations/README.md index 5986dcca0..e591f8694 100644 --- a/packages/operations/README.md +++ b/packages/operations/README.md @@ -210,13 +210,13 @@ The digest is Node's, behind the one module of the repository for it: the digest ## The files a package ships -`ShippedFiles` (`src/shipped-files/shipped-files.ts`) is the port through which a package reads the files it ships beside its code: `text(file)`, `bytes(file)` and `list(folder)`, the names of the files directly in a folder, sorted, each of a `URL` the package computes from its own `import.meta.url`, as `new URL('../entries/', import.meta.url)`. A package computes where its files are and never how they are read: it takes the reader as a dependency of the function or factory that reads, never from an import of the reader, and the reader never knows what a file is for, so how they are read is one decision, taken once by the server that composes the packages. What a package read it hands on as content, never as a path for another package to read again: `@on.auto/providers` hands the tool servers the texts of the documents beside its entries, and a pool of `@on.auto/workflow-engine` is handed the bytes its workers start with. `shippedFilesOnDisk` (`src/shipped-files/shipped-files-on-disk.ts`) is the one reader, and the one module of the port that imports `node:fs`; it is an entry of its own, `@on.auto/operations/shipped-files-on-disk`, as the drivers of `@on.auto/ledger` are, so that the main entry, which every package imports, and the testing entry reach no file system for it (`src/shipped-files/shipped-files-entries.test.ts`). +`ShippedFiles` (`src/shipped-files/shipped-files.ts`) is the port through which a package reads the files it ships beside its code: `text(file)`, `bytes(file)` and `list(folder)`, the names of the files directly in a folder, sorted, each a `ShippedFile`, the name of the package the file is shipped in and its path within it, as `{ package: '@on.auto/providers', path: 'src/entries/slack.yaml' }`; `shippedFileIn(folder, name)` is a file of a folder, and `shippedFileName(file)` the name a file goes by in an error and among the files a pool hands its workers, `@on.auto/providers/src/entries/slack.yaml`. No module computes where a file is when it loads, so a host that bundles the modules names the files as the server does. A package that ships files declares its root once, a `ShippedRoots` of its name, `new URL('../../', import.meta.url)` of the module that names the files, and exports it from the entry that reads them: `providersShippedRoots`, `apiShippedRoots` and `checkShippedRoots` of `@on.auto/definitions/check`. A file of a dependency, the engine's WebAssembly or the compiler's lib, is read by the package that depends on it, under that dependency's name, at the root the dependent resolves with `import.meta.resolve` of the dependency's `package.json`, since pnpm's layout lets the dependent alone resolve it (`sandboxShippedRoots` of `@on.auto/workflow-engine/dsl`). A package names its files and never says how they are read: it takes the reader as a dependency of the function or factory that reads, never from an import of the reader, and the reader never knows what a file is for, so how they are read is one decision, taken once by the server that composes the packages. What a package read it hands on as content, never as a path for another package to read again: `@on.auto/providers` hands the tool servers the texts of the documents beside its entries, and a pool of `@on.auto/workflow-engine` is handed the bytes its workers start with. `shippedFilesOnDisk(roots)` (`src/shipped-files/shipped-files-on-disk.ts`) is the one reader: it reads a file under the root `roots` gives its package, resolves no package itself, and refuses the file of a package it is given no root for; the server gives it the roots of every package whose files it reads (`packages/server/src/server-options/shipped-roots.ts`). It is the one module of the port that imports `node:fs`; it is an entry of its own, `@on.auto/operations/shipped-files-on-disk`, as the drivers of `@on.auto/ledger` are, so that the main entry, which every package imports, and the testing entry reach no file system for it (`src/shipped-files/shipped-files-entries.test.ts`). ## Testing `@on.auto/operations/testing` exports an in-memory `Ledger`, `PlaybookRegistry` and `IncidentReporter` for the tests of packages that define or serve operations. The in-memory ledger names and links its messages as the ledger does. `memoryPlaybookRegistry(active, retired)` takes the active playbooks and, optionally, the retired ones; any other playbook is unknown. `memoryLedger(runOutcomes)` keeps the outcomes of runs with the mapping it is given, and none without one; an append whose mapping throws keeps nothing. `runTallies` is a small `RunOutcomeMapping` over `runFacts`, a decider that appends the `run_began` and `run_ended` facts it is given, for the tests of a ledger. `memoryLedger(runOutcomes, projections)` keeps the projections it is given too, and `runTallyRows`, `tallyRowsOf(version)` and `readTallyRows` are a small projection over the same facts and a query of it, and `topicRows` over `topicFacts` a projection keyed by its mapping over two stream kinds with columns its reader advances, for the tests of a ledger's projections. The in-memory ledger reads what a playbook recorded as the ledger does, with the context of each message, and the behaviour suite of `@on.auto/ledger` runs on it; it keeps recorded content whole in memory, `memoryRecordedContent()`, by the rules of the port the ledger keeps, its refusals, its first media type and its end of staging among them, which the tests of other packages take alone too, and `keptTextOf(content, playbook, sha256)` reads a kept text back as text. `internalTermsIn(text, measuredIn?)` (`src/plain-language/internal-terms.ts`), beside the plain words it guards, finds the terms a person's words must not hold, which the tests of every package that words a result run: the internal words, a name in capitals with underscores, an id, a path, and a number from 100 to 599 that reads as a status code, unless a unit follows it, one of the units the runtime measures in or one the text names in `measuredIn`, such as the unit of a prediction function, alone or after a range written `N to M`, as in `100 to 160 orders`; a number before `and` is not in a range, so the `404` of `404 and 3 rows` is still found. -`shippedFilesInMemory(files)` (`src/shipped-files/shipped-files-in-memory.ts`) is a `ShippedFiles` over a record of contents by the `href` of their files, each a text or bytes, for the unit tests of a package that reads what it ships: a folder lists the files directly under it, and a file it does not hold is refused, as is a folder that holds none, since it holds no empty folder; the same tests run on it and on the reader on disk (`src/shipped-files/shipped-files.test.ts`). +`shippedFilesInMemory(files)` (`src/shipped-files/shipped-files-in-memory.ts`) is a `ShippedFiles` over a record of contents by package and by path within it, each a text or bytes, for the unit tests of a package that reads what it ships: a folder lists the files directly under it, and a file it does not hold is refused, as is a folder that holds none, since it holds no empty folder; the same tests run on it and on the reader on disk (`src/shipped-files/shipped-files.test.ts`). `recordedContentBehaviour(entry)` is the behaviour suite of the content port, in `src/content-testing`: a content of no bytes to 64 MiB kept and streamed back, a text kept once with its first media type in its playbook alone, a content put again whole or with its digest, the refusals of a wrong digest, a wrong size and a stream that fails, wrong bytes put with the digest of a content held, a wrong put racing a right one, and staging, read through a second client, ended, refused and raced. An adapter runs it with a `ContentEntry` whose `aStore()` answers `{ content, elsewhere, unnamedLeft }`: the port, the port of a second client of the same store, and how many pieces of puts no kept content names the store holds, which is 0 after every test. This package runs it on `memoryRecordedContent`, the ledger on SQLite and PostgreSQL, and `@on.auto/object-storage` on its fake bucket and on a real S3-compatible server. The samples it is made of are exported with it: `generated(bytes, seed)` and `generatedStream`, bytes made the same for a seed, `answerOfAbout(bytes)`, a JSON answer of about that size, `readBack` and `readThrough`, which digest what a read streams, `deliveredBeforeFailing(stream)`, how many bytes a failing stream handed over, `brokenAfter(bytes)` and `unread()`, streams that fail or are never read, and `resultOf`, `refusalOf` and `present`, which run an effect for its value, its failure or its value present. diff --git a/packages/operations/src/content/streamed-content.ts b/packages/operations/src/content/streamed-content.ts index fbf6c8507..ec6b69fbb 100644 --- a/packages/operations/src/content/streamed-content.ts +++ b/packages/operations/src/content/streamed-content.ts @@ -35,7 +35,7 @@ export function textToKeep(text: string, mediaType: string): ContentToKeep { } async function decoded(stream: ReadableStream): Promise { - const utf8 = new TextDecoder('utf-8', { fatal: true }); + const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: false }); let text = ''; for await (const part of stream) { text += utf8.decode(part, { stream: true }); diff --git a/packages/operations/src/index.ts b/packages/operations/src/index.ts index c0ef0b3c4..233ca3f36 100644 --- a/packages/operations/src/index.ts +++ b/packages/operations/src/index.ts @@ -284,4 +284,10 @@ export { uuidV5 } from './uuid/uuid-v5.ts'; export { randomUUIDv7, uuidV7Before, uuidV7TimeOf } from './uuid/uuid-v7.ts'; export { loopbackNames, namesTheLoopback } from './addresses/loopback-names.ts'; export { cacheFolderMode, cacheFolderProblem } from './folders/cache-folder.ts'; -export type { ShippedFiles } from './shipped-files/shipped-files.ts'; +export { + shippedFileIn, + shippedFileName, + type ShippedFile, + type ShippedFiles, + type ShippedRoots, +} from './shipped-files/shipped-files.ts'; diff --git a/packages/operations/src/shipped-files/shipped-files-in-memory.ts b/packages/operations/src/shipped-files/shipped-files-in-memory.ts index 79e95ccb9..0b94448d8 100644 --- a/packages/operations/src/shipped-files/shipped-files-in-memory.ts +++ b/packages/operations/src/shipped-files/shipped-files-in-memory.ts @@ -1,13 +1,17 @@ -import type { ShippedFiles } from './shipped-files.ts'; +import { shippedFileName, type ShippedFile, type ShippedFiles } from './shipped-files.ts'; type Content = string | Readonly; -type Held = ReadonlyMap; +type Held = Readonly>>>; -function contentOf(held: Held, file: Readonly): Promise { - const content = held.get(file.href); +function heldIn(held: Held, file: ShippedFile): Readonly> { + return held[file.package] ?? {}; +} + +function contentOf(held: Held, file: ShippedFile): Promise { + const content = heldIn(held, file)[file.path]; return content === undefined - ? Promise.reject(new Error(`No file is shipped at ${file.href}`)) + ? Promise.reject(new Error(`No file is shipped at ${shippedFileName(file)}`)) : Promise.resolve(content); } @@ -19,21 +23,21 @@ function bytesOf(content: Content): Uint8Array { return typeof content === 'string' ? new TextEncoder().encode(content) : new Uint8Array(content); } -function namesIn(held: Held, folder: Readonly): Promise { - const under = [...held.keys()].filter((href) => href.startsWith(folder.href)); +function namesIn(held: Held, folder: ShippedFile): Promise { + const prefix = `${folder.path}/`; + const under = Object.keys(heldIn(held, folder)).filter((path) => path.startsWith(prefix)); if (under.length === 0) { - return Promise.reject(new Error(`No folder is shipped at ${folder.href}`)); + return Promise.reject(new Error(`No folder is shipped at ${shippedFileName(folder)}`)); } return Promise.resolve( under - .map((href) => href.slice(folder.href.length)) + .map((path) => path.slice(prefix.length)) .filter((name) => !name.includes('/')) .toSorted(), ); } -export function shippedFilesInMemory(files: Readonly>): ShippedFiles { - const held: Held = new Map(Object.entries(files)); +export function shippedFilesInMemory(held: Held): ShippedFiles { return { text: (file) => contentOf(held, file).then(textOf), bytes: (file) => contentOf(held, file).then(bytesOf), diff --git a/packages/operations/src/shipped-files/shipped-files-on-disk.ts b/packages/operations/src/shipped-files/shipped-files-on-disk.ts index e6e2e5aaa..3f0240cb2 100644 --- a/packages/operations/src/shipped-files/shipped-files-on-disk.ts +++ b/packages/operations/src/shipped-files/shipped-files-on-disk.ts @@ -1,16 +1,29 @@ import type { Dirent } from 'node:fs'; import { readdir, readFile } from 'node:fs/promises'; -import type { ShippedFiles } from './shipped-files.ts'; +import { shippedFileName, type ShippedFile, type ShippedFiles, type ShippedRoots } from './shipped-files.ts'; -export const shippedFilesOnDisk: ShippedFiles = { - text: (file) => readFile(file, 'utf8'), - bytes: (file) => readFile(file), - list: async (folder) => { - const entries = await readdir(folder, { withFileTypes: true }); - return entries - .filter((entry: Readonly) => entry.isFile()) - .map(({ name }: Readonly) => name) - .toSorted(); - }, -}; +function placeOf(roots: ShippedRoots, file: ShippedFile): Promise { + const root = roots[file.package]; + return root === undefined + ? Promise.reject( + new Error(`No root is given for the files of ${file.package}, so ${shippedFileName(file)} is not read`), + ) + : Promise.resolve(new URL(file.path, root)); +} + +async function namesIn(folder: Readonly): Promise { + const entries = await readdir(folder, { withFileTypes: true }); + return entries + .filter((entry: Readonly) => entry.isFile()) + .map(({ name }: Readonly) => name) + .toSorted(); +} + +export function shippedFilesOnDisk(roots: ShippedRoots): ShippedFiles { + return { + text: async (file) => readFile(await placeOf(roots, file), 'utf8'), + bytes: async (file) => readFile(await placeOf(roots, file)), + list: async (folder) => namesIn(await placeOf(roots, folder)), + }; +} diff --git a/packages/operations/src/shipped-files/shipped-files.test.ts b/packages/operations/src/shipped-files/shipped-files.test.ts index 8dc783ce0..246e92bf5 100644 --- a/packages/operations/src/shipped-files/shipped-files.test.ts +++ b/packages/operations/src/shipped-files/shipped-files.test.ts @@ -7,7 +7,7 @@ import { afterAll, describe, expect, it } from 'vitest'; import { shippedFilesInMemory } from './shipped-files-in-memory.ts'; import { shippedFilesOnDisk } from './shipped-files-on-disk.ts'; -import type { ShippedFiles } from './shipped-files.ts'; +import { shippedFileIn, shippedFileName, type ShippedFile, type ShippedFiles } from './shipped-files.ts'; const notes = 'Notes, in UTF-8: café\n'; @@ -15,47 +15,60 @@ const mark = Uint8Array.of(0, 255, 1); const inner = 'Inner notes\n'; +const shipper = '@acme/shipper'; + const onDisk = mkdtempSync(join(tmpdir(), 'auto-runtime-shipped-files-')); -mkdirSync(join(onDisk, 'nested')); -writeFileSync(join(onDisk, 'notes.txt'), notes); -writeFileSync(join(onDisk, 'mark.bin'), mark); -writeFileSync(join(onDisk, 'nested', 'inner.txt'), inner); +mkdirSync(join(onDisk, 'files', 'nested'), { recursive: true }); +writeFileSync(join(onDisk, 'files', 'notes.txt'), notes); +writeFileSync(join(onDisk, 'files', 'mark.bin'), mark); +writeFileSync(join(onDisk, 'files', 'nested', 'inner.txt'), inner); afterAll(() => { rmSync(onDisk, { recursive: true, force: true }); }); -const inMemory = new URL('file:///shipped/'); +const folder: ShippedFile = { package: shipper, path: 'files' }; -const readers: readonly (readonly [string, ShippedFiles, Readonly])[] = [ - ['on disk', shippedFilesOnDisk, pathToFileURL(`${onDisk}/`)], +const readers: readonly (readonly [string, ShippedFiles])[] = [ + ['on disk', shippedFilesOnDisk({ [shipper]: pathToFileURL(`${onDisk}/`) })], [ 'in memory', shippedFilesInMemory({ - [new URL('notes.txt', inMemory).href]: notes, - [new URL('mark.bin', inMemory).href]: mark, - [new URL('nested/inner.txt', inMemory).href]: inner, + [shipper]: { 'files/notes.txt': notes, 'files/mark.bin': mark, 'files/nested/inner.txt': inner }, }), - inMemory, ], ]; -describe.each(readers)('the files shipped %s', (_, reader, folder) => { +describe('a file shipped beside the code of a package', () => { + it('is named by its package and its path within it, and a file of a folder by the path of the folder', () => { + expect(shippedFileIn(folder, 'notes.txt')).toEqual({ package: shipper, path: 'files/notes.txt' }); + expect(shippedFileName(shippedFileIn(folder, 'notes.txt'))).toBe('@acme/shipper/files/notes.txt'); + }); +}); + +describe.each(readers)('the files shipped %s', (_, reader) => { it('are read as text in UTF-8 and as bytes', async () => { - expect(await reader.text(new URL('notes.txt', folder))).toBe(notes); - expect([...(await reader.bytes(new URL('notes.txt', folder)))]).toEqual([...new TextEncoder().encode(notes)]); - expect([...(await reader.bytes(new URL('mark.bin', folder)))]).toEqual([0, 255, 1]); - expect(await reader.text(new URL('mark.bin', folder))).toBe('\u0000�\u0001'); + const notesFile = shippedFileIn(folder, 'notes.txt'); + const markFile = shippedFileIn(folder, 'mark.bin'); + + expect(await reader.text(notesFile)).toBe(notes); + expect([...(await reader.bytes(notesFile))]).toEqual([...new TextEncoder().encode(notes)]); + expect([...(await reader.bytes(markFile))]).toEqual([0, 255, 1]); + expect(await reader.text(markFile)).toBe('\u0000�\u0001'); }); it('are listed by the names of the files directly in a folder, sorted', async () => { expect(await reader.list(folder)).toEqual(['mark.bin', 'notes.txt']); - expect(await reader.list(new URL('nested/', folder))).toEqual(['inner.txt']); + expect(await reader.list(shippedFileIn(folder, 'nested'))).toEqual(['inner.txt']); }); - it('refuse a file or a folder that is not shipped', async () => { - await expect(reader.text(new URL('missing.txt', folder))).rejects.toThrow('missing.txt'); - await expect(reader.bytes(new URL('missing.txt', folder))).rejects.toThrow('missing.txt'); - await expect(reader.list(new URL('missing/', folder))).rejects.toThrow('missing'); + it('refuse a file or a folder that is not shipped, and the files of a package it holds none of', async () => { + const elsewhere = { package: '@acme/elsewhere', path: 'files/notes.txt' }; + + await expect(reader.text(shippedFileIn(folder, 'missing.txt'))).rejects.toThrow('missing.txt'); + await expect(reader.bytes(shippedFileIn(folder, 'missing.txt'))).rejects.toThrow('missing.txt'); + await expect(reader.list(shippedFileIn(folder, 'missing'))).rejects.toThrow('missing'); + await expect(reader.text(elsewhere)).rejects.toThrow('@acme/elsewhere'); + await expect(reader.list(elsewhere)).rejects.toThrow('@acme/elsewhere'); }); }); diff --git a/packages/operations/src/shipped-files/shipped-files.ts b/packages/operations/src/shipped-files/shipped-files.ts index e28e8a539..1735610f9 100644 --- a/packages/operations/src/shipped-files/shipped-files.ts +++ b/packages/operations/src/shipped-files/shipped-files.ts @@ -1,5 +1,20 @@ +export interface ShippedFile { + readonly package: string; + readonly path: string; +} + +export type ShippedRoots = Readonly>>; + export interface ShippedFiles { - readonly text: (file: Readonly) => Promise; - readonly bytes: (file: Readonly) => Promise; - readonly list: (folder: Readonly) => Promise; + readonly text: (file: ShippedFile) => Promise; + readonly bytes: (file: ShippedFile) => Promise; + readonly list: (folder: ShippedFile) => Promise; +} + +export function shippedFileIn(folder: ShippedFile, name: string): ShippedFile { + return { package: folder.package, path: `${folder.path}/${name}` }; +} + +export function shippedFileName(file: ShippedFile): string { + return `${file.package}/${file.path}`; } diff --git a/packages/providers/README.md b/packages/providers/README.md index 845a97cbe..b187944bc 100644 --- a/packages/providers/README.md +++ b/packages/providers/README.md @@ -38,7 +38,7 @@ The fields of a way, `wayFields`, are the one place of the placement, `in`, `nam ## The shipped entries -`readProviders(shippedFiles)` (`src/reading/shipped-providers.ts`) reads the entries once, at start, from `new URL('../entries/', import.meta.url)` with a listing of the folder, through the `ShippedFiles` of `@on.auto/operations` it is given, which the server composes, then `readYaml` of `@on.auto/config` and the schema, and answers `ProviderList`: the providers sorted by their names, not by the names of their files, and `documents`, each document beside the entries as its text and the bytes it takes, both of the bytes read through the same reader while the entry naming it is checked (`shippedDocumentOf` of `@on.auto/tool-servers`), keyed by the name the entry writes, which the tool servers describe the provider's API from, so no package reads a shipped document again. A provider is its entry and its `address`, the address its calls go to as the entry writes it: `url`, `base_url`, the first server a document beside the entries names, read at start, or the URL of a document that is not. An entry the schema refuses, a file not named for its entry, a document beside the entries that is no JSON, and one whose first server, where the entry gives no `base_url`, is missing or is no https URL, or http URL on a loopback address, without a variable, are `ProvidersInvalid`, naming each problem at the file, its line and its key, as a settings file is named, and the server stops at start. The entries ship because they sit in `src`, in the image and in the npm package alike. +`readProviders(shippedFiles)` (`src/reading/shipped-providers.ts`) reads the entries once, at start, from `src/entries` of the package, `shippedEntries`, with a listing of the folder, through the `ShippedFiles` of `@on.auto/operations` it is given, which the server composes, then `readYaml` of `@on.auto/config` and the schema, and answers `ProviderList`: the providers sorted by their names, not by the names of their files, and `documents`, each document beside the entries as its text and the bytes it takes, both of the bytes read through the same reader while the entry naming it is checked (`shippedDocumentOf` of `@on.auto/tool-servers`), keyed by the name the entry writes, which the tool servers describe the provider's API from, so no package reads a shipped document again. A provider is its entry and its `address`, the address its calls go to as the entry writes it: `url`, `base_url`, the first server a document beside the entries names, read at start, or the URL of a document that is not. An entry the schema refuses, a file not named for its entry, a document beside the entries that is no JSON, and one whose first server, where the entry gives no `base_url`, is missing or is no https URL, or http URL on a loopback address, without a variable, are `ProvidersInvalid`, naming each problem at the file, its line and its key, as a settings file is named, and the server stops at start. The entries ship because they sit in `src`, in the image and in the npm package alike. The entries this runtime ships, each verified against its vendor's documentation when it was written: @@ -157,4 +157,4 @@ The conversion, `src/translating/swagger-conversion.ts`, writes an OpenAPI 3.1 d ## Source -`src/index.ts` is the entry point, and `translate-documents.ts` the command `documents:translate` runs, which writes what `src/translating/document-manifests.ts` translates: the command reads the manifests through `shippedFilesOnDisk` of `@on.auto/operations/shipped-files-on-disk`, and the file a `=` names, which is no file the package ships, itself. `src/entry` holds the schema of an entry, its ways, its placeholders and the reading of its templates; `src/reading` the reading of the shipped entries, the list and what a provider reaches; `src/resolution` the resolution and the checks of a placeholder's value; `src/translating` the translations of a Discovery document and of OpenAPI documents, the conversion of Swagger documents, the `oauth2` scheme they write, and the reading of the manifests of each kind; `src/entries` the shipped entries, their documents and the manifests of those translated; `src/testing` what the tests share, and `src/testing/index.ts`, the entry `@on.auto/providers/testing`, the fixture providers `tracker`, `ledgers`, `regions` and `metrics` and their list, `fixtureProviders`, `drive` and `adverts`, whose way is consent, the one discovering its server's metadata and the other giving its endpoints with a placeholder in them, and `audiences`, whose header holds `{client_id}` beside its way of consent, `providerOf(entry, documentServer?)`, `providerListOf(...providers)` and `providersOf(...entries)`, which the tests of the resolution and of the connections use in place of the shipped ones. +`src/index.ts` is the entry point, and `translate-documents.ts` the command `documents:translate` runs, which writes what `src/translating/document-manifests.ts` translates: the command reads the manifests through `shippedFilesOnDisk(providersShippedRoots)` of `@on.auto/operations/shipped-files-on-disk`, and the file a `=` names, which is no file the package ships, itself, and writes each document the translation answers as a file of the package where it is shipped. `providersShippedRoots`, of the main entry, is the root of the package, which the server gives the reader it composes. `src/entry` holds the schema of an entry, its ways, its placeholders and the reading of its templates; `src/reading` the reading of the shipped entries, the list and what a provider reaches; `src/resolution` the resolution and the checks of a placeholder's value; `src/translating` the translations of a Discovery document and of OpenAPI documents, the conversion of Swagger documents, the `oauth2` scheme they write, and the reading of the manifests of each kind; `src/entries` the shipped entries, their documents and the manifests of those translated; `src/testing` what the tests share, and `src/testing/index.ts`, the entry `@on.auto/providers/testing`, the fixture providers `tracker`, `ledgers`, `regions` and `metrics` and their list, `fixtureProviders`, `drive` and `adverts`, whose way is consent, the one discovering its server's metadata and the other giving its endpoints with a placeholder in them, and `audiences`, whose header holds `{client_id}` beside its way of consent, `providerOf(entry, documentServer?)`, `providerListOf(...providers)` and `providersOf(...entries)`, which the tests of the resolution and of the connections use in place of the shipped ones. diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 6433035d3..92ef7e4a6 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -30,7 +30,7 @@ export { operatorVariablesIn, operatorVariablesOf, } from './reading/provider-reach.ts'; -export { ProvidersInvalid, readProviders } from './reading/shipped-providers.ts'; +export { ProvidersInvalid, providersShippedRoots, readProviders } from './reading/shipped-providers.ts'; export { additionOfProvider, type GivenToProvider, diff --git a/packages/providers/src/reading/search-console-properties.test.ts b/packages/providers/src/reading/search-console-properties.test.ts index 1a50d3a6f..e4196b433 100644 --- a/packages/providers/src/reading/search-console-properties.test.ts +++ b/packages/providers/src/reading/search-console-properties.test.ts @@ -7,7 +7,9 @@ import { describedAccess, patientTiming, serveFakeApi } from '@on.auto/tool-serv import { Effect, Result } from 'effect'; import { afterEach, describe, expect, it } from 'vitest'; -import { providerNamed, readProviders } from '../index.ts'; +import { providerNamed, readProviders, providersShippedRoots } from '../index.ts'; + +const providersOnDisk = shippedFilesOnDisk(providersShippedRoots); const page = readFileSync(new URL('../../../../docs/self-host/connections.md', import.meta.url), 'utf8'); @@ -19,7 +21,7 @@ const [, domainProperty = '', urlPrefixProperty = '', sentPath = ''] = [...sente ([, quoted = '']: readonly string[]) => quoted, ); -const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); +const shipped = Result.getOrThrow(await readProviders(providersOnDisk)); const entriesFolder = new URL('../entries/', import.meta.url); diff --git a/packages/providers/src/reading/shipped-documents.test.ts b/packages/providers/src/reading/shipped-documents.test.ts index e95fa7af5..20e5ca844 100644 --- a/packages/providers/src/reading/shipped-documents.test.ts +++ b/packages/providers/src/reading/shipped-documents.test.ts @@ -10,6 +10,9 @@ import { manifestNamesIn, type DocumentManifest, } from '../translating/document-manifests.ts'; +import { providersShippedRoots } from './shipped-providers.ts'; + +const providersOnDisk = shippedFilesOnDisk(providersShippedRoots); const RequirementsSchema = Schema.Array(Schema.Record(Schema.String, Schema.Unknown)); @@ -32,7 +35,7 @@ type Held = ReturnType; function heldBy(name: string) { const { security, paths, components } = decodeOperations( - readFileSync(new URL(`${name}.openapi.json`, documentsBesideTheEntries), 'utf8'), + readFileSync(new URL(`../entries/documents/${name}.openapi.json`, import.meta.url), 'utf8'), ); const operations = Object.values(paths).flatMap((methods) => Object.values(methods)); const requirements = [...security, ...operations.flatMap((operation) => operation.security ?? [])]; @@ -69,9 +72,9 @@ interface Kept { } const manifests: readonly Kept[] = await Promise.all( - (await manifestNamesIn(shippedFilesOnDisk, documentsBesideTheEntries)).map(async (name) => ({ + (await manifestNamesIn(providersOnDisk, documentsBesideTheEntries)).map(async (name) => ({ name, - manifest: await manifestIn(shippedFilesOnDisk, documentsBesideTheEntries, name), + manifest: await manifestIn(providersOnDisk, documentsBesideTheEntries, name), })), ); diff --git a/packages/providers/src/reading/shipped-providers.test.ts b/packages/providers/src/reading/shipped-providers.test.ts index 10b83b880..1bb96fa13 100644 --- a/packages/providers/src/reading/shipped-providers.test.ts +++ b/packages/providers/src/reading/shipped-providers.test.ts @@ -1,13 +1,23 @@ import { readdirSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; +import { shippedFileIn } from '@on.auto/operations'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Result } from 'effect'; import { describe, expect, it } from 'vitest'; -import { documentServerOf, operatorVariablesOf, readProviders, type Provider } from '../index.ts'; +import { + documentServerOf, + operatorVariablesOf, + readProviders, + type Provider, + providersShippedRoots, +} from '../index.ts'; +import { shippedEntries } from './shipped-providers.ts'; -const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); +const providersOnDisk = shippedFilesOnDisk(providersShippedRoots); + +const shipped = Result.getOrThrow(await readProviders(providersOnDisk)); const entriesUrl = new URL('../entries/', import.meta.url); @@ -175,8 +185,8 @@ describe('the servers of the shipped providers', () => { it('hand on the documents beside the entries as the texts they read with the bytes each takes, by the name each entry writes', async () => { const onDisk = await Promise.all( documentsBesideTheEntries.map(async (written) => { - const file = new URL(written, entriesUrl); - return { text: await shippedFilesOnDisk.text(file), bytes: (await shippedFilesOnDisk.bytes(file)).byteLength }; + const file = shippedFileIn(shippedEntries, written); + return { text: await providersOnDisk.text(file), bytes: (await providersOnDisk.bytes(file)).byteLength }; }), ); diff --git a/packages/providers/src/reading/shipped-providers.ts b/packages/providers/src/reading/shipped-providers.ts index 6c3a1b1ec..736f36952 100644 --- a/packages/providers/src/reading/shipped-providers.ts +++ b/packages/providers/src/reading/shipped-providers.ts @@ -1,5 +1,3 @@ -import { fileURLToPath } from 'node:url'; - import { placeholdersIn, placeIn, @@ -9,7 +7,13 @@ import { type Position, type YamlKind, } from '@on.auto/config'; -import type { ShippedFiles } from '@on.auto/operations'; +import { + shippedFileIn, + shippedFileName, + type ShippedFile, + type ShippedFiles, + type ShippedRoots, +} from '@on.auto/operations'; import { isPinnableUrl, shippedDocumentOf, type ShippedDocument } from '@on.auto/tool-servers'; import { Data, Option, Result, Schema, SchemaIssue, type StandardSchema } from 'effect'; @@ -33,12 +37,16 @@ type Read = Result.Result; interface Entries { readonly shippedFiles: ShippedFiles; - readonly folder: Readonly; + readonly folder: ShippedFile; } type PathSegment = PropertyKey | StandardSchema.StandardSchemaV1.PathSegment; -const shippedEntries = new URL('../entries/', import.meta.url); +const providersPackage = '@on.auto/providers'; + +export const providersShippedRoots: ShippedRoots = { [providersPackage]: new URL('../../', import.meta.url) }; + +export const shippedEntries: ShippedFile = { package: providersPackage, path: 'src/entries' }; const entryYaml: YamlKind = { noun: "a provider's entry", @@ -84,7 +92,7 @@ async function documentBesideTheEntries( written: string, ): Promise { try { - const shipped = shippedDocumentOf(await shippedFiles.bytes(new URL(written, folder))); + const shipped = shippedDocumentOf(await shippedFiles.bytes(shippedFileIn(folder, written))); const parsed: unknown = JSON.parse(shipped.text); return Result.getOrUndefined(Result.map(decodeServers(parsed), (servers) => ({ shipped, servers }))); } catch { @@ -144,12 +152,12 @@ async function providerOf(entry: ProviderEntry, file: string, entries: Entries): } async function readProvider(entries: Entries, file: string): Promise> { - const at = new URL(file, entries.folder); - const path = fileURLToPath(at); + const at = shippedFileIn(entries.folder, file); + const name = shippedFileName(at); const reading = readYaml(await entries.shippedFiles.text(at), entryYaml); if ('problems' in reading) { return Result.fail( - reading.problems.map(({ position, detail }) => ({ place: placeIn(path, position, ''), detail })), + reading.problems.map(({ position, detail }) => ({ place: placeIn(name, position, ''), detail })), ); } const { document } = reading; @@ -159,7 +167,7 @@ async function readProvider(entries: Entries, file: string): Promise { const position: Position = document.locate(pointer); - return { place: placeIn(path, position, pointer), detail }; + return { place: placeIn(name, position, pointer), detail }; }; return Result.mapError(read, (problems) => problems.map((problem) => located(problem))); } @@ -171,7 +179,7 @@ function invalid(located: readonly Located[]): ProvidersInvalid { export async function readProviders( shippedFiles: ShippedFiles, - folder: Readonly = shippedEntries, + folder: ShippedFile = shippedEntries, ): Promise> { const entries: Entries = { shippedFiles, folder }; const files = (await shippedFiles.list(folder)).filter((file) => file.endsWith('.yaml')); diff --git a/packages/providers/src/reading/shipped-ways.test.ts b/packages/providers/src/reading/shipped-ways.test.ts index d6d948050..b27f9d41a 100644 --- a/packages/providers/src/reading/shipped-ways.test.ts +++ b/packages/providers/src/reading/shipped-ways.test.ts @@ -2,9 +2,17 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Result } from 'effect'; import { describe, expect, it } from 'vitest'; -import { hostAlternativesOf, readProviders, type CredentialWay, type ProviderEntry } from '../index.ts'; +import { + hostAlternativesOf, + readProviders, + type CredentialWay, + type ProviderEntry, + providersShippedRoots, +} from '../index.ts'; -const shipped = Result.getOrThrow(await readProviders(shippedFilesOnDisk)); +const providersOnDisk = shippedFilesOnDisk(providersShippedRoots); + +const shipped = Result.getOrThrow(await readProviders(providersOnDisk)); const google = 'accounts.google.com'; diff --git a/packages/providers/src/testing/entry-problems.ts b/packages/providers/src/testing/entry-problems.ts index 84d08efd0..551edfc07 100644 --- a/packages/providers/src/testing/entry-problems.ts +++ b/packages/providers/src/testing/entry-problems.ts @@ -1,5 +1,4 @@ -import { pathToFileURL } from 'node:url'; - +import { shippedFileIn, shippedFileName, type ShippedFile } from '@on.auto/operations'; import { shippedFilesInMemory } from '@on.auto/operations/testing'; import { Result, Schema, SchemaIssue } from 'effect'; @@ -26,16 +25,16 @@ export interface EntryFolder { readonly read: () => Promise; } -const folder = '/shipped/entries'; +const entries: ShippedFile = { package: '@acme/shipped', path: 'entries' }; -const entries = pathToFileURL(`${folder}/`); +const folder = shippedFileName(entries); export function entryFolderWith(files: Readonly>): EntryFolder { - const shipped = shippedFilesInMemory( - Object.fromEntries( - Object.entries(files).map(([name, text]: readonly [string, string]) => [new URL(name, entries).href, text]), + const shipped = shippedFilesInMemory({ + [entries.package]: Object.fromEntries( + Object.entries(files).map(([name, text]: readonly [string, string]) => [shippedFileIn(entries, name).path, text]), ), - ); + }); return { read: async () => Result.match(await readProviders(shipped, entries), { diff --git a/packages/providers/src/translating/document-manifests.test.ts b/packages/providers/src/translating/document-manifests.test.ts index 582f4eec0..c24356308 100644 --- a/packages/providers/src/translating/document-manifests.test.ts +++ b/packages/providers/src/translating/document-manifests.test.ts @@ -1,6 +1,4 @@ -import { fileURLToPath, pathToFileURL } from 'node:url'; - -import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFileIn, type ShippedFile, type ShippedFiles } from '@on.auto/operations'; import { shippedFilesInMemory } from '@on.auto/operations/testing'; import { describe, expect, it, onTestFinished } from 'vitest'; @@ -47,24 +45,24 @@ function fetchingFrom(answered: ReadonlyMap): Fetching { }; } -const folder = pathToFileURL('/shipped/documents/'); +const folder: ShippedFile = { package: '@acme/shipped', path: 'documents' }; -function pathOf(file: string): string { - return fileURLToPath(new URL(file, folder)); +function fileOf(name: string): ShippedFile { + return shippedFileIn(folder, name); } function shippedWith(files: Readonly>): ShippedFiles { onTestFinished(() => { fetched.splice(0); }); - return shippedFilesInMemory( - Object.fromEntries( + return shippedFilesInMemory({ + [folder.package]: Object.fromEntries( Object.entries(files).map(([name, content]: readonly [string, unknown]) => [ - new URL(name, folder).href, + fileOf(name).path, JSON.stringify(content), ]), ), - ); + }); } function parsed(text: string): unknown { @@ -108,7 +106,7 @@ describe('the documents translated from their manifests', () => { const written = await translatedIn(shipped); const read = written.map(({ text }) => parsed(text)); - expect(written.map(({ path }) => path)).toEqual([pathOf('tracker.openapi.json'), pathOf('tracker.manifest.json')]); + expect(written.map(({ file }) => file)).toEqual([fileOf('tracker.openapi.json'), fileOf('tracker.manifest.json')]); expect(read).toEqual([trackerTranslated, remade]); expect(written.map(({ text }) => text)).toEqual(read.map((value) => textOf(value))); expect(fetched).toEqual([trackerManifest.discovery, trackerManifest.authorization_server]); @@ -120,7 +118,7 @@ describe('the documents translated from their manifests', () => { const written = await translatedIn(shipped, ['tracker=/given/tracker-v1.json'], answers, given); - expect(written.map(({ path }) => path)).toEqual([pathOf('tracker.openapi.json'), pathOf('tracker.manifest.json')]); + expect(written.map(({ file }) => file)).toEqual([fileOf('tracker.openapi.json'), fileOf('tracker.manifest.json')]); expect(fetched).toEqual([trackerManifest.authorization_server]); expect(await manifestNamesIn(shipped, folder)).toEqual(['bare', 'tracker']); }); @@ -220,8 +218,8 @@ describe('the documents translated from the manifests of OpenAPI and Swagger doc const read = written.map(({ text }) => parsed(text)); const converted = { info: notesManifest.info, security: [{ oauth2: [] }] }; - expect(written.map(({ path }) => path)).toEqual( - ['about', 'notes', 'tracker'].map((name) => pathOf(`${name}.openapi.json`)), + expect(written.map(({ file }) => file)).toEqual( + ['about', 'notes', 'tracker'].map((name) => fileOf(`${name}.openapi.json`)), ); expect(read).toEqual([aboutTranslated, expect.objectContaining(converted), trackerOpenApiTranslated]); expect(written.map(({ text }) => text)).toEqual(read.map((value) => textOf(value))); diff --git a/packages/providers/src/translating/document-manifests.ts b/packages/providers/src/translating/document-manifests.ts index 970791ac1..a00994e1c 100644 --- a/packages/providers/src/translating/document-manifests.ts +++ b/packages/providers/src/translating/document-manifests.ts @@ -1,15 +1,14 @@ -import { fileURLToPath } from 'node:url'; - -import { listed, type ShippedFiles } from '@on.auto/operations'; +import { listed, shippedFileIn, type ShippedFile, type ShippedFiles } from '@on.auto/operations'; import { Schema } from 'effect'; +import { shippedEntries } from '../reading/shipped-providers.ts'; import { CutSchema } from './discovery-cuts.ts'; import { OpenApiTranslationSchema } from './openapi-operations.ts'; import { convertedDocumentOf, SwaggerConversionSchema } from './swagger-conversion.ts'; import { translatedDiscoveryOf } from './translate-discovery.ts'; import { curatedOpenApiOf } from './translate-openapi.ts'; -export const documentsBesideTheEntries = new URL('../entries/documents/', import.meta.url); +export const documentsBesideTheEntries: ShippedFile = shippedFileIn(shippedEntries, 'documents'); const AuthorizationServerAddressSchema = Schema.String.annotate({ description: 'The address of the metadata of the authorization server whose endpoints the oauth2 scheme names', @@ -66,14 +65,14 @@ const documentSuffix = '.openapi.json'; export type Fetching = (address: string) => Promise; export interface Translating { - readonly folder: Readonly; + readonly folder: ShippedFile; readonly shippedFiles: ShippedFiles; readonly fetching: Fetching; readonly readGiven: (path: string) => Promise; } export interface Written { - readonly path: string; + readonly file: ShippedFile; readonly text: string; } @@ -85,7 +84,7 @@ interface Asked { const usage = 'Usage: pnpm --filter @on.auto/providers documents:translate [[=]]..., each name a manifest beside the documents, every manifest when none is named'; -export async function manifestNamesIn(shippedFiles: ShippedFiles, folder: Readonly): Promise { +export async function manifestNamesIn(shippedFiles: ShippedFiles, folder: ShippedFile): Promise { return (await shippedFiles.list(folder)) .filter((file) => file.endsWith(manifestSuffix)) .map((file) => file.slice(0, -manifestSuffix.length)) @@ -94,10 +93,10 @@ export async function manifestNamesIn(shippedFiles: ShippedFiles, folder: Readon export async function manifestIn( shippedFiles: ShippedFiles, - folder: Readonly, + folder: ShippedFile, name: string, ): Promise { - return decodeManifest(await shippedFiles.text(new URL(`${name}${manifestSuffix}`, folder))); + return decodeManifest(await shippedFiles.text(shippedFileIn(folder, `${name}${manifestSuffix}`))); } function askedOf(commandArgument: string, names: readonly string[]): Asked { @@ -128,10 +127,6 @@ function jsonText(value: unknown): string { return `${JSON.stringify(value, null, 2)}\n`; } -function pathIn(folder: Readonly, file: string): string { - return fileURLToPath(new URL(file, folder)); -} - async function discoveryTranslated( { name, given }: Asked, manifest: DiscoveryManifest, @@ -144,9 +139,9 @@ async function discoveryTranslated( const { type, discovery: address, authorization_server: authorizationServer, methods, cuts } = manifest; const remade = { type, discovery: address, revision, authorization_server: authorizationServer, methods }; return [ - { path: pathIn(folder, `${name}${documentSuffix}`), text: jsonText(document) }, + { file: shippedFileIn(folder, `${name}${documentSuffix}`), text: jsonText(document) }, { - path: pathIn(folder, `${name}${manifestSuffix}`), + file: shippedFileIn(folder, `${name}${manifestSuffix}`), text: jsonText(cuts === undefined ? remade : { ...remade, cuts }), }, ]; @@ -173,7 +168,7 @@ async function openApiTranslated( const documents = await fetchedAt(asked, manifest.sources, translating); const metadata = await fetchedJson(manifest.authorization_server, translating.fetching); const document = curatedOpenApiOf(documents, manifest, metadata); - return [{ path: pathIn(translating.folder, `${asked.name}${documentSuffix}`), text: jsonText(document) }]; + return [{ file: shippedFileIn(translating.folder, `${asked.name}${documentSuffix}`), text: jsonText(document) }]; } async function swaggerConverted( @@ -188,7 +183,7 @@ async function swaggerConverted( ); const metadata = await fetchedJson(manifest.authorization_server, translating.fetching); const document = convertedDocumentOf(documents, manifest, metadata); - return [{ path: pathIn(translating.folder, `${asked.name}${documentSuffix}`), text: jsonText(document) }]; + return [{ file: shippedFileIn(translating.folder, `${asked.name}${documentSuffix}`), text: jsonText(document) }]; } async function translatedOne(asked: Asked, translating: Translating): Promise { diff --git a/packages/providers/translate-documents.ts b/packages/providers/translate-documents.ts index d7c22f320..edd99be79 100644 --- a/packages/providers/translate-documents.ts +++ b/packages/providers/translate-documents.ts @@ -5,6 +5,7 @@ import { fileURLToPath } from 'node:url'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { providersShippedRoots } from './src/reading/shipped-providers.ts'; import { documentsBesideTheEntries, translatedDocuments } from './src/translating/document-manifests.ts'; const repository = fileURLToPath(new URL('../../', import.meta.url)); @@ -13,12 +14,16 @@ const formatter = fileURLToPath(new URL('../../node_modules/.bin/oxfmt', import. const written = await translatedDocuments(process.argv.slice(2), { folder: documentsBesideTheEntries, - shippedFiles: shippedFilesOnDisk, + shippedFiles: shippedFilesOnDisk(providersShippedRoots), fetching: fetch, readGiven: (path) => readFile(path, 'utf8'), }); -const paths = written.map(({ path }) => path); -for (const { path, text } of written) { +const onDisk: readonly { readonly path: string; readonly text: string }[] = written.map(({ file, text }) => ({ + path: fileURLToPath(new URL(file.path, import.meta.url)), + text, +})); +const paths = onDisk.map(({ path }) => path); +for (const { path, text } of onDisk) { writeFileSync(path, text); } execFileSync(formatter, paths, { cwd: repository, stdio: 'inherit' }); diff --git a/packages/server/README.md b/packages/server/README.md index e5bc8af4d..3783a5072 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -1,6 +1,6 @@ # @on.auto/server -The Node.js server of Auto Runtime: `src/main.ts` starts it on the composition root of `src/composition/composition-root.ts`, `dev.ts` restarts it on every save, and `auto-runtime.ts` is the command of the npm package `@on.auto/runtime`, which `npm/pack.ts` packs with the `README.md` in `npm/`; the `Dockerfile` builds its container image. How the files the packages ship are read is composed once, `shippedFilesOnDisk` of `@on.auto/operations/shipped-files-on-disk` in the options of `src/server-options/composed-options.ts`, and handed on as `ServerOptions.shippedFiles`: at the start, to the providers' entries the settings are read against (`readShippedSettings`, `src/settings/served-settings.ts`) and to the style of the API's pages, `readPageStyle` of `@on.auto/api`, each read once and handed on as content, so a file that cannot be read stops the start (`src/lifecycle/lifecycle.ts`, `src/lifecycle/page-style-startup.test.ts`); then to `composed`, and from there to the files the pool of workers starts with, read where the one pool is made (`src/composition/served-computation.ts`), and to the guides and recipes the server serves (`src/guides`). `src/server-options/server-options.ts` holds what a server is started with and answers, `ServerOptions`, `Composed`, `Served` and `RunningServer` among them, which the start in `src/lifecycle` and every part the composition serves import, so nothing imports the start for them. The composition imports the entry of the ledger of the store its settings name, and no other: `ledgerDatabaseOf` of `src/database/ledger-database.ts` imports `src/database/sqlite-ledger-database.ts`, over `@on.auto/ledger/sqlite3`, or `src/database/postgresql-ledger-database.ts`, over `@on.auto/ledger/postgresql` and `pg`, through a dynamic import, and answers the database the workflow host opens with the layer of the ledger and the rows of the secrets on it, which `composed` awaits before it builds anything, so a server on PostgreSQL never loads the native addon of `sqlite3` and one on SQLite never loads `pg` (`src/database/ledger-entries.test.ts`); `composed` may therefore answer a promise, which the start awaits. +The Node.js server of Auto Runtime: `src/main.ts` starts it on the composition root of `src/composition/composition-root.ts`, `dev.ts` restarts it on every save, and `auto-runtime.ts` is the command of the npm package `@on.auto/runtime`, which `npm/pack.ts` packs with the `README.md` in `npm/`; the `Dockerfile` builds its container image. How the files the packages ship are read is composed once, `shippedFilesOnDisk(shippedRoots)` of `@on.auto/operations/shipped-files-on-disk` in the options of `src/server-options/composed-options.ts`, over the roots each package declares, the API's, the check's with the sandbox's, the providers' and the server's own (`src/server-options/shipped-roots.ts`): the recipes are the server's, `src/guides/recipes` of `@on.auto/server`, and the documentation it serves, `docs` of `@on.auto/runtime`, sits two folders above the server's root in the repository, the image and the npm package alike, so the server declares that root too (`src/guides/guide-files.ts`); and handed on as `ServerOptions.shippedFiles`: at the start, to the providers' entries the settings are read against (`readShippedSettings`, `src/settings/served-settings.ts`) and to the style of the API's pages, `readPageStyle` of `@on.auto/api`, each read once and handed on as content, so a file that cannot be read stops the start (`src/lifecycle/lifecycle.ts`, `src/lifecycle/page-style-startup.test.ts`); then to `composed`, and from there to the files the pool of workers starts with, read where the one pool is made (`src/composition/served-computation.ts`), and to the guides and recipes the server serves (`src/guides`). `src/server-options/server-options.ts` holds what a server is started with and answers, `ServerOptions`, `Composed`, `Served` and `RunningServer` among them, which the start in `src/lifecycle` and every part the composition serves import, so nothing imports the start for them. The composition imports the entry of the ledger of the store its settings name, and no other: `ledgerDatabaseOf` of `src/database/ledger-database.ts` imports `src/database/sqlite-ledger-database.ts`, over `@on.auto/ledger/sqlite3`, or `src/database/postgresql-ledger-database.ts`, over `@on.auto/ledger/postgresql` and `pg`, through a dynamic import, and answers the database the workflow host opens with the layer of the ledger and the rows of the secrets on it, which `composed` awaits before it builds anything, so a server on PostgreSQL never loads the native addon of `sqlite3` and one on SQLite never loads `pg` (`src/database/ledger-entries.test.ts`); `composed` may therefore answer a promise, which the start awaits. ## What is composed at start, and what on first use diff --git a/packages/server/src/guides/guide-files.ts b/packages/server/src/guides/guide-files.ts new file mode 100644 index 000000000..183302a4b --- /dev/null +++ b/packages/server/src/guides/guide-files.ts @@ -0,0 +1,14 @@ +import type { ShippedFile, ShippedRoots } from '@on.auto/operations'; + +const serverPackage = '@on.auto/server'; + +const runtimePackage = '@on.auto/runtime'; + +export const serverShippedRoots: ShippedRoots = { + [serverPackage]: new URL('../../', import.meta.url), + [runtimePackage]: new URL('../../../../', import.meta.url), +}; + +export const recipesFolder: ShippedFile = { package: serverPackage, path: 'src/guides/recipes' }; + +export const documentation: ShippedFile = { package: runtimePackage, path: 'docs' }; diff --git a/packages/server/src/guides/recipes.ts b/packages/server/src/guides/recipes.ts index a93e80785..172b3f1ab 100644 --- a/packages/server/src/guides/recipes.ts +++ b/packages/server/src/guides/recipes.ts @@ -1,5 +1,7 @@ import type { Guide, Recipe } from '@on.auto/api'; -import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFileIn, type ShippedFiles } from '@on.auto/operations'; + +import { recipesFolder } from './guide-files.ts'; type RecipeOutline = Omit; @@ -89,7 +91,7 @@ async function textOf( { name }: RecipeOutline, carried: ReadonlySet, ): Promise { - const text = await shippedFiles.text(new URL(`recipes/${name}.md`, import.meta.url)); + const text = await shippedFiles.text(shippedFileIn(recipesFolder, `${name}.md`)); const insertion = carried.has(interactionGuide) ? whereInteractionIsServed[name] : undefined; return insertion === undefined ? text : text.replace(insertion.after, `${insertion.after}${insertion.words}`); } diff --git a/packages/server/src/guides/served-guides.test.ts b/packages/server/src/guides/served-guides.test.ts index f599d5b55..643f2da21 100644 --- a/packages/server/src/guides/served-guides.test.ts +++ b/packages/server/src/guides/served-guides.test.ts @@ -1,4 +1,3 @@ -import { Buffer } from 'node:buffer'; import { readFileSync } from 'node:fs'; import { mostGuideBytes, mostRecipeBytes } from '@on.auto/api'; @@ -7,10 +6,13 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; +import { serverShippedRoots } from './guide-files.ts'; import { withLinksResolved } from './page-links.ts'; import { servedGuidesOf, type ServedGuides } from './served-guides.ts'; import { withoutSiteMarkup } from './site-markup.ts'; +const guideFiles = shippedFilesOnDisk(serverShippedRoots); + function capabilityOf(name: string, noun: string, guide: CapabilityGuide): Capability { return defineCapability({ type: name, @@ -41,18 +43,22 @@ const everyType = [reasoning, computation, recall, prediction, workflow]; const interaction = capabilityOf('interaction', 'interaction function', { name: 'interaction-function' }); -const served = await servedGuidesOf(shippedFilesOnDisk, everyType); +const served = await servedGuidesOf(guideFiles, everyType); -const servedWithInteraction = await servedGuidesOf(shippedFilesOnDisk, [...everyType, interaction]); +const servedWithInteraction = await servedGuidesOf(guideFiles, [...everyType, interaction]); -const servedForReasoningAndWorkflows = await servedGuidesOf(shippedFilesOnDisk, [reasoning, workflow]); +const servedForReasoningAndWorkflows = await servedGuidesOf(guideFiles, [reasoning, workflow]); function recipeTextsOf({ recipes }: ServedGuides): Readonly> { return Object.fromEntries(recipes.map(({ name, text }) => [name, text])); } +function byteLengthOf(text: string): number { + return new TextEncoder().encode(text).byteLength; +} + function bytesOf(texts: Readonly>, ...names: readonly string[]): readonly number[] { - return names.map((name) => Buffer.byteLength(texts[name] ?? '')); + return names.map((name) => byteLengthOf(texts[name] ?? '')); } function page(path: string): string { @@ -147,8 +153,8 @@ describe('the size and the words of the guides of a server', () => { const everyGuide = servedWithInteraction.guides; expect(everyGuide.map(({ name }) => name)).toContain('interaction-function'); - expect(everyGuide.filter(({ text }) => Buffer.byteLength(text, 'utf8') > mostGuideBytes)).toEqual([]); - expect(recipes.filter(({ text }) => Buffer.byteLength(text, 'utf8') > mostRecipeBytes)).toEqual([]); + expect(everyGuide.filter(({ text }) => byteLengthOf(text) > mostGuideBytes)).toEqual([]); + expect(recipes.filter(({ text }) => byteLengthOf(text) > mostRecipeBytes)).toEqual([]); }); it('describe each guide in a sentence for a person or a client that lists them', () => { @@ -172,7 +178,7 @@ describe('the recipes of a server', () => { }); it('are none for a server that runs no type of definition', async () => { - expect((await servedGuidesOf(shippedFilesOnDisk, [])).recipes).toEqual([]); + expect((await servedGuidesOf(guideFiles, [])).recipes).toEqual([]); }); it('ask the person in their own words, filled into what the prompt asks for', () => { @@ -207,10 +213,9 @@ describe('the recipes that give a playbook tools', () => { expect(String(texts['first-playbook'])).toContain( "When it should use a tool server's tools, follow the give-tools recipe: to learn what a tool answers, call it with execute, and never make a function to look.", ); - expect([ - Buffer.byteLength(String(texts['give-tools'])), - Buffer.byteLength(String(texts['first-playbook'])), - ]).toEqual([2716, 1898]); + expect([byteLengthOf(String(texts['give-tools'])), byteLengthOf(String(texts['first-playbook']))]).toEqual([ + 2716, 1898, + ]); }); }); @@ -285,7 +290,7 @@ describe('the terminology guide of a server', () => { describe('a server without the page of a type it runs', () => { it('fails to start', async () => { await expect( - servedGuidesOf(shippedFilesOnDisk, [capabilityOf('drafting', 'draft', { name: 'drafting-function' })]), + servedGuidesOf(guideFiles, [capabilityOf('drafting', 'draft', { name: 'drafting-function' })]), ).rejects.toThrow(/no such file or directory.*drafting-format\.md/u); }); }); diff --git a/packages/server/src/guides/served-guides.ts b/packages/server/src/guides/served-guides.ts index 33870653a..7cb2224a9 100644 --- a/packages/server/src/guides/served-guides.ts +++ b/packages/server/src/guides/served-guides.ts @@ -1,7 +1,8 @@ import type { DefinitionType, Guide, Recipe } from '@on.auto/api'; import type { Capability } from '@on.auto/definitions'; -import { capitalized, type ShippedFiles } from '@on.auto/operations'; +import { capitalized, shippedFileIn, type ShippedFiles } from '@on.auto/operations'; +import { documentation } from './guide-files.ts'; import { withLinksResolved } from './page-links.ts'; import { recipesFor } from './recipes.ts'; import { withoutSiteMarkup } from './site-markup.ts'; @@ -13,8 +14,6 @@ export interface ServedGuides { readonly recipes: readonly Recipe[]; } -const documentation = new URL('../../../../docs/', import.meta.url); - const terminologyPage = 'concepts/terminology.md'; function pageOfGuide(guide: string): string { @@ -22,7 +21,7 @@ function pageOfGuide(guide: string): string { } function pageText(shippedFiles: ShippedFiles, page: string): Promise { - return shippedFiles.text(new URL(page, documentation)); + return shippedFiles.text(shippedFileIn(documentation, page)); } async function typeGuideOf( diff --git a/packages/server/src/lifecycle/page-style-startup.test.ts b/packages/server/src/lifecycle/page-style-startup.test.ts index 46cfec3e5..43f8adc22 100644 --- a/packages/server/src/lifecycle/page-style-startup.test.ts +++ b/packages/server/src/lifecycle/page-style-startup.test.ts @@ -1,16 +1,19 @@ -import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFileName, type ShippedFiles } from '@on.auto/operations'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; +import { shippedRoots } from '../server-options/shipped-roots.ts'; import { defaultServerOptions } from '../testing/entries/serving-options.ts'; import { startServer } from './lifecycle.ts'; +const shippedOnDisk = shippedFilesOnDisk(shippedRoots); + const withoutTypefaces: ShippedFiles = { - ...shippedFilesOnDisk, + ...shippedOnDisk, bytes: (file) => - file.pathname.endsWith('.woff2') - ? Promise.reject(new Error(`No file is shipped at ${file.href}`)) - : shippedFilesOnDisk.bytes(file), + file.path.endsWith('.woff2') + ? Promise.reject(new Error(`No file is shipped at ${shippedFileName(file)}`)) + : shippedOnDisk.bytes(file), }; describe('a server whose pages cannot be styled', () => { @@ -18,7 +21,7 @@ describe('a server whose pages cannot be styled', () => { const options = { ...defaultServerOptions, shippedFiles: withoutTypefaces }; await expect(startServer({ HOST: '127.0.0.1', PORT: '0' }, options)).rejects.toThrow( - /^No file is shipped at file:.*\.woff2$/u, + /^No file is shipped at @on\.auto\/api\/.*\.woff2$/u, ); }); }); diff --git a/packages/server/src/server-options/composed-options.ts b/packages/server/src/server-options/composed-options.ts index 44c890fbd..0c44112c3 100644 --- a/packages/server/src/server-options/composed-options.ts +++ b/packages/server/src/server-options/composed-options.ts @@ -3,9 +3,10 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { exitDeadlineMs, shutdownTimeoutMs } from '../lifecycle/shutdown.ts'; import type { ServerOptions } from './server-options.ts'; +import { shippedRoots } from './shipped-roots.ts'; export type ComposedServerOptions = ServerOptions; export function serverOptionsComposing(composed: ServerOptions['composed']): ServerOptions { - return { composed, shutdownTimeoutMs, exitDeadlineMs, shippedFiles: shippedFilesOnDisk }; + return { composed, shutdownTimeoutMs, exitDeadlineMs, shippedFiles: shippedFilesOnDisk(shippedRoots) }; } diff --git a/packages/server/src/server-options/shipped-roots.ts b/packages/server/src/server-options/shipped-roots.ts new file mode 100644 index 000000000..4564df3af --- /dev/null +++ b/packages/server/src/server-options/shipped-roots.ts @@ -0,0 +1,13 @@ +import { apiShippedRoots } from '@on.auto/api'; +import { checkShippedRoots } from '@on.auto/definitions/check'; +import type { ShippedRoots } from '@on.auto/operations'; +import { providersShippedRoots } from '@on.auto/providers'; + +import { serverShippedRoots } from '../guides/guide-files.ts'; + +export const shippedRoots: ShippedRoots = { + ...apiShippedRoots, + ...checkShippedRoots, + ...providersShippedRoots, + ...serverShippedRoots, +}; diff --git a/packages/server/src/testing/settings/shipped-settings.ts b/packages/server/src/testing/settings/shipped-settings.ts index 2d405db89..fa373e013 100644 --- a/packages/server/src/testing/settings/shipped-settings.ts +++ b/packages/server/src/testing/settings/shipped-settings.ts @@ -1,10 +1,13 @@ import type { Environment } from '@on.auto/config'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { providersShippedRoots } from '@on.auto/providers'; import { readShippedSettings, type ShippedSettings } from '../../settings/served-settings.ts'; import { readSettings, type Settings } from '../../settings/settings.ts'; -export const shippedSettings: ShippedSettings = await readShippedSettings(shippedFilesOnDisk); +const providersFiles = shippedFilesOnDisk(providersShippedRoots); + +export const shippedSettings: ShippedSettings = await readShippedSettings(providersFiles); export function readSettingsOf(environment: Environment): Settings { return readSettings(environment, shippedSettings); diff --git a/packages/server/src/workflows/host-work.test.ts b/packages/server/src/workflows/host-work.test.ts index d4b75c7a8..30b9474e6 100644 --- a/packages/server/src/workflows/host-work.test.ts +++ b/packages/server/src/workflows/host-work.test.ts @@ -8,6 +8,8 @@ import { handedWorkersOf, machineSandboxOf, programPool, + sandboxModuleIn, + sandboxShippedRoots, } from '@on.auto/workflow-engine/dsl'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -15,6 +17,10 @@ import { describe, expect, it } from 'vitest'; import { memoryRuntime } from '../testing/records/memory-runtime.ts'; import { hostWorkOf } from './host-work.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const sandboxModule = await sandboxModuleIn(sandboxFiles); + const unknownRun = { org: 'acme', playbook: 'alpha', id: '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a' }; const lineage = { causationId: 'cancel-1', correlationId: unknownRun.id }; @@ -23,7 +29,7 @@ describe('the machine the workflow host decides runs with', () => { it("is the coordination's functions, runtime and context of each record on the pool's sandbox, so no setting of the server reaches the machine's bound on tasks without waiting", async () => { const runtime = await memoryRuntime(); const pool = programPool({ - handed: await handedWorkersOf(shippedFilesOnDisk), + handed: await handedWorkersOf(sandboxFiles), instances: 'fresh', workers: 1, heapMegabytes: 64, @@ -50,8 +56,8 @@ describe('the cancels the workflow host hands to the runtime of the server', () startVersion: defineStartVersion([echo]), names: playbookOwnNamesOf([echo]), evaluations: { - machine: machineSandboxOf(freshInstances(shippedFilesOnDisk)), - filters: filterSandboxOf(freshInstances(shippedFilesOnDisk)), + machine: machineSandboxOf(freshInstances(sandboxModule)), + filters: filterSandboxOf(freshInstances(sandboxModule)), }, }); diff --git a/packages/tool-servers/README.md b/packages/tool-servers/README.md index 18c966100..0b1a5ef0b 100644 --- a/packages/tool-servers/README.md +++ b/packages/tool-servers/README.md @@ -4,7 +4,7 @@ How a playbook reaches the outside world: the tool servers of its org, MCP serve ## Entry points -- `@on.auto/tool-servers`: the settings, with `ServerEntrySchema`, the schema of one entry, `readEntry`, the reading of one entry of the file, with which `@on.auto/connections` reads a connection's entry, its credential placed, into the connection's `ServerSettings`, and whose context's `placedValues`, the values the runtime places by the name of their placeholder, stand in the entry's header and query values once the entry as written is judged, and whose context's `providers`, a `ProviderResolver`, resolves the provider an entry of the file names (see [An entry that names a provider](#an-entry-that-names-a-provider)), `noProviders` where it is left out; `givenByTheProvider(field)`, the words of a field of a server written beside a provider, and `secretOrKeyExpected`, the words of client credentials given neither a client secret nor a private key with its algorithm; `makeConsentClient`, the runtime acting as a client at the authorization server of a provider's way `authorization`, and the types of its contract, `ConsentClient`, `ClientRegistrations` and `ConnectionTokens`, the store of the tokens of a connection connected by consent, with `KeptRow`, a row as a store reads it, and `HeldTokens`, that row decoded, among them, and `keptTokensIn`, which decodes the value of a row, which `@on.auto/connections` calls and implements, and `ConsentCredential`, the third kind of the credential of `AuthSettings`, the tokens of a consent, which it composes, with `ClientSourceSchema`, `clientSources` and `isSameIssuer`, the client sources an entry may list, in the runtime's order, and the comparison of two issuers, `clientMetadataDocumentOf` and its `ClientMetadataDocument`, the document the runtime names its client by where `PUBLIC_URL` is https, which the server gives `@on.auto/api` to answer, `ClientIdSchema`, the rule of a client id the record and a registration's answer are held to, and `isScope`, the rule of a scope a provider's way and a grant are held to (see [The runtime as a client](#the-runtime-as-a-client)); `serverNameProblems`, the check of a server's name, with which it checks a connection's name, and `servedFieldsOf`, the playbooks, tools, `record_content` and `request_id` an entry is served by, as it writes them or by default where it leaves them out, with which it builds the serving of every connection it lists, and `isPinnableUrl` and `headerNamePattern`; `redactedValuesOf`, the values of a record each redacted under its key, which the reasoning capability takes for the headers and the query parameters of a gateway; `@on.auto/providers` takes the server part of a provider's entry from the fields of `ServerEntrySchema`, as a connection's entry does, and checks a provider's name with `serverNameProblems` and its addresses with `isPinnableUrl`; `namesEveryTool`, whether a tool reference names every tool of its server with `*`, as a reasoning function's `tools` may; the contract of a source of connections, `ConnectionSource` and its types, which `@on.auto/connections` implements, and `noConnections`; `describedToolServers`, `makeToolAccess`, the five operations, `defineSearch` and its org twin `defineSearchInOrg`, with `ProviderOffering` and `OfferedProvider`, the contract of the providers offered to a caller, which `@on.auto/connections` implements, `defineIntrospect`, `defineValidate`, `defineDryRun` and `defineExecute`, `handCallPresenter`, which shows a call made by hand, and the facts of a call that `@on.auto/definitions` builds the run's events from. The access loads the MCP client, its OAuth providers and `node:child_process` (`src/access/linked-access.ts`) through a dynamic import when a run first opens its tools, a search or a read first asks a server or `execute` first calls a tool, so a server that never does any of them never loads them: the server's start loaded 55 files and 0.94 MiB more with them (measured 2026-10-06 with a module load hook, 1,241 files against main's 1,186). Without them it loads 84 files and 236 KiB of this package, the five operations, the reading of the settings of the three types and of an entry that names a provider, `describedToolServers` and the naming of the servers of an org and of its connections among them (measured 2026-10-20 with a module load hook over both entries, the sizes of the files as they are on disk, against 82 files and 219 KiB before the switch of Auto's consent service and the listing of its clients, `src/consent-through-auto/auto-clients.ts`, and the contract they call, `src/consent/consent-service.ts`, came to load with the consent client, against 82 files and 217 KiB before the metadata of the runtime's client, which its document and a dynamic registration share, was written in `src/consent/client-sources.ts`, against 81 files and 211 KiB before the tokens of a consent were served, which added `src/consent-tokens/kept-tokens.ts`, the value a token source of a consent reads, and against 77 files and 192 KiB before an entry could name a provider); the reading resolves a provider through the resolver its context gives, so it loads no code of a provider's; the reading of an OpenAPI document, its decoding and the tools derived from it (`src/openapi`, `described-servers.ts` and the types aside) load through a dynamic import too, when an entry of type `openapi` is described at start. The server's start loads neither the TypeScript compiler, which the check worker of `@on.auto/definitions` loads with the first check, nor the driver of a store its settings do not name: measured on 2026-10-23 on one machine of 16 cores, with Node 26.10.0, by the same hook on the thread that serves requests, from the start of the process to its ready line, at the median of 11 starts interleaved with main's, it loaded on SQLite 2,504 files and 21.0 MiB in 802 ms, against main's 2,573 files and 30.4 MiB in 1,143 ms, at a load average of 20 to 13, and on PostgreSQL 2,551 files and 19.5 MiB in 810 ms, against 2,573 files and 30.4 MiB in 1,134 ms, at 11 to 7. The compiler was 12 files, 8.8 MiB and 318 to 328 ms of it, the warming of the check worker among them, which the first save of a program now pays: 245 ms against 17 ms at the median of 9 servers interleaved with main's, and 6.7 ms against 5.9 ms for the second, at a load average of about 5. The driver of the other store, `pg` with the PostgreSQL entry on SQLite and `sqlite3` with its native addon on PostgreSQL, was 57 files and 0.6 MiB, and 10 files and 2.1 MiB, with a time, 13 and 6 ms, within the spread of the starts. Nor does it load the SDK of the workflow specification and the validator of its schema, which the coordination capability imports with the first document it reads, the S3 client, which the server imports when its settings name a bucket, `google-auth-library` with the AI SDK's Vertex providers, which model access imports when Vertex is configured, or the credential chain of AWS, which the first client that asks for it imports: measured on 2026-10-23 on the same machine, with Node 26.10.0, by the same hook, at the median of 11 starts on SQLite interleaved with main's, at a load average of 5 to 4, it loaded 2,307 files and 18.5 MiB in 654 ms, against main's 2,636 files and 21.4 MiB in 821 ms. The SDK and the validator were 222 files, 1.4 MiB and 130 ms of it, the SDK's 171 validators and Ajv among them, which the first save of a workflow now pays, 175 ms against 34 ms at the median of 9 servers; the S3 client was 26 files, 0.7 MiB and 13 ms, within the spread of the starts, and the Google library and the credential chain 81 files, 0.9 MiB and 24 ms. A CPU profile of 7 starts of each, sampled every 100 µs, gave the SDK with Ajv 115 ms of main's start, the validator 14 ms, the S3 client 11 ms, the Google library 10 ms and the credential chain 6 ms. Nor does the start compose what a server builds only for a request of a kind, which the server's capability registry composes on first use (see [the server](../server/README.md#what-is-composed-at-start-and-what-on-first-use)): the model access of reasoning functions, with a client for each provider and the credential libraries of Vertex, Bedrock and Azure without a key, and the access to the tool servers, with the document of each `openapi` entry read and described; and the prediction caches check the files they hold when a prediction first reads them. Measured on 2026-10-23 on the same machine, with Node 26.10.0, by the same hook, at the median of 11 starts on SQLite interleaved with main's: with nothing configured, 2,312 files and 18.5 MiB in 727 ms, against main's 2,310 files and 18.5 MiB in 727 ms, the two files more being the registry's, at a load average of 16 to 13; with OpenAI, Anthropic, Vertex, Bedrock, Azure without a key and two `openapi` entries over two of the documents the providers ship, of 1.4 MiB and 484 KiB, served on loopback, 2,312 files and 18.5 MiB in 732 ms, against 2,757 files and 20.8 MiB in 833 ms, at 13 to 12; and with 320 MiB of model files and rows in the folder of the caches, 723 ms against 871 ms, at 12 to 10. The first request of each kind pays its composition instead, at the median of 9 servers of that second setting interleaved with main's, at a load average of 10 to 9: the first `list_models` took 65.6 ms against 1.0 ms and the first `search` 51.0 ms against 17.8 ms, the second 2.0 ms against 0.8 ms and 1.6 ms against 2.4 ms, converging by the eighth, and the first listing of the playbooks, which composes nothing, 1.8 ms against 2.1 ms. Readying both caches over those 320 MiB, which the first prediction to read them pays, took 146 ms at the median of 9, and 0.04 ms once ready; one ask of the registry for what it composed takes about 10 µs. +- `@on.auto/tool-servers`: the settings, with `ServerEntrySchema`, the schema of one entry, `readEntry`, the reading of one entry of the file, with which `@on.auto/connections` reads a connection's entry, its credential placed, into the connection's `ServerSettings`, and whose context's `placedValues`, the values the runtime places by the name of their placeholder, stand in the entry's header and query values once the entry as written is judged, and whose context's `providers`, a `ProviderResolver`, resolves the provider an entry of the file names (see [An entry that names a provider](#an-entry-that-names-a-provider)), `noProviders` where it is left out; `givenByTheProvider(field)`, the words of a field of a server written beside a provider, and `secretOrKeyExpected`, the words of client credentials given neither a client secret nor a private key with its algorithm; `makeConsentClient`, the runtime acting as a client at the authorization server of a provider's way `authorization`, and the types of its contract, `ConsentClient`, `ClientRegistrations` and `ConnectionTokens`, the store of the tokens of a connection connected by consent, with `KeptRow`, a row as a store reads it, and `HeldTokens`, that row decoded, among them, and `keptTokensIn`, which decodes the value of a row, which `@on.auto/connections` calls and implements, and `ConsentCredential`, the third kind of the credential of `AuthSettings`, the tokens of a consent, which it composes, with `ClientSourceSchema`, `clientSources` and `isSameIssuer`, the client sources an entry may list, in the runtime's order, and the comparison of two issuers, `clientMetadataDocumentOf` and its `ClientMetadataDocument`, the document the runtime names its client by where `PUBLIC_URL` is https, which the server gives `@on.auto/api` to answer, `ClientIdSchema`, the rule of a client id the record and a registration's answer are held to, and `isScope`, the rule of a scope a provider's way and a grant are held to (see [The runtime as a client](#the-runtime-as-a-client)); `serverNameProblems`, the check of a server's name, with which it checks a connection's name, and `servedFieldsOf`, the playbooks, tools, `record_content` and `request_id` an entry is served by, as it writes them or by default where it leaves them out, with which it builds the serving of every connection it lists, and `isPinnableUrl` and `headerNamePattern`; `redactedValuesOf`, the values of a record each redacted under its key, which the reasoning capability takes for the headers and the query parameters of a gateway; `@on.auto/providers` takes the server part of a provider's entry from the fields of `ServerEntrySchema`, as a connection's entry does, and checks a provider's name with `serverNameProblems` and its addresses with `isPinnableUrl`; `namesEveryTool`, whether a tool reference names every tool of its server with `*`, as a reasoning function's `tools` may; the contract of a source of connections, `ConnectionSource` and its types, which `@on.auto/connections` implements, and `noConnections`; `describedToolServers`, `makeToolAccess`, the five operations, `defineSearch` and its org twin `defineSearchInOrg`, with `ProviderOffering` and `OfferedProvider`, the contract of the providers offered to a caller, which `@on.auto/connections` implements, `defineIntrospect`, `defineValidate`, `defineDryRun` and `defineExecute`, `handCallPresenter`, which shows a call made by hand, and the facts of a call that `@on.auto/definitions` builds the run's events from. The access loads the MCP client and its OAuth providers (`src/access/linked-access.ts`) through a dynamic import when a run first opens its tools, a search or a read first asks a server or `execute` first calls a tool, so a server that never does any of them never loads them: the server's start loaded 55 files and 0.94 MiB more with them (measured 2026-10-06 with a module load hook, 1,241 files against main's 1,186). The transport of a `stdio` server, the one module of the package that imports `node:child_process` (`src/clients/stdio-transport.ts`), loads through a dynamic import of its own when the access first opens a `stdio` server, so an access whose settings name none never loads child processes (`src/access/tool-access-loading.test.ts`). Without them it loads 84 files and 236 KiB of this package, the five operations, the reading of the settings of the three types and of an entry that names a provider, `describedToolServers` and the naming of the servers of an org and of its connections among them (measured 2026-10-20 with a module load hook over both entries, the sizes of the files as they are on disk, against 82 files and 219 KiB before the switch of Auto's consent service and the listing of its clients, `src/consent-through-auto/auto-clients.ts`, and the contract they call, `src/consent/consent-service.ts`, came to load with the consent client, against 82 files and 217 KiB before the metadata of the runtime's client, which its document and a dynamic registration share, was written in `src/consent/client-sources.ts`, against 81 files and 211 KiB before the tokens of a consent were served, which added `src/consent-tokens/kept-tokens.ts`, the value a token source of a consent reads, and against 77 files and 192 KiB before an entry could name a provider); the reading resolves a provider through the resolver its context gives, so it loads no code of a provider's; the reading of an OpenAPI document, its decoding and the tools derived from it (`src/openapi`, `described-servers.ts` and the types aside) load through a dynamic import too, when an entry of type `openapi` is described at start. The server's start loads neither the TypeScript compiler, which the check worker of `@on.auto/definitions` loads with the first check, nor the driver of a store its settings do not name: measured on 2026-10-23 on one machine of 16 cores, with Node 26.10.0, by the same hook on the thread that serves requests, from the start of the process to its ready line, at the median of 11 starts interleaved with main's, it loaded on SQLite 2,504 files and 21.0 MiB in 802 ms, against main's 2,573 files and 30.4 MiB in 1,143 ms, at a load average of 20 to 13, and on PostgreSQL 2,551 files and 19.5 MiB in 810 ms, against 2,573 files and 30.4 MiB in 1,134 ms, at 11 to 7. The compiler was 12 files, 8.8 MiB and 318 to 328 ms of it, the warming of the check worker among them, which the first save of a program now pays: 245 ms against 17 ms at the median of 9 servers interleaved with main's, and 6.7 ms against 5.9 ms for the second, at a load average of about 5. The driver of the other store, `pg` with the PostgreSQL entry on SQLite and `sqlite3` with its native addon on PostgreSQL, was 57 files and 0.6 MiB, and 10 files and 2.1 MiB, with a time, 13 and 6 ms, within the spread of the starts. Nor does it load the SDK of the workflow specification and the validator of its schema, which the coordination capability imports with the first document it reads, the S3 client, which the server imports when its settings name a bucket, `google-auth-library` with the AI SDK's Vertex providers, which model access imports when Vertex is configured, or the credential chain of AWS, which the first client that asks for it imports: measured on 2026-10-23 on the same machine, with Node 26.10.0, by the same hook, at the median of 11 starts on SQLite interleaved with main's, at a load average of 5 to 4, it loaded 2,307 files and 18.5 MiB in 654 ms, against main's 2,636 files and 21.4 MiB in 821 ms. The SDK and the validator were 222 files, 1.4 MiB and 130 ms of it, the SDK's 171 validators and Ajv among them, which the first save of a workflow now pays, 175 ms against 34 ms at the median of 9 servers; the S3 client was 26 files, 0.7 MiB and 13 ms, within the spread of the starts, and the Google library and the credential chain 81 files, 0.9 MiB and 24 ms. A CPU profile of 7 starts of each, sampled every 100 µs, gave the SDK with Ajv 115 ms of main's start, the validator 14 ms, the S3 client 11 ms, the Google library 10 ms and the credential chain 6 ms. Nor does the start compose what a server builds only for a request of a kind, which the server's capability registry composes on first use (see [the server](../server/README.md#what-is-composed-at-start-and-what-on-first-use)): the model access of reasoning functions, with a client for each provider and the credential libraries of Vertex, Bedrock and Azure without a key, and the access to the tool servers, with the document of each `openapi` entry read and described; and the prediction caches check the files they hold when a prediction first reads them. Measured on 2026-10-23 on the same machine, with Node 26.10.0, by the same hook, at the median of 11 starts on SQLite interleaved with main's: with nothing configured, 2,312 files and 18.5 MiB in 727 ms, against main's 2,310 files and 18.5 MiB in 727 ms, the two files more being the registry's, at a load average of 16 to 13; with OpenAI, Anthropic, Vertex, Bedrock, Azure without a key and two `openapi` entries over two of the documents the providers ship, of 1.4 MiB and 484 KiB, served on loopback, 2,312 files and 18.5 MiB in 732 ms, against 2,757 files and 20.8 MiB in 833 ms, at 13 to 12; and with 320 MiB of model files and rows in the folder of the caches, 723 ms against 871 ms, at 12 to 10. The first request of each kind pays its composition instead, at the median of 9 servers of that second setting interleaved with main's, at a load average of 10 to 9: the first `list_models` took 65.6 ms against 1.0 ms and the first `search` 51.0 ms against 17.8 ms, the second 2.0 ms against 0.8 ms and 1.6 ms against 2.4 ms, converging by the eighth, and the first listing of the playbooks, which composes nothing, 1.8 ms against 2.1 ms. Readying both caches over those 320 MiB, which the first prediction to read them pays, took 146 ms at the median of 9, and 0.04 ms once ready; one ask of the registry for what it composed takes about 10 µs. - `@on.auto/tool-servers/policy`: the pure helpers a caller needs without connecting to anything (`toolReferenceOf`, `toolReferenceShape`, `writtenOf`, `namesEveryTool`, `runBoundMs` and `runIdKey`). It loads no transport code, so the reasoning function adapter, which parses a function's `tools` and bounds its run, does not load the MCP client wherever it is bundled; it takes `ToolAccess` as a type only. - `@on.auto/tool-servers/testing`: the fake server and the helpers of the tests (see Testing). diff --git a/packages/tool-servers/src/access/tool-access-loading.test.ts b/packages/tool-servers/src/access/tool-access-loading.test.ts index 8e6d06833..b58766122 100644 --- a/packages/tool-servers/src/access/tool-access-loading.test.ts +++ b/packages/tool-servers/src/access/tool-access-loading.test.ts @@ -1,3 +1,6 @@ +import { join } from 'node:path'; + +import { modulesReachedFrom } from '@on.auto/operations/testing'; import { afterEach, describe, expect, it } from 'vitest'; import { fakeConnections } from '../testing/fake-connections.ts'; @@ -32,6 +35,15 @@ describe('an access to tool servers', () => { }); }); +describe('the modules an access loads when it is first used', () => { + it('reach child processes through the transport of a stdio server alone, which the access imports when it first opens one', () => { + const source = join(import.meta.dirname, '..'); + + expect(modulesReachedFrom(source, 'access/linked-access.ts').packages).not.toContain('node:child_process'); + expect(modulesReachedFrom(source, 'clients/stdio-client.ts').packages).toContain('node:child_process'); + }); +}); + describe('an access to the tool servers of connections alone', () => { it('may reach outside where people may add connections, and not where nothing serves a playbook', async () => { const { access } = reportingAccess({}, { connections: fakeConnections() }); diff --git a/packages/tool-servers/src/clients/link-openers.ts b/packages/tool-servers/src/clients/link-openers.ts index 77e7e00c1..13068e852 100644 --- a/packages/tool-servers/src/clients/link-openers.ts +++ b/packages/tool-servers/src/clients/link-openers.ts @@ -14,7 +14,7 @@ import { type TraceMeta, } from './mcp-client.ts'; import type { ServerClient } from './server-client.ts'; -import { openStdio } from './stdio-client.ts'; +import type { StdioOpening } from './stdio-client.ts'; import { tokenSource, type ConsentTokenSources, type Discovery, type TokenSource } from './token-source.ts'; export interface LinkOptions { @@ -59,6 +59,14 @@ function tokensOf({ settings, serverUrl, discovery }: Remote, options: Linking): ); } +async function openedStdio( + settings: Extract, + opening: StdioOpening, +): Promise { + const { openStdio } = await import('./stdio-client.ts'); + return openStdio(settings, opening); +} + function tracedOpener(open: () => Promise, traceMeta?: TraceMeta): () => Promise { return traceMeta === undefined ? open : async () => tracedClient(await open(), traceMeta); } @@ -73,7 +81,7 @@ export function openerOf(settings: LinkedServerSettings, linkOptions: LinkOption }; if (settings.type === 'stdio') { return tracedOpener( - () => openStdio(settings, { clientInfo: options.clientInfo, timeoutMs: options.timing.openMs, output }), + () => openedStdio(settings, { clientInfo: options.clientInfo, timeoutMs: options.timing.openMs, output }), options.traceMeta, ); } diff --git a/packages/workflow-engine/README.md b/packages/workflow-engine/README.md index 542ba7d9f..30bed7220 100644 --- a/packages/workflow-engine/README.md +++ b/packages/workflow-engine/README.md @@ -11,7 +11,7 @@ The machine runs in Node, in the server: [`@on.auto/workflow-host`](../workflow- - `@on.auto/workflow-engine/worker`: the answerers a worker module hands its jobs to, `answerOf(request, host)`, which runs a program's request in the instance of the sandbox its `host` gives, with the host's output `check`, and `foldAnswerOf(request, host)`, which folds a page in the instances, with the clock, the place it marks and the view checks of its `host` (see [A page of folds](#a-page-of-folds)), with `progressOf(shared)`, the place of a page's folds in memory shared with the pool, `mostValueDepth`, and `sandboxRemovals`, the one list of what the sandbox takes away, which the prelude is written from and the definitions package generates the lib of its check from (`src/worker.ts`). Neither they nor anything they import reaches the pool, the instances or a Node module: the instance a request runs in comes from the `host` the worker module gives them (`src/engine/portability.test.ts`). - `@on.auto/workflow-engine/job-loop`: `serveJobs({ program, fold, check, checks })`, the loop a worker module of the pool runs, `serveJobsWith(handlersOf)`, the same loop over the handlers a module makes of the files the pool handed its worker, and `handedFileIn(handed)`, which answers a file of those handed and fails for any other (`src/job-loop.ts`, `src/program-pool/job-loop.ts`; see [The workers of the pool](#the-workers-of-the-pool)). It is an entry of its own because it listens on `node:worker_threads`' parent port and takes the instances of the sandbox for each job from the source the pool names, which the main, testing and worker entries never reach; the pool's own two worker modules run it, as do the checked worker of `@on.auto/definitions` and its check worker. - `@on.auto/workflow-engine/process-loop`: `serveProcessJobs(handlers, process)`, the loop a module the pool of processes runs serves its jobs with (`src/process-loop.ts`, `src/process-pool/process-loop.ts`; see [The processes of the pool](#the-processes-of-the-pool)). It is an entry of its own because a process loads it alone: it reaches the loop, the messages and the values of the process pool and the ready signal both pools send, `src/jobs/ready-signal.ts`, and no module of the worker pool or the instances of the sandbox (`src/engine/portability.test.ts`). -- `@on.auto/workflow-engine/dsl`: the sandbox's Node host: its two sources of instances, `freshInstances(shippedFiles)` and `keptInstances(shippedFiles)`, each an `InstanceSource`, `(memoryBytes) => Promise`, which answers an instance of the QuickJS module, compiled once from its file as the `ShippedFiles` of `@on.auto/operations` it is given reads it, with a memory of that maximum (see [The sources of instances](#the-sources-of-instances)), `machineSandboxOf(source, clock)`, the instances a host prepares from its source for the inputs of its workflows on its own thread, `filterSandboxOf(source, clock)`, a context on an instance of its source for each group of filters, on its own thread too, `hostClock` and `sandboxAnswers(source, expressions)`, what the sandbox answers to probes; `handedWorkersOf(shippedFiles, modules)` and `handedFilesOf(shippedFiles, files)`, which read the files a pool hands its workers; the bounds of the sandbox; the format's duration reader, `readDuration`, which an interaction function reads its `expires` with; and a pool of worker threads, kept warm between jobs, that runs a program, a page of folds or a check in a worker of the module the job names, and whose `evaluations` are the sandboxes of `decide` and of a host's filters in the pool's evaluation worker, which the server's workflow host uses (`src/dsl.ts`; see [The evaluation worker](#the-evaluation-worker); see [The sandbox](#the-sandbox), [A page of folds](#a-page-of-folds) and [The workers of the pool](#the-workers-of-the-pool)). The computation capability runs computation functions with it, the recall capability the `answer` of a recall function, the workflow host the folds of the views recall functions keep, and the definitions package its checks at save. It is an entry of its own because the pool imports `node:worker_threads` and arms host timers, and the instances compile the WebAssembly module: the main, testing and worker entries reach neither the pool, the instances nor anything that imports them, and only this entry and the job loop's do (`src/engine/portability.test.ts`). The entry also exports `processPool(settings)`, a pool of child processes for jobs a caller's module serves (see [The processes of the pool](#the-processes-of-the-pool)), which the server composes for the model processes of prediction functions. +- `@on.auto/workflow-engine/dsl`: the sandbox's Node host: its two sources of instances, `freshInstances(sandboxModule)` and `keptInstances(sandboxModule)`, each an `InstanceSource`, `(memoryBytes) => Promise`, which answers an instance of the QuickJS module, compiled once from the bytes of its file or taken as a host compiled it, with a memory of that maximum (see [The sources of instances](#the-sources-of-instances)), and `sandboxModuleIn(shippedFiles)`, the bytes of the module's file as the `ShippedFiles` of `@on.auto/operations` it is given reads them, `machineSandboxOf(source, clock)`, the instances a host prepares from its source for the inputs of its workflows on its own thread, `filterSandboxOf(source, clock)`, a context on an instance of its source for each group of filters, on its own thread too, `hostClock` and `sandboxAnswers(source, expressions)`, what the sandbox answers to probes; `handedWorkersOf(shippedFiles, modules)` and `handedFilesOf(shippedFiles, files)`, which read the files a pool hands its workers; `sandboxShippedRoots`, the root of `@jitl/quickjs-wasmfile-release-sync`, the package the module of the sandbox is shipped in, which the engine resolves with `import.meta.resolve` since it depends on that package, for the reader the server composes; the bounds of the sandbox; the format's duration reader, `readDuration`, which an interaction function reads its `expires` with; and a pool of worker threads, kept warm between jobs, that runs a program, a page of folds or a check in a worker of the module the job names, and whose `evaluations` are the sandboxes of `decide` and of a host's filters in the pool's evaluation worker, which the server's workflow host uses (`src/dsl.ts`; see [The evaluation worker](#the-evaluation-worker); see [The sandbox](#the-sandbox), [A page of folds](#a-page-of-folds) and [The workers of the pool](#the-workers-of-the-pool)). The computation capability runs computation functions with it, the recall capability the `answer` of a recall function, the workflow host the folds of the views recall functions keep, and the definitions package its checks at save. It is an entry of its own because the pool imports `node:worker_threads` and arms host timers, and the instances compile the WebAssembly module: the main, testing and worker entries reach neither the pool, the instances nor anything that imports them, and only this entry and the job loop's do (`src/engine/portability.test.ts`). The entry also exports `processPool(settings)`, a pool of child processes for jobs a caller's module serves (see [The processes of the pool](#the-processes-of-the-pool)), which the server composes for the model processes of prediction functions. ## How a run moves @@ -68,7 +68,7 @@ An expression's budget is checkpoints of the sandbox (below): 250 for one expres ## The sandbox -Every program a person writes inside the playbook, a computation function, a recall function's fold and answer, a workflow's expressions and the `data` of a filter, is TypeScript in the erasable subset, run by QuickJS compiled to WebAssembly: `quickjs-emscripten-core` with the synchronous release variant, `@jitl/quickjs-wasmfile-release-sync`, both pinned at 0.32.0, since the work a program costs is counted by that build. `src/programs` is the one place that knows the sandbox, and it is portable: it imports the library by type alone and is given an instance and a clock. Each host compiles the module once, asynchronously, from the bytes of its file, `sandboxModuleFile`, which a host's own sandboxes read through the `ShippedFiles` they are given and a worker of the pool is handed (see [The workers of the pool](#the-workers-of-the-pool)), and takes the instances of it from a source (`src/instances`; see [The sources of instances](#the-sources-of-instances)): an instance has a `WebAssembly.Memory` whose `maximum` is the unit's memory bound and whose `grow` is wrapped so the host sees a refusal even when the program catches the engine's `out of memory` (`src/instances/compiled-module.ts`). +Every program a person writes inside the playbook, a computation function, a recall function's fold and answer, a workflow's expressions and the `data` of a filter, is TypeScript in the erasable subset, run by QuickJS compiled to WebAssembly: `quickjs-emscripten-core` with the synchronous release variant, `@jitl/quickjs-wasmfile-release-sync`, both pinned at 0.32.0, since the work a program costs is counted by that build. `src/programs` is the one place that knows the sandbox, and it is portable: it imports the library by type alone and is given an instance and a clock. Each host compiles the module once, asynchronously, from the bytes of its file, `sandboxModuleFile`, `dist/emscripten-module.wasm` of `@jitl/quickjs-wasmfile-release-sync`, which a host's own sandboxes read through the `ShippedFiles` they are given and a worker of the pool is handed (see [The workers of the pool](#the-workers-of-the-pool)), and takes the instances of it from a source (`src/instances`; see [The sources of instances](#the-sources-of-instances)): an instance has a `WebAssembly.Memory` whose `maximum` is the unit's memory bound and whose `grow` is wrapped so the host sees a refusal even when the program catches the engine's `out of memory` (`src/instances/compiled-module.ts`). - **An instance of its own for every unit.** Every program job, every page of folds, every input of `decide` and every group of filters takes an instance of its own from its source, which no other unit uses while it runs, and makes a runtime and a context of its own in it, disposed after the unit, so nothing of one unit reaches the next. The memory's maximum is the one memory wall, in bytes: 256 MiB for a run, 64 MiB for a page, a group of filters and an input. A unit holds what the module's own memory of that maximum holds: when it needs more, the glue asks to grow by a fifth more than it needs, then a tenth, then a twentieth, and gives up after the three, so the host marks a refusal only when three attempts in a row fail and not at the first, and a program keeps 58 strings of a MiB under a maximum of 64 MiB, as the module alone does (`src/instances/fresh-instances.test.ts`). The engine's own `setMemoryLimit` refuses one allocation larger than its limit but counts every allocation it makes as 8 bytes, whatever its size, since this build has no `malloc_usable_size`, so it bounds no unit's bytes and is not relied on: the instance's maximum is the wall. - **The prelude** (`src/programs/sandbox-prelude.ts`) makes a context the playbook's: `Date` answers the run's moment with no argument, takes one number, one `Date`, a date alone or a date and time that names its offset, and refuses whatever reads the server's time zone, its original constructor out of reach through `getPrototypeOf`, `Reflect.construct`, `bind` and `class extends`; `Math.random`, `eval`, the `Function` constructors, `WeakRef`, `FinalizationRegistry` and `Promise`, whose callbacks the sandbox never runs, go, with every local-time method of `Date`, from the one list `sandboxRemovals` (`src/programs/sandbox-names.ts`), which also says that `Date` is not called and takes at most one argument, the rule the prelude keeps and the definitions package writes the lib of its check from. An expression runs as strict code, `'use strict'` the first line of the function it is wrapped in, as the check compiles it. Nothing is added: no `console`, no timers, no `fetch`, no `Intl`, no module loader. @@ -86,12 +86,12 @@ A unit answers a `ProgramRun`: `answered` with the text of its JSON, `raised` wi ### The sources of instances -A unit takes its instance from an `InstanceSource`, `(memoryBytes) => Promise` (`src/instances/instance-source.ts`), and gives it back when it closes its runtime, `release(intact)`, intact unless the runtime broke (`sandboxRuntimeOf`). Two sources stand behind the port, each made over a reader of the module's file, `SandboxFiles`, which a `ShippedFiles` is. They share the module compiled once for each reader, the bounded memory and the instance made of the two (`src/instances/compiled-module.ts`), and differ only in how an instance comes to be: +A unit takes its instance from an `InstanceSource`, `(memoryBytes) => Promise` (`src/instances/instance-source.ts`), and gives it back when it closes its runtime, `release(intact)`, intact unless the runtime broke (`sandboxRuntimeOf`). Two sources stand behind the port, each made over the module of the sandbox, a `SandboxModule` (`src/instances/instance-source.ts`): either the bytes of its file to compile, `{ bytes }`, as `sandboxModuleIn` reads them and a worker of the pool is handed them, or a `WebAssembly.Module` compiled already, `{ compiled }`, for a host that gives the module compiled rather than its bytes. They share the module compiled once for each `SandboxModule` of bytes, the bounded memory and the instance made of the two (`src/instances/compiled-module.ts`), and differ only in how an instance comes to be: -- `freshInstances(files)`, named `fresh`, instantiates the module anew for every unit, and lets the instance go once the unit gives it back (`src/instances/fresh-instances.ts`). -- `keptInstances(files)`, named `kept`, keeps an instance a unit gave back intact, one for each memory maximum, and lends it to the next unit of that maximum, which makes a runtime and a context of its own in it; it lets go of an instance whose memory refused to grow or whose runtime broke, lends an instance to one unit at a time, and instantiates the module for a unit that asks while no instance of its maximum waits (`src/instances/kept-instances.ts`). A unit on a kept instance has the whole of its memory, as on a fresh one: the runtime before it freed its heap when it was disposed, and the next unit keeps as many strings of a MiB under the same maximum as the first did. The runtime's own memory limit counts blocks rather than bytes (above), so a kept instance serves one unit at a time rather than runtimes side by side, and its maximum is the unit's wall. +- `freshInstances(sandboxModule)`, named `fresh`, instantiates the module anew for every unit, and lets the instance go once the unit gives it back (`src/instances/fresh-instances.ts`). +- `keptInstances(sandboxModule)`, named `kept`, keeps an instance a unit gave back intact, one for each memory maximum, and lends it to the next unit of that maximum, which makes a runtime and a context of its own in it; it lets go of an instance whose memory refused to grow or whose runtime broke, lends an instance to one unit at a time, and instantiates the module for a unit that asks while no instance of its maximum waits (`src/instances/kept-instances.ts`). A unit on a kept instance has the whole of its memory, as on a fresh one: the runtime before it freed its heap when it was disposed, and the next unit keeps as many strings of a MiB under the same maximum as the first did. The runtime's own memory limit counts blocks rather than bytes (above), so a kept instance serves one unit at a time rather than runtimes side by side, and its maximum is the unit's wall. -Both are held to the same tests. Every test of the engine of how a program, an expression, a fold or a filter runs, of its bounds and of what one unit leaves for the next runs on each source, `sourcesUnderTest` of `src/pool-testing/test-pools.ts`, and the tests of the job loop and the evaluation worker on each name; `src/instances/instance-isolation.test.ts` finds nothing of a value, a global, a prototype changed, a `Symbol` registered or a byte written by one unit in the next, and the next answering after one that broke the stack of its thread, and `src/instances/instance-sources.test.ts` the next keeping the whole of its memory after one that exhausted it. +Both are held to the same tests. Every test of the engine of how a program, an expression, a fold or a filter runs, of its bounds and of what one unit leaves for the next runs on each source, `sourcesUnderTest` of `src/pool-testing/test-pools.ts`, and the tests of the job loop and the evaluation worker on each name; `src/instances/instance-isolation.test.ts` finds nothing of a value, a global, a prototype changed, a `Symbol` registered or a byte written by one unit in the next, and the next answering after one that broke the stack of its thread, and `src/instances/instance-sources.test.ts` the next keeping the whole of its memory after one that exhausted it. `src/instances/fresh-instances.test.ts` runs a program on instances of a module compiled before it was given, as on those of the bytes the source compiles itself. Which source evaluates is the host's decision, not the unit's: `programPool` takes `instances`, the name of the source its workers make theirs of, the evaluation workers among them, `fresh` or `kept`, and the sandboxes of a host's own thread, `machineSandboxOf`, `filterSandboxOf` and `sandboxAnswers`, take the source itself. The server's pool names `fresh` (`packages/server/src/composition/served-computation.ts`), and so do the tests of the capabilities, the workflow host and the server: a fresh instance is the strongest isolation, where nothing of a unit outlives it, not even the memory it grew, and a memory stop never leaves a wounded module behind; a kept instance saves each unit its instantiation, and holds while it waits the memory its units grew it to, up to its maximum. Measured on 2026-10-23 (see [The sandbox, measured](#the-sandbox-measured)), a warm unit of the smallest program took 0.560 ms on a kept instance against 0.780 ms on a fresh one on the thread that measures, a warm run of it through a pool 0.792 ms against 0.870 ms, and the first unit of a source made anew and a unit that spends a run's budget of 20,000 checkpoints the same on both, 0.67 ms and 448 ms. @@ -119,7 +119,7 @@ The pool keeps the workers of each module URL between jobs, every module's worke A worker module runs `serveJobs({ program, fold, check, checks })` of the `job-loop` entry, which listens on the parent port, decodes each envelope with the schema the pool encoded it with, and hands the whole request to the handler of its kind with a host: the clock, an instance of the sandbox for a program and one for each view of a page, from the source the pool names and prepared while the job before ran, and the output check of the request's `context` or the view checks of each view's schema, compiled once for each schema from the `checks` the module gives and kept, by schema, in a cache of at most 262,144 characters (`src/jobs/job-kit.ts`); with no `checks` a program goes unchecked and a view that keeps a schema stalls. A request the loop cannot read is answered `unreadable`, which a page's job reads as such and a program's as a crash, and the worker is not kept. The pool's own `src/workers/program-worker.ts` and `fold-worker.ts` serve `answerOf` and `foldAnswerOf` so; the pool names them for a request that names no worker, as the engine's own tests do, while the capabilities name the checked worker of `@on.auto/definitions` for every run. -The pool hands every worker it starts, in its `workerData`, the files its module reads, from the `handed` of its settings: the files `handed.modules` names for the worker's module, or `handed.sandbox`, the module of the sandbox, for a module it does not name, so a worker reads no file itself, and the name of the source of instances its settings give, `instances`, which a worker makes its source of over the module of the sandbox it was handed (`handedSandboxOf`), since a thread can be handed no function. A thread cannot be handed a reader, so the files are read before the pool is made, once, through the `ShippedFiles` of `@on.auto/operations`: `handedWorkersOf(shippedFiles, modules)` reads the module of the sandbox and `handedFilesOf(shippedFiles, files)` the files of a module, and the server reads them where it composes its one pool, with the files of the check worker of `@on.auto/definitions` (`handedWorkersWithChecks`). `programPool` therefore stays synchronous, and every worker starts with its files, the evaluation worker of a unit asked before any `reserve` among them, which answers once it is ready (`src/program-pool/evaluation-thread.test.ts`), as does a worker the pool starts again while the thread that decides blocks on it. A worker compiles the sandbox from what it was handed, and a module that makes its handlers of handed files, as the check worker makes its checker of the lib it compiles against, serves them with `serveJobsWith(handlersOf)`, whose `handed(file)` answers the bytes of a file handed to the worker and fails for any other (`src/program-pool/handed-files.ts`, `src/program-pool/pool-checks.test.ts`). +The pool hands every worker it starts, in its `workerData`, the files its module reads, from the `handed` of its settings: the files `handed.modules` names for the worker's module, or `handed.sandbox`, the module of the sandbox, for a module it does not name, so a worker reads no file itself, and the name of the source of instances its settings give, `instances`, which a worker makes its source of over the bytes of the module of the sandbox it was handed, read when its first unit asks for an instance, so a worker handed no module of the sandbox, as the check worker, fails only if a unit asks it for one (`handedSandboxOf`), since a thread can be handed no function. A thread cannot be handed a reader, so the files are read before the pool is made, once, through the `ShippedFiles` of `@on.auto/operations`: `handedWorkersOf(shippedFiles, modules)` reads the module of the sandbox and `handedFilesOf(shippedFiles, files)` the files of a module, and the server reads them where it composes its one pool, with the files of the check worker of `@on.auto/definitions` (`handedWorkersWithChecks`). `programPool` therefore stays synchronous, and every worker starts with its files, the evaluation worker of a unit asked before any `reserve` among them, which answers once it is ready (`src/program-pool/evaluation-thread.test.ts`), as does a worker the pool starts again while the thread that decides blocks on it. A worker compiles the sandbox from what it was handed, and a module that makes its handlers of handed files, as the check worker makes its checker of the lib it compiles against, serves them with `serveJobsWith(handlersOf)`, whose `handed(file)` answers the bytes of a file handed to the worker, by the `ShippedFile` it was read as and handed under its name, and fails for any other (`src/program-pool/handed-files.ts`, `src/program-pool/pool-checks.test.ts`). The answer says whether the worker may be kept, by the rule of its kind: a program keeps it unless its evaluator ended it at the deadline, since the thread was busy past the job's deadline; a page keeps it unless a view ran past its fold's deadline, `overtime`, or the page was `unreadable`. The pool lets a worker go, terminating it before the job's permit is released: diff --git a/packages/workflow-engine/measure/evaluations.ts b/packages/workflow-engine/measure/evaluations.ts index b1b6a1d9f..17e475d5b 100644 --- a/packages/workflow-engine/measure/evaluations.ts +++ b/packages/workflow-engine/measure/evaluations.ts @@ -1,7 +1,14 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Effect } from 'effect'; -import { freshInstances, handedWorkersOf, machineSandboxOf, programPool } from '../src/dsl.ts'; +import { + freshInstances, + handedWorkersOf, + machineSandboxOf, + programPool, + sandboxModuleIn, + sandboxShippedRoots, +} from '../src/dsl.ts'; import { mostFilterMs } from '../src/dsl/evaluation.ts'; import { filterVerdictsOf } from '../src/filters/filter-verdicts.ts'; import { decidedOnce } from '../src/pool-testing/decided-once.ts'; @@ -9,6 +16,8 @@ import { answerGraceMs } from '../src/program-pool/evaluation-thread.ts'; import type { MachineSandbox } from '../src/programs/reserved-instances.ts'; import { medianMillisecondsOf, startedAt } from './common.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + const decision = { decision: { decision: 'approve', reason: 'the budget is stated' } }; const closed = { type: 'com.acme.closed' }; @@ -64,7 +73,7 @@ function evaluationMs(sandbox: MachineSandbox): number { export async function evaluationsMeasured(): Promise { const pool = programPool({ - handed: await handedWorkersOf(shippedFilesOnDisk), + handed: await handedWorkersOf(sandboxFiles), instances: 'fresh', workers: 1, heapMegabytes: 256, @@ -72,7 +81,7 @@ export async function evaluationsMeasured(): Promise { const { machine, filters } = pool.evaluations; const coldMs = await timed(() => Effect.runPromise(machine.reserve)); const inWorker = evaluationMs(machine); - const here = machineSandboxOf(freshInstances(shippedFilesOnDisk)); + const here = machineSandboxOf(freshInstances(await sandboxModuleIn(sandboxFiles))); await Effect.runPromise(here.reserve); const inThread = evaluationMs(here); const decideMs = await timed(() => decidedOnce(machine, churning(4_194_304), startedAt)); diff --git a/packages/workflow-engine/measure/pages.ts b/packages/workflow-engine/measure/pages.ts index 77005667e..a6c448dc7 100644 --- a/packages/workflow-engine/measure/pages.ts +++ b/packages/workflow-engine/measure/pages.ts @@ -10,10 +10,16 @@ import { type FoldRequest, type JsonObject, type ProgramPool, + sandboxModuleIn, + sandboxShippedRoots, } from '../src/dsl.ts'; import { foldPage } from '../src/folds/fold-page.ts'; import { millisecondsOf } from './common.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const sandboxModule = await sandboxModuleIn(sandboxFiles); + const reviewsFold = [ 'function fieldOf(value, name) {', " return typeof value === 'object' && value !== null && !Array.isArray(value) ? Reflect.get(value, name) : undefined;", @@ -103,7 +109,7 @@ function inTurn(items: readonly A[], step: (item: A) => Promise): Promi async function onAFreshPool(page: FoldRequest): Promise { const pool = programPool({ - handed: await handedWorkersOf(shippedFilesOnDisk), + handed: await handedWorkersOf(sandboxFiles), instances: 'fresh', workers: 1, heapMegabytes: 256, @@ -117,7 +123,7 @@ async function foldedOnThisThread( page: FoldRequest, ): Promise<{ readonly milliseconds: number; readonly views: string }> { const folded = { views: '' }; - const instances = await Promise.all(page.views.map(() => freshInstances(shippedFilesOnDisk)(unitMemoryBytes))); + const instances = await Promise.all(page.views.map(() => freshInstances(sandboxModule)(unitMemoryBytes))); const host = { now: () => performance.now(), folding: Function.constVoid, @@ -161,7 +167,7 @@ export async function pagesMeasured(): Promise { const page = pageOf(events); const start = await inTurn(Array.from({ length: turns }), () => onAFreshPool(pageOf(0))); const pool = programPool({ - handed: await handedWorkersOf(shippedFilesOnDisk), + handed: await handedWorkersOf(sandboxFiles), instances: 'fresh', workers: 1, heapMegabytes: 256, diff --git a/packages/workflow-engine/measure/sandbox.ts b/packages/workflow-engine/measure/sandbox.ts index 66bd94956..5eccea5f4 100644 --- a/packages/workflow-engine/measure/sandbox.ts +++ b/packages/workflow-engine/measure/sandbox.ts @@ -8,6 +8,8 @@ import { threadStackBytes, unitMemoryBytes, type ProgramRun, + sandboxModuleIn, + sandboxShippedRoots, } from '../src/dsl.ts'; import { sandboxModuleFile } from '../src/instances/compiled-module.ts'; import { expressionUnitOf } from '../src/programs/expression-units.ts'; @@ -17,6 +19,10 @@ import type { Evaluation } from '../src/programs/program-run.ts'; import { sandboxRuntimeOf } from '../src/programs/sandbox-session.ts'; import { medianMillisecondsOf, millisecondsOf } from './common.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const sandboxModule = await sandboxModuleIn(sandboxFiles); + const unbounded: Evaluation = { budget: Number.POSITIVE_INFINITY, deadlineAt: Number.POSITIVE_INFINITY, moment: 0 }; const settings = { stackBytes: threadStackBytes, mostAnswerBytes: 16_777_216, clock: () => performance.now() }; @@ -52,7 +58,7 @@ async function ranModule( args: readonly string[], evaluation: Evaluation = unbounded, ): Promise { - const run = moduleRun(await freshInstances(shippedFilesOnDisk)(unitMemoryBytes), settings, { + const run = moduleRun(await freshInstances(sandboxModule)(unitMemoryBytes), settings, { source, entry: 'default', arguments: args, @@ -65,16 +71,16 @@ async function ranModule( } async function instanceLines(): Promise { - const wasm = await shippedFilesOnDisk.bytes(sandboxModuleFile); - await freshInstances(shippedFilesOnDisk)(unitMemoryBytes); + const wasm = await sandboxFiles.bytes(sandboxModuleFile); + await freshInstances(sandboxModule)(unitMemoryBytes); const fresh = await medianAsync(50, async () => { const started = performance.now(); - await freshInstances(shippedFilesOnDisk)(unitMemoryBytes); + await freshInstances(sandboxModule)(unitMemoryBytes); return performance.now() - started; }); const withPrelude = await medianAsync(50, async () => { const started = performance.now(); - const runtime = sandboxRuntimeOf(await freshInstances(shippedFilesOnDisk)(runMemoryBytes), settings); + const runtime = sandboxRuntimeOf(await freshInstances(sandboxModule)(runMemoryBytes), settings); runtime.context().close(); const milliseconds = performance.now() - started; runtime.close(); @@ -115,13 +121,13 @@ async function checkpointLines(): Promise { } async function expressionLines(): Promise { - const instance = await freshInstances(shippedFilesOnDisk)(unitMemoryBytes); + const instance = await freshInstances(sandboxModule)(unitMemoryBytes); const unit = expressionUnitOf(() => instance, settings); const fresh = medianMillisecondsOf(201, () => { unit.evaluate('$context.decision.decision == "approve"', { context: decision }, unbounded); }); unit.close(); - const filters = filterContextOf(await freshInstances(shippedFilesOnDisk)(unitMemoryBytes), settings, unbounded); + const filters = filterContextOf(await freshInstances(sandboxModule)(unitMemoryBytes), settings, unbounded); const test = filters.define(' $data.decision.decision == "approve" '); const freezing = millisecondsOf(() => { filters.freeze(); @@ -170,7 +176,7 @@ async function valueLines(): Promise { async function stackLines(): Promise { const pool = programPool({ - handed: await handedWorkersOf(shippedFilesOnDisk), + handed: await handedWorkersOf(sandboxFiles), instances: 'fresh', workers: 1, heapMegabytes: 256, diff --git a/packages/workflow-engine/measure/sources.ts b/packages/workflow-engine/measure/sources.ts index 50a229919..5cf4df017 100644 --- a/packages/workflow-engine/measure/sources.ts +++ b/packages/workflow-engine/measure/sources.ts @@ -10,12 +10,18 @@ import { type InstanceSource, type InstanceSourceName, type ProgramPool, + sandboxModuleIn, + sandboxShippedRoots, } from '../src/dsl.ts'; import { instanceSources } from '../src/instances/instance-sources.ts'; import { moduleRun } from '../src/programs/module-runs.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + type Timings = Readonly>; +const sandboxModule = await sandboxModuleIn(sandboxFiles); + const answering = 'export default function () {\n return 0;\n}'; const spending = 'export default function () {\n for (;;) {}\n}'; @@ -62,7 +68,7 @@ async function alternately(times: number, sample: (name: InstanceSourceName) => } function sourceOf(name: InstanceSourceName): InstanceSource { - return instanceSources[name](shippedFilesOnDisk); + return instanceSources[name](sandboxModule); } async function onThread(): Promise>> { @@ -77,7 +83,7 @@ async function onThread(): Promise { return programPool({ - handed: await handedWorkersOf(shippedFilesOnDisk), + handed: await handedWorkersOf(sandboxFiles), instances: name, workers: 1, heapMegabytes: 256, diff --git a/packages/workflow-engine/src/dsl.ts b/packages/workflow-engine/src/dsl.ts index b13b08aca..644f425e8 100644 --- a/packages/workflow-engine/src/dsl.ts +++ b/packages/workflow-engine/src/dsl.ts @@ -52,8 +52,9 @@ export type { ViewCheck, } from './folds/fold-page.ts'; export type { FoldPlace } from './folds/fold-progress.ts'; +export { sandboxModuleIn, sandboxShippedRoots } from './instances/compiled-module.ts'; export { freshInstances } from './instances/fresh-instances.ts'; -export type { InstanceSource } from './instances/instance-source.ts'; +export type { InstanceSource, SandboxModule } from './instances/instance-source.ts'; export type { InstanceSourceName } from './instances/instance-sources.ts'; export { keptInstances } from './instances/kept-instances.ts'; export { filterSandboxOf, hostClock, machineSandboxOf } from './instances/host-sandboxes.ts'; diff --git a/packages/workflow-engine/src/engine/portability.test.ts b/packages/workflow-engine/src/engine/portability.test.ts index 52c3ec977..4ef0aba8f 100644 --- a/packages/workflow-engine/src/engine/portability.test.ts +++ b/packages/workflow-engine/src/engine/portability.test.ts @@ -18,7 +18,7 @@ const importedEverywhere: ReadonlySet = new Set(['effect', '@on.auto/ope const importedInAFolder: ReadonlyMap = new Map([ ['programs/', ['quickjs-emscripten-core']], - ['program-pool/', ['node:worker_threads']], + ['program-pool/', ['node:url', 'node:worker_threads']], ['process-pool/', ['node:buffer', 'node:child_process', 'node:fs', 'node:path', 'node:url']], ['instances/', ['quickjs-emscripten-core', '@jitl/quickjs-wasmfile-release-sync']], ]); diff --git a/packages/workflow-engine/src/instances/compiled-module.ts b/packages/workflow-engine/src/instances/compiled-module.ts index e34e69cd3..00bc45595 100644 --- a/packages/workflow-engine/src/instances/compiled-module.ts +++ b/packages/workflow-engine/src/instances/compiled-module.ts @@ -1,14 +1,18 @@ import * as releaseSync from '@jitl/quickjs-wasmfile-release-sync'; +import type { ShippedFile, ShippedFiles, ShippedRoots } from '@on.auto/operations'; import { Schema } from 'effect'; import { newQuickJSWASMModuleFromVariant, newVariant, type QuickJSSyncVariant } from 'quickjs-emscripten-core'; import type { SandboxInstance } from '../programs/sandbox-session.ts'; -import type { SandboxFiles } from './instance-source.ts'; +import type { SandboxModule } from './instance-source.ts'; export type Instantiated = Omit; interface GrowingMemory { - readonly grow: (pages: number) => number; + readonly buffer: ArrayBuffer; + grow(pages: number): number; + toFixedLengthBuffer(): ArrayBuffer; + toResizableBuffer(): ArrayBuffer; } interface MemoryLimits { @@ -27,7 +31,13 @@ interface Bounded { export const pageBytes = 65_536; -export const sandboxModuleFile = new URL(import.meta.resolve('@jitl/quickjs-wasmfile-release-sync/wasm')); +const sandboxPackage = '@jitl/quickjs-wasmfile-release-sync'; + +export const sandboxModuleFile: ShippedFile = { package: sandboxPackage, path: 'dist/emscripten-module.wasm' }; + +export const sandboxShippedRoots: ShippedRoots = { + [sandboxPackage]: new URL('.', import.meta.resolve(`${sandboxPackage}/package.json`)), +}; const initialPages = 256; @@ -44,7 +54,7 @@ const variant = Schema.decodeUnknownSync( Schema.declare((value: unknown): value is QuickJSSyncVariant => Reflect.get(new Object(value), 'type') === 'sync'), )(releaseSync.default); -const compiledThrough = new WeakMap>(); +const compiledOnce = new WeakMap>(); function boundedMemory(memoryBytes: number): Bounded { const growth = { failedInARow: 0, refused: false }; @@ -66,15 +76,22 @@ function boundedMemory(memoryBytes: number): Bounded { return { memory, refusedGrowth: () => growth.refused }; } -function compiledFrom(files: SandboxFiles): Promise { - const compiled = compiledThrough.get(files) ?? files.bytes(sandboxModuleFile).then(webAssembly.compile); - compiledThrough.set(files, compiled); +export async function sandboxModuleIn(shippedFiles: ShippedFiles): Promise { + return { bytes: await shippedFiles.bytes(sandboxModuleFile) }; +} + +function compiledFrom(sandboxModule: SandboxModule): Promise { + if ('compiled' in sandboxModule) { + return Promise.resolve(sandboxModule.compiled); + } + const compiled = compiledOnce.get(sandboxModule) ?? webAssembly.compile(sandboxModule.bytes); + compiledOnce.set(sandboxModule, compiled); return compiled; } -export async function instantiated(files: SandboxFiles, memoryBytes: number): Promise { +export async function instantiated(sandboxModule: SandboxModule, memoryBytes: number): Promise { const { memory, refusedGrowth } = boundedMemory(memoryBytes); - const wasmModule = await compiledFrom(files); + const wasmModule = await compiledFrom(sandboxModule); const module = await newQuickJSWASMModuleFromVariant(newVariant(variant, { wasmModule, wasmMemory: memory })); return { module, memoryBytes, refusedGrowth }; } diff --git a/packages/workflow-engine/src/instances/fresh-instances.test.ts b/packages/workflow-engine/src/instances/fresh-instances.test.ts index 78980dded..e869d7fcc 100644 --- a/packages/workflow-engine/src/instances/fresh-instances.test.ts +++ b/packages/workflow-engine/src/instances/fresh-instances.test.ts @@ -1,16 +1,51 @@ -import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; +import { Schema } from 'effect'; import { describe, expect, it } from 'vitest'; -import { unitMemoryBytes } from '../programs/sandbox-bounds.ts'; +import { sandboxFilesOnDisk, sandboxModuleOnDisk } from '../pool-testing/test-pools.ts'; +import { moduleRun } from '../programs/module-runs.ts'; +import { threadStackBytes, unitMemoryBytes } from '../programs/sandbox-bounds.ts'; +import type { SandboxInstance } from '../programs/sandbox-session.ts'; +import { sandboxModuleFile } from './compiled-module.ts'; import { freshInstances } from './fresh-instances.ts'; +type Compile = (bytes: Readonly) => Promise; + +const { compile } = Schema.decodeUnknownSync( + Schema.Struct({ compile: Schema.declare((value: unknown): value is Compile => typeof value === 'function') }), +)(Reflect.get(globalThis, 'WebAssembly')); + +const settings = { stackBytes: threadStackBytes, mostAnswerBytes: 1024, clock: () => 0 }; + +const evaluation = { budget: 1_000_000, deadlineAt: Number.POSITIVE_INFINITY, moment: 0 }; + +function answerOn(instance: SandboxInstance): unknown { + return moduleRun(instance, settings, { + source: 'export default function () {\n return 6 * 7;\n}', + entry: 'default', + arguments: [], + evaluation, + }); +} + describe('the fresh source of instances', () => { it('makes a fresh instance for every unit, from the one module it compiled, even after a unit gave one back', async () => { - const source = freshInstances(shippedFilesOnDisk); + const source = freshInstances(sandboxModuleOnDisk); const [first, second] = await Promise.all([source(unitMemoryBytes), source(unitMemoryBytes)]); first.release(true); const third = await source(unitMemoryBytes); expect(new Set([first.module, second.module, third.module]).size).toBe(3); }); + + it('makes its instances of a module the host compiled already, as of the bytes it compiles itself', async () => { + const compiled = await compile(await sandboxFilesOnDisk.bytes(sandboxModuleFile)); + const source = freshInstances({ compiled }); + const [first, second] = [await source(unitMemoryBytes), await source(unitMemoryBytes)]; + + expect([answerOn(first), answerOn(second)]).toMatchObject([ + { ran: 'answered', text: '42' }, + { ran: 'answered', text: '42' }, + ]); + expect(first.module).not.toBe(second.module); + }); }); diff --git a/packages/workflow-engine/src/instances/fresh-instances.ts b/packages/workflow-engine/src/instances/fresh-instances.ts index e30467cbf..d4acb4403 100644 --- a/packages/workflow-engine/src/instances/fresh-instances.ts +++ b/packages/workflow-engine/src/instances/fresh-instances.ts @@ -3,7 +3,7 @@ import { Function } from 'effect'; import { instantiated } from './compiled-module.ts'; import type { InstanceSourceOf } from './instance-source.ts'; -export const freshInstances: InstanceSourceOf = (files) => async (memoryBytes) => ({ - ...(await instantiated(files, memoryBytes)), +export const freshInstances: InstanceSourceOf = (sandboxModule) => async (memoryBytes) => ({ + ...(await instantiated(sandboxModule, memoryBytes)), release: Function.constVoid, }); diff --git a/packages/workflow-engine/src/instances/instance-source.ts b/packages/workflow-engine/src/instances/instance-source.ts index 2501a7fb3..5ba34839c 100644 --- a/packages/workflow-engine/src/instances/instance-source.ts +++ b/packages/workflow-engine/src/instances/instance-source.ts @@ -2,11 +2,9 @@ import type { SandboxInstance } from '../programs/sandbox-session.ts'; export type InstanceSource = (memoryBytes: number) => Promise; -export interface SandboxFiles { - readonly bytes: (file: Readonly) => Promise>; -} +export type SandboxModule = { readonly bytes: Readonly } | { readonly compiled: WebAssembly.Module }; -export type InstanceSourceOf = (files: SandboxFiles) => InstanceSource; +export type InstanceSourceOf = (module: SandboxModule) => InstanceSource; export function instanceStock(source: InstanceSource): InstanceSource { const ready = new Map>(); diff --git a/packages/workflow-engine/src/instances/kept-instances.test.ts b/packages/workflow-engine/src/instances/kept-instances.test.ts index ca7445f9c..8cf6fc76b 100644 --- a/packages/workflow-engine/src/instances/kept-instances.test.ts +++ b/packages/workflow-engine/src/instances/kept-instances.test.ts @@ -1,6 +1,6 @@ -import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { describe, expect, it } from 'vitest'; +import { sandboxModuleOnDisk } from '../pool-testing/test-pools.ts'; import { expressionUnitOf } from '../programs/expression-units.ts'; import { moduleRun } from '../programs/module-runs.ts'; import { threadStackBytes, unitMemoryBytes, workerStackBytes } from '../programs/sandbox-bounds.ts'; @@ -23,7 +23,7 @@ function ranOn(instance: SandboxInstance, body: string): unknown { describe('the kept source of instances', () => { it('lends the next unit of the same maximum the instance the last one gave back intact', async () => { - const source = keptInstances(shippedFilesOnDisk); + const source = keptInstances(sandboxModuleOnDisk); const first = await source(unitMemoryBytes); ranOn(first, 'return 1;'); @@ -35,7 +35,7 @@ describe('the kept source of instances', () => { }); it('lends a fresh instance after a unit whose instance refused to grow, or whose runtime broke', async () => { - const source = keptInstances(shippedFilesOnDisk); + const source = keptInstances(sandboxModuleOnDisk); const refusing = await source(pageBytes * 256); const refused = ranOn(refusing, 'return "x".repeat(2 ** 25).length;'); const afterRefusal = await source(pageBytes * 256); @@ -58,7 +58,7 @@ describe('the kept source of instances', () => { }); it('keeps one instance for each maximum, and takes nothing back from a unit that gave its instance back before', async () => { - const source = keptInstances(shippedFilesOnDisk); + const source = keptInstances(sandboxModuleOnDisk); const [first, second] = [await source(unitMemoryBytes), await source(unitMemoryBytes)]; first.release(true); second.release(true); diff --git a/packages/workflow-engine/src/instances/kept-instances.ts b/packages/workflow-engine/src/instances/kept-instances.ts index aab8c742a..e2e99b35a 100644 --- a/packages/workflow-engine/src/instances/kept-instances.ts +++ b/packages/workflow-engine/src/instances/kept-instances.ts @@ -2,7 +2,7 @@ import type { SandboxInstance } from '../programs/sandbox-session.ts'; import { instantiated, type Instantiated } from './compiled-module.ts'; import type { InstanceSourceOf } from './instance-source.ts'; -export const keptInstances: InstanceSourceOf = (files) => { +export const keptInstances: InstanceSourceOf = (sandboxModule) => { const kept = new Map(); const lent = (instance: Instantiated): SandboxInstance => { const lending = { open: true }; @@ -20,6 +20,6 @@ export const keptInstances: InstanceSourceOf = (files) => { return async (memoryBytes) => { const waiting = kept.get(memoryBytes); kept.delete(memoryBytes); - return lent(waiting ?? (await instantiated(files, memoryBytes))); + return lent(waiting ?? (await instantiated(sandboxModule, memoryBytes))); }; }; diff --git a/packages/workflow-engine/src/pool-testing/test-pools.ts b/packages/workflow-engine/src/pool-testing/test-pools.ts index 0e5e85f0d..d1aa58636 100644 --- a/packages/workflow-engine/src/pool-testing/test-pools.ts +++ b/packages/workflow-engine/src/pool-testing/test-pools.ts @@ -1,9 +1,10 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { measuredEnvironment } from '@on.auto/operations/testing'; +import { sandboxModuleIn, sandboxShippedRoots } from '../instances/compiled-module.ts'; import { freshInstances } from '../instances/fresh-instances.ts'; import { machineSandboxOf } from '../instances/host-sandboxes.ts'; -import type { InstanceSource } from '../instances/instance-source.ts'; +import type { InstanceSource, SandboxModule } from '../instances/instance-source.ts'; import { InstanceSourceNameSchema, instanceSources, type InstanceSourceName } from '../instances/instance-sources.ts'; import type { HandedFiles, HandedWorkers, PoolSettings } from '../jobs/pool-contract.ts'; import { handedWorkersOf } from '../program-pool/handed-files.ts'; @@ -14,13 +15,17 @@ export interface SourceUnderTest { readonly source: InstanceSource; } -export const handedWorkers: HandedWorkers = await handedWorkersOf(shippedFilesOnDisk); +export const sandboxFilesOnDisk = shippedFilesOnDisk(sandboxShippedRoots); + +export const sandboxModuleOnDisk: SandboxModule = await sandboxModuleIn(sandboxFilesOnDisk); + +export const handedWorkers: HandedWorkers = await handedWorkersOf(sandboxFilesOnDisk); export const handedSandbox: HandedFiles = handedWorkers.sandbox; export const sourcesUnderTest: readonly SourceUnderTest[] = InstanceSourceNameSchema.literals.map((name) => ({ name, - source: instanceSources[name](shippedFilesOnDisk), + source: instanceSources[name](sandboxModuleOnDisk), })); export function testPoolSettings(workers: number): PoolSettings { @@ -28,5 +33,5 @@ export function testPoolSettings(workers: number): PoolSettings { } export function machineOnTheThread(): MachineSandbox { - return machineSandboxOf(freshInstances(shippedFilesOnDisk)); + return machineSandboxOf(freshInstances(sandboxModuleOnDisk)); } diff --git a/packages/workflow-engine/src/pool-testing/test-sandbox.ts b/packages/workflow-engine/src/pool-testing/test-sandbox.ts index 3033f1ffa..50723cb78 100644 --- a/packages/workflow-engine/src/pool-testing/test-sandbox.ts +++ b/packages/workflow-engine/src/pool-testing/test-sandbox.ts @@ -1,5 +1,3 @@ -import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; - import { workflowMachine } from '../decider/workflow-machine.ts'; import { freshInstances } from '../instances/fresh-instances.ts'; import type { RunDecider } from '../machine/run-decider.ts'; @@ -8,9 +6,10 @@ import { unitMemoryBytes } from '../programs/sandbox-bounds.ts'; import type { MachineOptions } from '../runner/run-descriptors.ts'; import { testMachineOf } from '../testing/driver-inputs.ts'; import { memoryDriver, type DriverOptions, type MemoryDriver } from '../testing/memory-driver.ts'; +import { sandboxModuleOnDisk } from './test-pools.ts'; export const testSandbox: MachineSandbox = reusedSandbox( - await freshInstances(shippedFilesOnDisk)(unitMemoryBytes), + await freshInstances(sandboxModuleOnDisk)(unitMemoryBytes), () => 0, ); diff --git a/packages/workflow-engine/src/process-pool/process-thread.ts b/packages/workflow-engine/src/process-pool/process-thread.ts index c45b39305..160239ab5 100644 --- a/packages/workflow-engine/src/process-pool/process-thread.ts +++ b/packages/workflow-engine/src/process-pool/process-thread.ts @@ -144,7 +144,7 @@ function watchOf( } function spawnedOf({ command, arguments: given }: ProcessCommand, module: Readonly): Spawned { - const child = spawn(command, [...given, fileURLToPath(module)], { + const child = spawn(command, [...given, fileURLToPath(module.href)], { env: {}, stdio: ['ignore', 'ignore', 'ignore', 'ipc'], serialization: 'json', diff --git a/packages/workflow-engine/src/program-pool/evaluation-thread.ts b/packages/workflow-engine/src/program-pool/evaluation-thread.ts index 724d04792..2e5fcbbd1 100644 --- a/packages/workflow-engine/src/program-pool/evaluation-thread.ts +++ b/packages/workflow-engine/src/program-pool/evaluation-thread.ts @@ -13,7 +13,7 @@ import { } from '../jobs/remote-evaluations.ts'; import { exhaustedBy, type ProgramRun } from '../programs/program-run.ts'; import { handedData } from './handed-files.ts'; -import { workerOptions } from './pool-threads.ts'; +import { workerModuleOf, workerOptions } from './pool-threads.ts'; export const answerGraceMs = 50; @@ -55,7 +55,7 @@ function absoluteNow(): number { function started(module: Readonly, settings: PoolSettings): Live { const { port1, port2 } = new MessageChannel(); const flags = new Int32Array(new SharedArrayBuffer(2 * Int32Array.BYTES_PER_ELEMENT)); - const worker = new Worker(module, { + const worker = new Worker(workerModuleOf(module), { ...workerOptions(settings), workerData: { port: port2, flags: flags.buffer, ...handedData(settings, module) }, transferList: [port2], diff --git a/packages/workflow-engine/src/program-pool/handed-files.test.ts b/packages/workflow-engine/src/program-pool/handed-files.test.ts index 50f61ad5a..ebdd1ee74 100644 --- a/packages/workflow-engine/src/program-pool/handed-files.test.ts +++ b/packages/workflow-engine/src/program-pool/handed-files.test.ts @@ -1,3 +1,4 @@ +import { shippedFileName, type ShippedFile } from '@on.auto/operations'; import { shippedFilesInMemory } from '@on.auto/operations/testing'; import { describe, expect, it } from 'vitest'; @@ -13,19 +14,23 @@ import { handedWorkersOf, } from './handed-files.ts'; -const shipped = new URL('file:///shipped/'); +const checking = '@acme/checking'; -const lib = new URL('lib.txt', shipped); +const lib: ShippedFile = { package: checking, path: 'lib/lib.txt' }; const checker = new URL('file:///workers/checker.ts'); const runner = new URL('file:///workers/runner.ts'); const memory = shippedFilesInMemory({ - [sandboxModuleFile.href]: Uint8Array.of(1, 2), - [lib.href]: Uint8Array.of(3), + [sandboxModuleFile.package]: { [sandboxModuleFile.path]: Uint8Array.of(1, 2) }, + [checking]: { [lib.path]: Uint8Array.of(3) }, }); +const libName = shippedFileName(lib); + +const sandboxName = shippedFileName(sandboxModuleFile); + function bytesOf(handed: Readonly>>): Readonly> { return Object.fromEntries( Object.entries(handed).map(([href, bytes]: readonly [string, Readonly]) => [href, [...bytes]]), @@ -35,44 +40,44 @@ function bytesOf(handed: Readonly>>): Readon describe('the files a composition hands the workers of a pool', () => { it('are the bytes of the files it names, read through the reader, by the address of each', async () => { expect(bytesOf(await handedFilesOf(memory, [lib, sandboxModuleFile]))).toEqual({ - [lib.href]: [3], - [sandboxModuleFile.href]: [1, 2], + [libName]: [3], + [sandboxName]: [1, 2], }); }); it('are, for every worker, the module of the sandbox, and for the workers of a module named, its own files instead', async () => { const handed = await handedWorkersOf(memory, { [checker.href]: await handedFilesOf(memory, [lib]) }); - expect(bytesOf(handedTo(handed, runner))).toEqual({ [sandboxModuleFile.href]: [1, 2] }); - expect(bytesOf(handedTo(handed, checker))).toEqual({ [lib.href]: [3] }); - expect(bytesOf(handedTo({ sandbox: handed.sandbox }, checker))).toEqual({ [sandboxModuleFile.href]: [1, 2] }); + expect(bytesOf(handedTo(handed, runner))).toEqual({ [sandboxName]: [1, 2] }); + expect(bytesOf(handedTo(handed, checker))).toEqual({ [libName]: [3] }); + expect(bytesOf(handedTo({ sandbox: handed.sandbox }, checker))).toEqual({ [sandboxName]: [1, 2] }); }); it('cross to a worker with the source of instances the pool names', async () => { const handed = await handedWorkersOf(memory, { [checker.href]: await handedFilesOf(memory, [lib]) }); expect(handedData({ handed, instances: 'kept' }, checker)).toEqual({ - handed: { [lib.href]: Uint8Array.of(3) }, + handed: { [libName]: Uint8Array.of(3) }, instances: 'kept', }); }); it('fail to be read when the reader cannot read one of them', async () => { - await expect(handedFilesOf(memory, [new URL('missing.bin', shipped)])).rejects.toThrow('missing.bin'); + await expect(handedFilesOf(memory, [{ package: checking, path: 'missing.bin' }])).rejects.toThrow('missing.bin'); }); }); describe('a file a worker reads of those it was handed', () => { it('is the bytes the pool handed it', () => { - expect([...handedFile(lib, { handed: { [lib.href]: Uint8Array.of(7) } })]).toEqual([7]); - expect([...handedFileIn({ [lib.href]: Uint8Array.of(8) })(lib)]).toEqual([8]); + expect([...handedFile(lib, { handed: { [libName]: Uint8Array.of(7) } })]).toEqual([7]); + expect([...handedFileIn({ [libName]: Uint8Array.of(8) })(lib)]).toEqual([8]); }); it('is refused when the pool handed the worker no such file, or nothing it can read', () => { - const refused = `The pool handed this worker no file at ${lib.href}`; + const refused = `The pool handed this worker no file at ${libName}`; expect(() => handedFile(lib, { handed: {} })).toThrow(refused); - expect(() => handedFile(lib, { handed: { [lib.href]: 'not bytes' } })).toThrow(refused); + expect(() => handedFile(lib, { handed: { [libName]: 'not bytes' } })).toThrow(refused); expect(() => handedFile(lib)).toThrow(refused); expect(() => handedFileIn({})(lib)).toThrow(refused); }); @@ -82,4 +87,10 @@ describe('the instances of the sandbox a worker takes', () => { it('are refused when the pool named the worker no source of them', async () => { await expect(handedSandboxOf({ handed: {} })(unitMemoryBytes)).rejects.toThrow('no source of instances'); }); + + it('come of the module of the sandbox read when a unit first asks for one, so a worker handed none is refused only then', async () => { + const source = handedSandboxOf({ handed: {}, instances: 'kept' }); + + await expect(source(unitMemoryBytes)).rejects.toThrow(`The pool handed this worker no file at ${sandboxName}`); + }); }); diff --git a/packages/workflow-engine/src/program-pool/handed-files.ts b/packages/workflow-engine/src/program-pool/handed-files.ts index 395bb343f..dd9fe4e9a 100644 --- a/packages/workflow-engine/src/program-pool/handed-files.ts +++ b/packages/workflow-engine/src/program-pool/handed-files.ts @@ -1,14 +1,14 @@ import { workerData } from 'node:worker_threads'; -import type { ShippedFiles } from '@on.auto/operations'; +import { shippedFileName, type ShippedFile, type ShippedFiles } from '@on.auto/operations'; import { Option, Schema } from 'effect'; import { sandboxModuleFile } from '../instances/compiled-module.ts'; -import type { InstanceSource, SandboxFiles } from '../instances/instance-source.ts'; +import type { InstanceSource, InstanceSourceOf } from '../instances/instance-source.ts'; import { InstanceSourceNameSchema, instanceSources, type InstanceSourceName } from '../instances/instance-sources.ts'; import type { HandedFiles, HandedWorkers, PoolSettings } from '../jobs/pool-contract.ts'; -export type HandedFile = (file: Readonly) => Readonly; +export type HandedFile = (file: ShippedFile) => Readonly; interface HandedData { readonly handed: HandedFiles; @@ -28,8 +28,10 @@ const decodeNamed = Schema.decodeUnknownOption(Schema.Struct({ instances: Instan const unnamedSource: InstanceSource = () => Promise.reject(new Error('The pool named this worker no source of instances of the sandbox')); -export async function handedFilesOf(shippedFiles: ShippedFiles, files: readonly Readonly[]): Promise { - const contents = await Promise.all(files.map(async (file) => [file.href, await shippedFiles.bytes(file)] as const)); +export async function handedFilesOf(shippedFiles: ShippedFiles, files: readonly ShippedFile[]): Promise { + const contents = await Promise.all( + files.map(async (file) => [shippedFileName(file), await shippedFiles.bytes(file)] as const), + ); return Object.fromEntries(contents); } @@ -54,24 +56,29 @@ export function handedData( export function handedFileIn(handed: HandedFiles): HandedFile { return (file) => Option.getOrThrowWith( - Option.fromNullishOr(handed[file.href]), - () => new Error(`The pool handed this worker no file at ${file.href}`), + Option.fromNullishOr(handed[shippedFileName(file)]), + () => new Error(`The pool handed this worker no file at ${shippedFileName(file)}`), ); } -export function handedFile(file: Readonly, data: unknown = workerData): Readonly { +export function handedFile(file: ShippedFile, data: unknown = workerData): Readonly { return handedFileIn(Option.match(decodeHanded(data), { onNone: () => ({}), onSome: ({ handed }: Handed) => handed }))( file, ); } -function handedSandboxFiles(data: unknown): SandboxFiles { - return { bytes: (file) => Promise.try(() => handedFile(file, data)) }; +function madeOnFirstInstance(sourceOf: InstanceSourceOf, data: unknown): InstanceSource { + const made: { source?: InstanceSource } = {}; + return (memoryBytes) => + Promise.try(() => { + made.source ??= sourceOf({ bytes: handedFile(sandboxModuleFile, data) }); + return made.source(memoryBytes); + }); } export function handedSandboxOf(data: unknown): InstanceSource { return Option.match(decodeNamed(data), { onNone: () => unnamedSource, - onSome: ({ instances }) => instanceSources[instances](handedSandboxFiles(data)), + onSome: ({ instances }) => madeOnFirstInstance(instanceSources[instances], data), }); } diff --git a/packages/workflow-engine/src/program-pool/job-loop.test.ts b/packages/workflow-engine/src/program-pool/job-loop.test.ts index 7e7a7201a..4c3f77c7f 100644 --- a/packages/workflow-engine/src/program-pool/job-loop.test.ts +++ b/packages/workflow-engine/src/program-pool/job-loop.test.ts @@ -134,13 +134,13 @@ describe.each(sourcesUnderTest)('a loop that starts serving jobs on $name instan }); it('makes its handlers of the files the pool handed its worker', async () => { - const note = new URL('file:///shipped/note.txt'); + const note = { package: '@acme/shipped', path: 'note.txt' }; const ask = servedWith( name, (handed) => ({ check: () => ({ ran: 'checked', issues: [{ at: 0, line: 1, detail: new TextDecoder().decode(handed(note)) }] }), }), - { [note.href]: new TextEncoder().encode('Handed at start') }, + { '@acme/shipped/note.txt': new TextEncoder().encode('Handed at start') }, ); expect(await ask({ job: 1, kind: 'check', request: checkRequest })).toMatchObject({ diff --git a/packages/workflow-engine/src/program-pool/pool-permits.test.ts b/packages/workflow-engine/src/program-pool/pool-permits.test.ts index b53de3d4d..07a712da2 100644 --- a/packages/workflow-engine/src/program-pool/pool-permits.test.ts +++ b/packages/workflow-engine/src/program-pool/pool-permits.test.ts @@ -181,8 +181,10 @@ describe('a pool that closes', () => { const script = [ `const { programPool } = await import('${new URL('program-pool.ts', import.meta.url).href}');`, `const { handedWorkersOf } = await import('${new URL('handed-files.ts', import.meta.url).href}');`, + `const { sandboxShippedRoots } = await import('${new URL('../instances/compiled-module.ts', import.meta.url).href}');`, `const { shippedFilesOnDisk } = await import('${import.meta.resolve('@on.auto/operations/shipped-files-on-disk')}');`, - "const pool = programPool({ handed: await handedWorkersOf(shippedFilesOnDisk), instances: 'fresh', workers: 2, heapMegabytes: 64 });", + 'const handed = await handedWorkersOf(shippedFilesOnDisk(sandboxShippedRoots));', + "const pool = programPool({ handed, instances: 'fresh', workers: 2, heapMegabytes: 64 });", "const source = 'export default function (input) { return input + 1; }';", "const outcome = await pool.run({ source, arguments: [1], entry: 'default', moment: 0, budget: 500, memoryBytes: 67108864, stackBytes: 1048576, deadlineMs: 10000, mostOutputBytes: 100 });", closing, diff --git a/packages/workflow-engine/src/program-pool/pool-threads.ts b/packages/workflow-engine/src/program-pool/pool-threads.ts index 38a08cd59..c89d0c589 100644 --- a/packages/workflow-engine/src/program-pool/pool-threads.ts +++ b/packages/workflow-engine/src/program-pool/pool-threads.ts @@ -1,3 +1,4 @@ +import { URL as NodeUrl } from 'node:url'; import { Worker, type WorkerOptions } from 'node:worker_threads'; import { Option, Schema } from 'effect'; @@ -203,8 +204,12 @@ export function workerOptions(settings: PoolSettings): WorkerOptions { }; } +export function workerModuleOf(module: Readonly): NodeUrl { + return new NodeUrl(module.href); +} + function workerStarting(module: Readonly, settings: PoolSettings): Worker { - return new Worker(module, { ...workerOptions(settings), workerData: handedData(settings, module) }); + return new Worker(workerModuleOf(module), { ...workerOptions(settings), workerData: handedData(settings, module) }); } function threadOf(start: Job, settings: PoolSettings, hooks: ThreadHooks): Thread { diff --git a/packages/workflow-engine/src/programs/reserved-instances.test.ts b/packages/workflow-engine/src/programs/reserved-instances.test.ts index c2f118a04..bdafae5b3 100644 --- a/packages/workflow-engine/src/programs/reserved-instances.test.ts +++ b/packages/workflow-engine/src/programs/reserved-instances.test.ts @@ -1,14 +1,14 @@ -import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; import { freshInstances } from '../instances/fresh-instances.ts'; +import { sandboxModuleOnDisk } from '../pool-testing/test-pools.ts'; import type { Evaluation, ProgramRun } from './program-run.ts'; import { reservedSandbox, reusedSandbox, type MachineSandbox } from './reserved-instances.ts'; import { unitMemoryBytes } from './sandbox-bounds.ts'; import type { SandboxInstance } from './sandbox-session.ts'; -const instance = await freshInstances(shippedFilesOnDisk)(unitMemoryBytes); +const instance = await freshInstances(sandboxModuleOnDisk)(unitMemoryBytes); const evaluation: Evaluation = { budget: 250, deadlineAt: Number.POSITIVE_INFINITY, moment: 0 }; diff --git a/packages/workflow-host/measure/measured-views.ts b/packages/workflow-host/measure/measured-views.ts index 37da70b29..5e93e8820 100644 --- a/packages/workflow-host/measure/measured-views.ts +++ b/packages/workflow-host/measure/measured-views.ts @@ -5,6 +5,7 @@ import { unitMemoryBytes, workerStackBytes, type ProgramPool, + sandboxShippedRoots, } from '@on.auto/workflow-engine/dsl'; import { Effect, type Schema } from 'effect'; @@ -15,7 +16,9 @@ import type { KeptView } from '../src/views/view-rows.ts'; import { viewsFolder } from '../src/views/views-folder.ts'; import { viewsPortOn } from '../src/views/views-port.ts'; -const handed = await handedWorkersOf(shippedFilesOnDisk); +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const handed = await handedWorkersOf(sandboxFiles); export function measuringPool(): ProgramPool { return programPool({ handed, instances: 'fresh', workers: 4, heapMegabytes: 256 }); diff --git a/packages/workflow-host/src/reaction-testing/recorded-reactions.ts b/packages/workflow-host/src/reaction-testing/recorded-reactions.ts index 5a6493085..765cbc2aa 100644 --- a/packages/workflow-host/src/reaction-testing/recorded-reactions.ts +++ b/packages/workflow-host/src/reaction-testing/recorded-reactions.ts @@ -1,11 +1,15 @@ import type { Emission } from '@on.auto/definitions'; import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; -import { filterSandboxOf, freshInstances } from '@on.auto/workflow-engine/dsl'; +import { filterSandboxOf, freshInstances, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { Effect } from 'effect'; import { StartRefused, type ReactionOptions, type ReactionStart } from '../reactions/reaction-options.ts'; import { StartRejected } from '../reactions/start-rejected.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const sandboxModule = await sandboxModuleIn(sandboxFiles); + export interface RecordedReactions { readonly options: ReactionOptions; readonly starts: () => readonly ReactionStart[]; @@ -34,7 +38,7 @@ export function recordedReactions(behaviour: ReactionBehaviour = {}): RecordedRe return { options: { definitionType: 'workflow', - filters: filterSandboxOf(freshInstances(shippedFilesOnDisk)), + filters: filterSandboxOf(freshInstances(sandboxModule)), start: (start) => Effect.suspend(() => { const failure = behaviour.failure?.(start) ?? null; diff --git a/packages/workflow-host/src/reaction-testing/trigger-starting.ts b/packages/workflow-host/src/reaction-testing/trigger-starting.ts index 6a73888d9..b546a4754 100644 --- a/packages/workflow-host/src/reaction-testing/trigger-starting.ts +++ b/packages/workflow-host/src/reaction-testing/trigger-starting.ts @@ -1,6 +1,6 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import type { FilterVerdict, MatchedFilter } from '@on.auto/workflow-engine'; -import { filterSandboxOf, freshInstances } from '@on.auto/workflow-engine/dsl'; +import { filterSandboxOf, freshInstances, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { Array, Effect, Function } from 'effect'; import { filterMatchingOf, filterStops, type FilterStops, type MatchFilters } from '../filtering/filter-matching.ts'; @@ -9,6 +9,10 @@ import { subscriptionStarts } from '../reactions/subscription-starts.ts'; import { onSQLite, openedOn } from '../testing/host-files.ts'; import { saidRefusals, type followedRecordOf } from './followed-records.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + +const sandboxModule = await sandboxModuleIn(sandboxFiles); + const topRuns: ReadonlyMap = new Map([ ['r-of-close', 'close'], ['r-of-other', 'other'], @@ -65,7 +69,7 @@ interface CountingMatches { export function countingMatches(): CountingMatches { const evaluated: string[] = []; - const matching = filterMatchingOf(filterSandboxOf(freshInstances(shippedFilesOnDisk))); + const matching = filterMatchingOf(filterSandboxOf(freshInstances(sandboxModule))); return { match: (filters, event, now) => { evaluated.push(...filters.map(({ reference }) => reference)); diff --git a/packages/workflow-host/src/testing/host-documents.ts b/packages/workflow-host/src/testing/host-documents.ts index e256fb673..e6ffbc2dd 100644 --- a/packages/workflow-host/src/testing/host-documents.ts +++ b/packages/workflow-host/src/testing/host-documents.ts @@ -1,6 +1,6 @@ import { shippedFilesOnDisk } from '@on.auto/operations/shipped-files-on-disk'; import type { MachineOptions } from '@on.auto/workflow-engine'; -import { freshInstances, machineSandboxOf } from '@on.auto/workflow-engine/dsl'; +import { freshInstances, machineSandboxOf, sandboxModuleIn, sandboxShippedRoots } from '@on.auto/workflow-engine/dsl'; import { defaultLimits, defaultSeed, testMachineOf } from '@on.auto/workflow-engine/testing'; import { Schema } from 'effect'; import { parse } from 'yaml'; @@ -8,13 +8,17 @@ import { parse } from 'yaml'; import type { RunStart } from '../host/run-requests.ts'; import type { RunAddress } from '../runs/run-address.ts'; +const sandboxFiles = shippedFilesOnDisk(sandboxShippedRoots); + type JsonObject = typeof Schema.JsonObject.Type; const decodeObject = Schema.decodeUnknownSync(Schema.JsonObject); const header = { dsl: '1.0.3', namespace: 'acme', name: 'test', version: '1.0.0' }; -export const testMachine: MachineOptions = testMachineOf(machineSandboxOf(freshInstances(shippedFilesOnDisk))); +export const testMachine: MachineOptions = testMachineOf( + machineSandboxOf(freshInstances(await sandboxModuleIn(sandboxFiles))), +); export function workflow(source: string): JsonObject { return { document: header, ...decodeObject(parse(source)) };