feat(server): the npm package carries every package as a subpath, with its testing entry

CLAUDE.md

@@ -4,7 +4,7 @@
 
 `@on.auto/runtime` is the runtime for business playbooks, a pnpm monorepo. It is source-available under the Elastic License 2.0 (see `LICENSING.md`); never call it open source. Its container image runs self-hosted and in Auto's cloud hosting. Auto Studio (the management plane, with governance and observability) lives in the private `studio` repo and the hosting side in the private `on.auto` repo, not here.
 
-- `packages/server`: the Node.js server (`@on.auto/server`), plus everything that packages it into a container (`Dockerfile`, `Dockerfile.dockerignore`, `container.vex.json`, the findings of image scanners that do not apply to it, and `docker-hub.md`, the image's description on Docker Hub) and into the npm package `@on.auto/runtime` (`npm/pack.ts`, `npm/README.md`), whose `auto-runtime` command is `auto-runtime.ts`. The package ships the same `.ts` sources as the image; its launcher, `src/command-line/launch.ts`, is shipped as JavaScript with its types stripped, so it imports only `node:` modules, and it copies the sources out of `node_modules`, where Node does not strip types, before it runs them
+- `packages/server`: the Node.js server (`@on.auto/server`), plus everything that packages it into a container (`Dockerfile`, `Dockerfile.dockerignore`, `container.vex.json`, the findings of image scanners that do not apply to it, and `docker-hub.md`, the image's description on Docker Hub) and into the npm package `@on.auto/runtime` (`npm/pack.ts`, `npm/README.md`), whose `auto-runtime` command is `auto-runtime.ts`. The package ships the same `.ts` sources as the image and, beside them, each package's testing entry, so a program that composes the core can test against it; its launcher, `src/command-line/launch.ts`, is shipped as JavaScript with its types stripped, so it imports only `node:` modules, and it copies the sources out of `node_modules`, where Node does not strip types, before it runs them
 - `packages/api`: the API (`@on.auto/api`), a Hono app that answers every request, with problem documents, the `Origin` and `Host` checks, authentication and the operation routes
 - `packages/identity`: API keys, the local-mode rule and the key command (`@on.auto/identity`)
 - `packages/*`: the server's libraries (`@on.auto/*`), including the ledger

packages/server/npm/README.md

@@ -66,6 +66,14 @@ npx @on.auto/runtime --help
 
 The same release is the image `docker.on.auto/onauto/runtime`, from Docker Hub. See [Run with Docker](https://on.auto/docs/self-host/docker).
 
+## Compose it in your own program
+
+The package carries each of the runtime's packages as a subpath, so a program can compose the core itself rather than run the server. `@on.auto/runtime/ledger` is the ledger, `@on.auto/runtime/ledger/sqlite3` one of the subpaths the ledger names, and `@on.auto/runtime/ledger/testing` its testing entry. A testing entry is for your tests as it is for the runtime's: `recordedContentBehaviour` of `@on.auto/runtime/operations/testing`, the behaviour suite of the content port, runs against a store of your own. The test tools a testing entry imports are your tests' to provide.
+
+The subpaths are the `.ts` sources the server runs, and they import one another by the packages' own names, such as `@on.auto/operations`. Where your bundler must resolve those names, alias each of them, with its subpaths, to the same subpath of `@on.auto/runtime`:
+
+<!-- npm/pack.ts writes the alias here -->
+
 ## License
 
 Auto Runtime is source-available under the [Elastic License 2.0](https://github.com/BeOnAuto/runtime/blob/main/LICENSE). [LICENSING.md](https://github.com/BeOnAuto/runtime/blob/main/LICENSING.md) explains the terms.

packages/server/npm/imported-packages.test.ts

@@ -38,6 +38,10 @@ describe('the packages a shipped source imports', () => {
       '];',
       "const said = 'import ';",
       "  export const indented = 'from ' + 'nowhere';",
+      'export const worker = workerOf([',
+      "  `import { serveJobs } from '${jobLoop}';`,",
+      "  `import { answerOf } from '${new URL('./answer.ts', import.meta.url).href}';`,",
+      ']);',
     ].join('\n');
 
     expect(bareSpecifiersIn(source)).toEqual([]);

packages/server/npm/imported-packages.ts

@@ -1,5 +1,5 @@
 const statements: readonly RegExp[] = [
-  /^(?:import|export)\b[^;'"]*?\bfrom\s*'([^']*)'/gmu,
+  /^(?:import|export)\b[^;'"`]*?\bfrom\s*'([^']*)'/gmu,
   /^import\s*'([^']*)'/gmu,
   /\bimport\(\s*'([^']*)'/gu,
   /\bimport\.meta\.resolve\(\s*'([^']*)'/gu,

packages/server/npm/pack.ts

@@ -7,13 +7,21 @@ import { fileURLToPath } from 'node:url';
 import { Schema } from 'effect';
 
 import { bareSpecifiersIn } from './imported-packages.ts';
+import { aliasedNamesOf, readmeWithAlias, subpathsOf, type PackedPackage } from './package-subpaths.ts';
+import { importedSubpaths } from './subpath-check.ts';
 
 const repository = fileURLToPath(new URL('../../../', import.meta.url));
 const distribution = join(repository, 'dist');
 const output = join(distribution, 'npm');
 const workspaceGroups: readonly string[] = ['packages', 'capabilities'];
 const developmentSources = join(repository, 'packages', 'server', 'src', 'development');
-const testHelperImports: readonly string[] = ['vitest'];
+const testHelperImports: readonly string[] = [
+  '@modelcontextprotocol/sdk',
+  'ajv',
+  'ajv-formats',
+  'typescript',
+  'vitest',
+];
 
 const Names = Schema.Record(Schema.String, Schema.String);
 
@@ -89,24 +97,40 @@ function closureOf(names: readonly string[], reached: ReadonlySet<string>): Read
   return closureOf(following, new Set([...reached, ...reachedNow]));
 }
 
-function isShipped(source: string): boolean {
+function isTestingFolder(name: string): boolean {
+  return name === 'testing' || name.endsWith('-testing');
+}
+
+function offersTesting({ manifest }: WorkspacePackage): boolean {
+  return Object.hasOwn(manifest.exports ?? {}, './testing');
+}
+
+function isShipped(shipped: WorkspacePackage, source: string): boolean {
   const name = basename(source);
   return (
-    !name.endsWith('.test.ts') && name !== 'testing' && !name.endsWith('-testing') && source !== developmentSources
+    !name.endsWith('.test.ts') && source !== developmentSources && (offersTesting(shipped) || !isTestingFolder(name))
   );
 }
 
-function shippedExports({ directory, manifest }: WorkspacePackage): Readonly<Record<string, string>> {
-  return Object.fromEntries(
-    Object.entries(manifest.exports ?? {}).filter(([, target]: readonly [string, string]) =>
-      existsSync(join(output, directory, target)),
+function packedOf(shipped: WorkspacePackage): PackedPackage {
+  const { directory, manifest } = shipped;
+  return {
+    name: manifest.name,
+    directory,
+    exports: Object.fromEntries(
+      Object.entries(manifest.exports ?? {}).filter(([, target]: readonly [string, string]) =>
+        existsSync(join(output, directory, target)),
+      ),
     ),
-  );
+  };
+}
+
+function installedIn(directory: string, dependency: string): string {
+  return join(repository, directory, 'node_modules', dependency);
 }
 
 function installedVersion({ directory }: WorkspacePackage, dependency: string): string {
-  return installedOf(readFileSync(join(repository, directory, 'node_modules', dependency, 'package.json'), 'utf8'))
-    .version;
+  return installedOf(readFileSync(join(installedIn(directory, dependency), 'package.json'), 'utf8')).version;
 }
 
 function dependenciesOf(names: readonly string[]): Readonly<Record<string, string>> {
@@ -131,6 +155,29 @@ function dependenciesOf(names: readonly string[]): Readonly<Record<string, strin
   );
 }
 
+function installedDependencies(packages: readonly WorkspacePackage[]): ReadonlyMap<string, string> {
+  return new Map(
+    packages.flatMap((shipped) =>
+      dependencyEntries(shipped)
+        .filter(([, specification]: readonly [string, string]) => !isWorkspace(specification))
+        .map(([dependency]: readonly [string, string]): [string, string] => [
+          dependency,
+          installedIn(shipped.directory, dependency),
+        ]),
+    ),
+  );
+}
+
+function installedTestHelpers(packages: readonly WorkspacePackage[]): ReadonlyMap<string, string> {
+  return new Map(
+    ['.', ...packages.map(({ directory }) => directory)]
+      .flatMap((directory) =>
+        testHelperImports.map((helper): [string, string] => [helper, installedIn(directory, helper)]),
+      )
+      .filter(([, installed]: readonly [string, string]) => existsSync(installed)),
+  );
+}
+
 function packageNameOf(specifier: string): string {
   const parts = specifier.split('/');
   return (specifier.startsWith('@') ? parts.slice(0, 2) : parts.slice(0, 1)).join('/');
@@ -156,7 +203,7 @@ mkdirSync(output, { recursive: true });
 for (const shipped of shippedPackages) {
   cpSync(join(repository, shipped.directory, 'src'), join(output, shipped.directory, 'src'), {
     recursive: true,
-    filter: isShipped,
+    filter: (source) => isShipped(shipped, source),
   });
 }
 
@@ -174,13 +221,20 @@ const extras = [
 for (const extra of extras) {
   cpSync(join(repository, extra), join(output, extra), { recursive: true });
 }
-cpSync(join(repository, 'packages', 'server', 'npm', 'README.md'), join(output, 'README.md'));
+const packed = shippedPackages.map((shipped) => packedOf(shipped));
+const aliasedNames = aliasedNamesOf(packed);
+const subpaths = subpathsOf(packed);
+writeFileSync(
+  join(output, 'README.md'),
+  readmeWithAlias(
+    readFileSync(join(repository, 'packages', 'server', 'npm', 'README.md'), 'utf8'),
+    root.name,
+    aliasedNames,
+  ),
+);
 
-for (const shipped of shippedPackages) {
-  writeFileSync(
-    join(output, shipped.directory, 'package.json'),
-    json({ name: shipped.manifest.name, type: 'module', exports: shippedExports(shipped) }),
-  );
+for (const { name, directory, exports } of packed) {
+  writeFileSync(join(output, directory, 'package.json'), json({ name, type: 'module', exports }));
 }
 
 const dependencies = dependenciesOf(shippedNames);
@@ -205,6 +259,7 @@ writeFileSync(
     repository: { type: 'git', url: 'git+https://github.com/BeOnAuto/runtime.git' },
     license: root.license,
     type: 'module',
+    exports: subpaths,
     bin: { 'auto-runtime': 'bin/auto-runtime.js' },
     engines: root.engines,
     os: ['darwin', 'linux'],
@@ -230,7 +285,15 @@ writeFileSync(
   { mode: 0o755 },
 );
 
+const tarball = join(distribution, `${root.name.slice(1).replace('/', '-')}-${root.version ?? ''}.tgz`);
 execFileSync('npm', ['pack', '--pack-destination', distribution], { cwd: output, stdio: 'inherit' });
+importedSubpaths({
+  packageName: root.name,
+  tarball,
+  modules: new Map([...installedDependencies(shippedPackages), ...installedTestHelpers(shippedPackages)]),
+  aliasedNames,
+  subpaths: Object.keys(subpaths),
+});
 process.stdout.write(
-  `Packed ${shippedNames.length} workspace packages and ${Object.keys(dependencies).length} dependencies into ${join(distribution, `${root.name.slice(1).replace('/', '-')}-${root.version ?? ''}.tgz`)}\n`,
+  `Packed ${shippedNames.length} workspace packages and ${Object.keys(dependencies).length} dependencies into ${tarball}, and imported its ${Object.keys(subpaths).length} subpaths\n`,
 );

packages/server/npm/package-subpaths.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+
+import { aliasPlace, aliasedNamesOf, readmeWithAlias, subpathsOf, type PackedPackage } from './package-subpaths.ts';
+
+const packed: readonly PackedPackage[] = [
+  {
+    name: '@on.auto/ledger',
+    directory: 'packages/ledger',
+    exports: { '.': './src/index.ts', './sqlite3': './src/sqlite3.ts', './testing': './src/testing/index.ts' },
+  },
+  { name: '@on.auto/recall', directory: 'capabilities/recall', exports: { '.': './src/index.ts' } },
+  { name: '@on.auto/server', directory: 'packages/server', exports: {} },
+];
+
+describe('the subpaths of the package', () => {
+  it('name each entry of each packed package under the name of the package within its scope', () => {
+    expect(subpathsOf(packed)).toEqual({
+      './ledger': './packages/ledger/src/index.ts',
+      './ledger/sqlite3': './packages/ledger/src/sqlite3.ts',
+      './ledger/testing': './packages/ledger/src/testing/index.ts',
+      './recall': './capabilities/recall/src/index.ts',
+    });
+  });
+
+  it('are aliased from the names of the packages that have entries', () => {
+    expect(aliasedNamesOf(packed)).toEqual(['ledger', 'recall']);
+  });
+});
+
+describe('the README of the package', () => {
+  it('holds, where it names the place of the alias, the names and the alias from each to its subpath', () => {
+    const readme = ['Alias them:', '', aliasPlace, '', '## License'].join('\n');
+
+    expect(readmeWithAlias(readme, '@on.auto/runtime', ['ledger', 'recall'])).toBe(
+      [
+        'Alias them:',
+        '',
+        '```js',
+        "const names = ['ledger', 'recall'];",
+        'const alias = Object.fromEntries(names.map((name) => [`@on.auto/${name}`, `@on.auto/runtime/${name}`]));',
+        '```',
+        '',
+        '## License',
+      ].join('\n'),
+    );
+  });
+
+  it('is refused when it names the place of the alias not once', () => {
+    expect(() => readmeWithAlias('Alias them.', '@on.auto/runtime', ['ledger'])).toThrow(
+      `The README must hold ${aliasPlace} once`,
+    );
+    expect(() => readmeWithAlias(`${aliasPlace}\n${aliasPlace}`, '@on.auto/runtime', ['ledger'])).toThrow(
+      `The README must hold ${aliasPlace} once`,
+    );
+  });
+});

packages/server/npm/package-subpaths.ts

@@ -0,0 +1,49 @@
+import { posix } from 'node:path';
+
+export interface PackedPackage {
+  readonly name: string;
+  readonly directory: string;
+  readonly exports: Readonly<Record<string, string>>;
+}
+
+export const aliasPlace = '<!-- npm/pack.ts writes the alias here -->';
+
+function scopeOf(name: string): string {
+  return name.slice(0, name.indexOf('/'));
+}
+
+function nameInScope(name: string): string {
+  return name.slice(name.indexOf('/') + 1);
+}
+
+export function subpathsOf(packed: readonly PackedPackage[]): Readonly<Record<string, string>> {
+  return Object.fromEntries(
+    packed.flatMap(({ name, directory, exports }: PackedPackage) =>
+      Object.entries(exports).map(([entry, target]: readonly [string, string]) => [
+        `./${posix.join(nameInScope(name), entry)}`,
+        `./${posix.join(directory, target)}`,
+      ]),
+    ),
+  );
+}
+
+export function aliasedNamesOf(packed: readonly PackedPackage[]): readonly string[] {
+  return packed
+    .filter(({ exports }: PackedPackage) => Object.keys(exports).length > 0)
+    .map(({ name }) => nameInScope(name));
+}
+
+export function readmeWithAlias(readme: string, root: string, names: readonly string[]): string {
+  const around = readme.split(aliasPlace);
+  if (around.length !== 2) {
+    throw new Error(`The README must hold ${aliasPlace} once`);
+  }
+  return around.join(
+    [
+      '```js',
+      `const names = [${names.map((name) => `'${name}'`).join(', ')}];`,
+      `const alias = Object.fromEntries(names.map((name) => [\`${scopeOf(root)}/\${name}\`, \`${root}/\${name}\`]));`,
+      '```',
+    ].join('\n'),
+  );
+}

packages/server/npm/subpath-check.ts

@@ -0,0 +1,47 @@
+import { execFileSync } from 'node:child_process';
+import { cpSync, mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, posix } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { linked } from '../src/command-line/launch.ts';
+
+export interface SubpathCheck {
+  readonly packageName: string;
+  readonly tarball: string;
+  readonly modules: ReadonlyMap<string, string>;
+  readonly aliasedNames: readonly string[];
+  readonly subpaths: readonly string[];
+}
+
+export function importedSubpaths({ packageName, tarball, modules, aliasedNames, subpaths }: SubpathCheck): void {
+  const program = mkdtempSync(join(tmpdir(), 'auto-runtime-program-'));
+  try {
+    execFileSync('tar', ['-xzf', tarball, '-C', program]);
+    linked(program, new Map([...modules, [packageName, join(program, 'package')]]));
+    cpSync(fileURLToPath(new URL('subpath-imports.ts', import.meta.url)), join(program, 'subpath-imports.test.ts'));
+    execFileSync(
+      process.execPath,
+      [
+        join(program, 'node_modules', 'vitest', 'vitest.mjs'),
+        'run',
+        '--root',
+        program,
+        '--experimental.viteModuleRunner=false',
+        'subpath-imports.test.ts',
+      ],
+      {
+        cwd: program,
+        stdio: 'inherit',
+        env: {
+          ...process.env,
+          PACKAGE_NAME: packageName,
+          PACKAGE_ALIASED_NAMES: aliasedNames.join(' '),
+          PACKAGE_SUBPATHS: subpaths.map((subpath) => posix.join(packageName, subpath)).join(' '),
+        },
+      },
+    );
+  } finally {
+    rmSync(program, { recursive: true, force: true });
+  }
+}

packages/server/npm/subpath-imports.ts

@@ -0,0 +1,31 @@
+import { registerHooks } from 'node:module';
+
+import { expect, it } from 'vitest';
+
+const packageName = process.env['PACKAGE_NAME'] ?? '';
+const scope = packageName.slice(0, packageName.indexOf('/'));
+const aliases = (process.env['PACKAGE_ALIASED_NAMES'] ?? '')
+  .split(' ')
+  .map((name): readonly [string, string] => [`${scope}/${name}`, `${packageName}/${name}`]);
+const subpaths = (process.env['PACKAGE_SUBPATHS'] ?? '').split(' ');
+
+function aliased(specifier: string): string {
+  const alias = aliases.find(
+    ([name]: readonly [string, string]) => specifier === name || specifier.startsWith(`${name}/`),
+  );
+  return alias === undefined ? specifier : `${alias[1]}${specifier.slice(alias[0].length)}`;
+}
+
+registerHooks({
+  resolve: (specifier: string, context: Readonly<{ parentURL: string | undefined }>, nextResolve) =>
+    nextResolve(aliased(specifier), context),
+});
+
+const outcomes = await Promise.allSettled(subpaths.map((subpath): Promise<unknown> => import(subpath)));
+const failures = outcomes.flatMap((outcome: Readonly<PromiseSettledResult<unknown>>, index) =>
+  outcome.status === 'rejected' ? [`${subpaths[index] ?? ''}: ${String(outcome.reason)}`] : [],
+);
+
+it('imports every subpath of the package, the names of its packages aliased to them', () => {
+  expect(failures).toEqual([]);
+});