diff --git a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs index ef396c79..0fee6c6f 100644 --- a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs +++ b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs @@ -41,13 +41,14 @@ // so any install from this repo or downstream consumer sees a // fresh bundle matching the source. -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; import { buildModuleGraph, parseCommandRegistry, partitionModules } from "./partition.mjs"; +import { SQLITE_WASM_PATH, sqliteWorkerPlugin } from "./sqlite-build-plugin.mjs"; const here = dirname(fileURLToPath(import.meta.url)); // Script lives at .../backends/worker-shell/script/build-bundle.mjs; @@ -141,6 +142,7 @@ try { "seek-bzip", ], plugins: [ + sqliteWorkerPlugin(), // curl reaches undici only through just-bash's DNS-pinning // connection owner, which the Worker backend never activates: // ShellWorker registers curl on the plain-`fetch` path @@ -220,6 +222,8 @@ const partition = partitionModules({ graph, registry, optionalFeatures: OPTIONAL // each by its @cloudflare/computer/shell/ subpath and // spreads them back together. const groupNames = ["core", ...Object.keys(OPTIONAL_FEATURES)]; +const sqliteWasm = await readFile(SQLITE_WASM_PATH); +const sqliteWasmBase64 = sqliteWasm.toString("base64"); let totalBytes = 0; for (const group of groupNames) { const names = (partition[group] ?? []).sort(); @@ -231,18 +235,32 @@ for (const group of groupNames) { const header = `// Generated by script/build-bundle.mjs — do not edit.\n` + `// Shell modules exclusive to the "${group}" feature group.\n`; - const body = `export default Object.freeze(${JSON.stringify( - record, - null, - 2, - )}) as Readonly>;\n`; + let serializedRecord = JSON.stringify(record, null, 2); + let moduleType = "{ js: string }"; + if (group === "sqlite") { + serializedRecord = + `{\n ...${serializedRecord},\n` + + ` "sql-wasm.wasm": {\n` + + ` wasm: Uint8Array.from(atob(${JSON.stringify( + sqliteWasmBase64, + )}), (byte) => byte.charCodeAt(0)).buffer,\n` + + ` },\n}`; + moduleType = "{ js: string } | { wasm: ArrayBuffer }"; + totalBytes += sqliteWasm.byteLength; + } + const body = + `export default Object.freeze(${serializedRecord}) as Readonly>;\n`; await writeFile(resolve(outDir, `${group}.ts`), `${header}\n${body}`); } const coreCount = (partition.core ?? []).length; const mainBytes = modules["shell.js"].length; const featureSummary = Object.keys(OPTIONAL_FEATURES) - .map((f) => `${f} ${(partition[f] ?? []).length}`) + .map((feature) => { + const extraModules = feature === "sqlite" ? 1 : 0; + return `${feature} ${(partition[feature] ?? []).length + extraModules}`; + }) .join(", "); console.log( `Wrote ${outDir} (core ${coreCount} modules, shell.js ${mainBytes} bytes, ` + diff --git a/packages/computer/src/backends/worker-shell/script/sqlite-build-plugin.mjs b/packages/computer/src/backends/worker-shell/script/sqlite-build-plugin.mjs new file mode 100644 index 00000000..3f6b5663 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/sqlite-build-plugin.mjs @@ -0,0 +1,167 @@ +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const justBashEntry = fileURLToPath(import.meta.resolve("just-bash")); +const sqliteWorker = resolve(dirname(justBashEntry), "../commands/sqlite3/worker.js"); +const sqlJsEntry = fileURLToPath(import.meta.resolve("sql.js")); + +export const SQLITE_WASM_PATH = resolve(dirname(sqlJsEntry), "sql-wasm.wasm"); + +const SQLITE_WORKER_ERROR = "sqlite3 worker not found. Run 'pnpm build' to compile the worker."; +const FIND_WORKER = + 'for(let r of t)if(ce(r))return r;throw new Error("' + SQLITE_WORKER_ERROR + '")'; +const CREATE_WORKER = + "return new he(e,{workerData:t,resourceLimits:{maxOldGenerationSizeMb:be,maxYoungGenerationSizeMb:we}})"; + +function replaceExactlyOnce(source, search, replacement, label) { + const first = source.indexOf(search); + if (first === -1 || source.indexOf(search, first + search.length) !== -1) { + throw new Error(`sqlite bundle: expected exactly one ${label}`); + } + return source.slice(0, first) + replacement + source.slice(first + search.length); +} + +export function sqliteWorkerPlugin() { + let commandAdapted = false; + let queryWorkerLoaded = false; + let sqlJsLoaded = false; + + return { + name: "adapt-sqlite-worker", + setup(build) { + // just-bash's generated command chunk looks for a worker file with + // node:fs and constructs node:worker_threads.Worker. Neither mechanism + // can reach a Dynamic Worker Loader module, so route both through a lazy + // adapter that implements the same small event protocol in this isolate. + build.onLoad( + { filter: /[\\/]just-bash[\\/]dist[\\/]bundle[\\/]chunks[\\/]chunk-[^/\\]+\.js$/ }, + async (args) => { + let source = await readFile(args.path, "utf8"); + if (!source.includes(SQLITE_WORKER_ERROR)) return undefined; + + source = replaceExactlyOnce( + source, + FIND_WORKER, + 'return "inline:sqlite3"', + "sqlite worker lookup", + ); + source = replaceExactlyOnce( + source, + CREATE_WORKER, + "return __createInlineSqliteWorker(t)", + "sqlite Worker constructor", + ); + source = + `import { createInlineSqliteWorker as __createInlineSqliteWorker } from ${JSON.stringify( + resolve(here, "sqlite-command-adapter.mjs"), + )};\n` + source; + // Wrap the exported command so WorkspaceFsAdapter gets a stable + // database-lock identity without changing the always-on adapter. + source = replaceExactlyOnce( + source, + "export{$e as a,_e as b,Fe as c};", + "const __sqliteCommand=__adaptSqliteCommand(_e);export{$e as a,__sqliteCommand as b,Fe as c};", + "sqlite command export", + ); + source = + `import { adaptSqliteCommand as __adaptSqliteCommand } from ${JSON.stringify( + resolve(here, "sqlite-command-adapter.mjs"), + )};\n` + source; + commandAdapted = true; + return { contents: source, loader: "js", resolveDir: dirname(args.path) }; + }, + ); + + // Pull the worker's query implementation into the sqlite feature graph. + // Its Node worker entrypoint is guarded by parentPort, which is null here; + // exporting executeQuery lets the adapter call the same implementation. + build.onResolve({ filter: /^computer:sqlite-query-worker$/ }, () => ({ + path: "query-worker", + namespace: "computer-sqlite", + })); + build.onLoad({ filter: /^query-worker$/, namespace: "computer-sqlite" }, async () => { + let source = await readFile(sqliteWorker, "utf8"); + // WorkerDefenseInDepth belongs around a dedicated thread. Running it + // in the shell's isolate would harden the shell itself after a query. + source = replaceExactlyOnce( + source, + " activateDefense();\n", + "", + "worker defense activation", + ); + queryWorkerLoaded = true; + return { + contents: `${source}\nexport { executeQuery };\n`, + loader: "js", + resolveDir: dirname(sqliteWorker), + }; + }); + + // sql.js normally fetches sql-wasm.wasm from a package filesystem. The + // Dynamic Worker instead imports a precompiled module supplied in its + // Loader module table, then hands that module to Emscripten explicitly. + build.onResolve({ filter: /^sql\.js$/ }, () => ({ + path: "sql.js", + namespace: "computer-sqlite", + })); + build.onLoad({ filter: /^sql\.js$/, namespace: "computer-sqlite" }, () => ({ + contents: ` + import initSqlJs from ${JSON.stringify(sqlJsEntry)}; + import sqliteWasm from "./sql-wasm.wasm"; + export default function init(options = {}) { + return initSqlJs({ + ...options, + instantiateWasm(imports, receiveInstance) { + const instance = new WebAssembly.Instance(sqliteWasm, imports); + receiveInstance(instance, sqliteWasm); + return instance.exports; + }, + }); + } + `, + loader: "js", + resolveDir: dirname(sqlJsEntry), + })); + build.onResolve({ filter: /^\.\/sql-wasm\.wasm$/ }, () => ({ + path: "./sql-wasm.wasm", + external: true, + })); + + // sql.js sees WorkerGlobalScope and process shims in workerd and otherwise + // chooses loading branches that require self.location or node:fs. Neither + // branch is needed when instantiateWasm is supplied by the wrapper above. + build.onLoad({ filter: /[\\/]sql\.js[\\/]dist[\\/]sql-wasm\.js$/ }, async (args) => { + let source = await readFile(args.path, "utf8"); + source = replaceExactlyOnce( + source, + "globalThis.WorkerGlobalScope", + "undefined", + "sql.js WorkerGlobalScope probe", + ); + source = replaceExactlyOnce( + source, + "globalThis.process?.versions?.node", + "undefined", + "sql.js Node probe", + ); + sqlJsLoaded = true; + return { contents: source, loader: "js", resolveDir: dirname(args.path) }; + }); + + build.onEnd((result) => { + if (result.errors.length > 0) return; + if (!commandAdapted || !queryWorkerLoaded || !sqlJsLoaded) { + throw new Error( + `sqlite bundle: incomplete adaptation (${JSON.stringify({ + commandAdapted, + queryWorkerLoaded, + sqlJsLoaded, + })})`, + ); + } + }); + }, + }; +} diff --git a/packages/computer/src/backends/worker-shell/script/sqlite-command-adapter.mjs b/packages/computer/src/backends/worker-shell/script/sqlite-command-adapter.mjs new file mode 100644 index 00000000..31c7023d --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/sqlite-command-adapter.mjs @@ -0,0 +1,91 @@ +// just-bash's sqlite3 command executes queries in a Node Worker thread. The +// Workers runtime exposes the node:worker_threads API surface but does not +// implement the Worker constructor. A WorkerShellBackend already runs in its +// own Dynamic Worker isolate, so execute the query worker in that isolate while +// preserving the small EventEmitter protocol just-bash expects. + +import { executeQuery } from "computer:sqlite-query-worker"; + +class InlineSqliteWorker { + #listeners = new Map(); + #terminated = false; + + constructor(workerData) { + queueMicrotask(() => { + void executeQuery(workerData) + .then((result) => { + this.#emit("message", { + ...result, + protocolToken: workerData.protocolToken, + }); + }) + .catch((error) => this.#emit("error", error)); + }); + } + + on(event, listener) { + let listeners = this.#listeners.get(event); + if (listeners === undefined) { + listeners = new Set(); + this.#listeners.set(event, listeners); + } + listeners.add(listener); + return this; + } + + removeListener(event, listener) { + this.#listeners.get(event)?.delete(listener); + return this; + } + + async terminate() { + this.#terminated = true; + return 0; + } + + #emit(event, value) { + if (this.#terminated) return; + for (const listener of this.#listeners.get(event) ?? []) listener(value); + } +} + +// WorkspaceFsAdapter has no Node dev/ino pair because its storage is remote. +// It also does not support hard links, so a canonical path is a stable lock +// identity for the database. Keep this compatibility layer in SQLite's lazy +// chunk instead of adding bytes to every Worker shell. +function filesystemWithStableIdentity(fs) { + return new Proxy(fs, { + get(target, property) { + if (property === "stat") { + return async (path) => { + const stat = await target.stat(path); + if (stat.identity !== undefined || (stat.dev !== undefined && stat.ino !== undefined)) { + return stat; + } + return { ...stat, identity: await target.realpath(path) }; + }; + } + + // WorkspaceFsAdapter uses private fields, so methods must retain the + // original receiver rather than receiving this Proxy as `this`. + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +export function createInlineSqliteWorker(workerData) { + return new InlineSqliteWorker(workerData); +} + +export function adaptSqliteCommand(command) { + return { + ...command, + execute(args, context) { + return command.execute(args, { + ...context, + fs: filesystemWithStableIdentity(context.fs), + }); + }, + }; +} diff --git a/packages/computer/src/backends/worker-shell/shell-modules.test.ts b/packages/computer/src/backends/worker-shell/shell-modules.test.ts index 445c3683..23a24a55 100644 --- a/packages/computer/src/backends/worker-shell/shell-modules.test.ts +++ b/packages/computer/src/backends/worker-shell/shell-modules.test.ts @@ -136,6 +136,21 @@ describe("shell feature groups", () => { expect(curlOnDisk).toBe(true); }); + it("ships SQLite's query worker and WebAssembly module only in the sqlite group", () => { + expect(sqliteModules["sql-wasm.wasm"]).toMatchObject({ + wasm: expect.any(ArrayBuffer), + }); + expect(SHELL_CORE_MODULES["sql-wasm.wasm"]).toBeUndefined(); + + const sqliteSource = Object.values(sqliteModules) + .filter((module): module is { js: string } => "js" in module) + .map((module) => module.js) + .join("\n"); + expect(sqliteSource).toContain("function executeQuery"); + expect(sqliteSource).toContain("InlineSqliteWorker"); + expect(sqliteSource).not.toContain("sqlite3 worker not found"); + }); + it("keeps feature groups disjoint from each other", () => { // A chunk owned by one feature must not also appear in another; // a shared chunk belongs in core. diff --git a/packages/computer/src/backends/worker-shell/shell-modules.ts b/packages/computer/src/backends/worker-shell/shell-modules.ts index 4a2ea850..d0d0d6f9 100644 --- a/packages/computer/src/backends/worker-shell/shell-modules.ts +++ b/packages/computer/src/backends/worker-shell/shell-modules.ts @@ -19,7 +19,7 @@ import coreModules from "@cloudflare/computer/shell/core"; // One generated feature group: module name -> source string. The // core group and every @cloudflare/computer/shell/ import // share this shape. -export type ShellModuleGroup = Readonly>; +export type ShellModuleGroup = Readonly>; // The always-on core group. Ships in every Worker shell; carries // the ShellWorker entry (shell.js), the base command set, and the @@ -29,10 +29,8 @@ export const SHELL_CORE_MODULES: ShellModuleGroup = Object.freeze({ ...coreModul // Merge the core group with the optional groups the consumer // imported and passed. Later groups win on key collisions, but the // build keeps groups disjoint so order never matters in practice. -export function assembleShellModules( - groups: readonly ShellModuleGroup[] = [], -): Readonly> { - const modules: Record = { ...coreModules }; +export function assembleShellModules(groups: readonly ShellModuleGroup[] = []): ShellModuleGroup { + const modules: Record = { ...coreModules }; for (const group of groups) { Object.assign(modules, group); } diff --git a/packages/computer/src/backends/worker-shell/worker-shell.ts b/packages/computer/src/backends/worker-shell/worker-shell.ts index 981059d1..59ea1c86 100644 --- a/packages/computer/src/backends/worker-shell/worker-shell.ts +++ b/packages/computer/src/backends/worker-shell/worker-shell.ts @@ -63,13 +63,40 @@ export interface WorkerShellLoader { }; } +interface WorkerShellSocketInfo { + remoteAddress?: string; + localAddress?: string; +} + +interface WorkerShellSocket { + readonly readable: ReadableStream; + readonly writable: WritableStream; + readonly closed: Promise; + readonly opened: Promise; + readonly upgraded: boolean; + readonly secureTransport: "on" | "off" | "starttls"; + close(): Promise; + startTls(options?: unknown): WorkerShellSocket; +} + +interface WorkerShellOutbound { + fetch(input: RequestInfo | URL, init?: RequestInit): Promise; + connect( + address: string | { hostname: string; port: number }, + options?: unknown, + ): WorkerShellSocket; +} + interface WorkerLoaderCode { compatibilityDate: string; compatibilityFlags?: string[]; mainModule: string; - modules: Record; + modules: Record< + string, + string | { js?: string; cjs?: string; text?: string; wasm?: ArrayBuffer } + >; env?: Record; - globalOutbound?: unknown; + globalOutbound?: WorkerShellOutbound | null; } // Subset of DurableObjectState the backend needs. ctx.exports is diff --git a/packages/computer/tests/worker-backend-worker.ts b/packages/computer/tests/worker-backend-worker.ts index a116d541..7a55e421 100644 --- a/packages/computer/tests/worker-backend-worker.ts +++ b/packages/computer/tests/worker-backend-worker.ts @@ -19,6 +19,7 @@ import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; import curlModules from "@cloudflare/computer/shell/curl"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; import { WorkerShellBackend } from "../src/backends/worker-shell/index.js"; import type { DurableObjectStorageLike, WorkspaceStub } from "../src/index.js"; import { Workspace } from "../src/index.js"; @@ -45,7 +46,7 @@ export class HostDO extends DurableObject { ctx, // Opt curl in by importing its group and passing it; the // fetch-path curl integration test exercises the wiring. - commands: [curlModules], + commands: [curlModules, sqliteModules], }), ], }); diff --git a/packages/computer/tests/worker-backend.test.ts b/packages/computer/tests/worker-backend.test.ts index 21f12882..3243654c 100644 --- a/packages/computer/tests/worker-backend.test.ts +++ b/packages/computer/tests/worker-backend.test.ts @@ -147,6 +147,18 @@ describe("WorkerShellBackend end-to-end", () => { expect(result.stdout).not.toMatch(/command not found/); }); + it("runs sqlite queries and persists the database in the host filesystem", async () => { + const id = freshId(); + const create = await exec( + id, + `sqlite3 data.db "CREATE TABLE notes(body TEXT); INSERT INTO notes VALUES ('hello');"`, + ); + expect(create).toEqual({ exitCode: 0, stdout: "", stderr: "" }); + + const query = await exec(id, `sqlite3 data.db "SELECT body FROM notes;"`); + expect(query).toEqual({ exitCode: 0, stdout: "hello\n", stderr: "" }); + }); + it("isolates state between separate workspace ids", async () => { // Two host-DO names → two distinct workspaces, two distinct // Dynamic Worker isolates (the loader caches by