diff --git a/.changeset/snapshot-runtime-core.md b/.changeset/snapshot-runtime-core.md new file mode 100644 index 0000000000..80ee5b3094 --- /dev/null +++ b/.changeset/snapshot-runtime-core.md @@ -0,0 +1,5 @@ +--- +"@workflow/core": minor +--- + +Add a new QuickJS WASM-based snapshot runtime that suspends and resumes workflows by serializing the VM heap. Now the default; the previous event-replay runtime remains available via `WORKFLOW_RUNTIME=replay`. diff --git a/.changeset/snapshot-runtime-world-local.md b/.changeset/snapshot-runtime-world-local.md new file mode 100644 index 0000000000..3cd3baf27e --- /dev/null +++ b/.changeset/snapshot-runtime-world-local.md @@ -0,0 +1,5 @@ +--- +"@workflow/world-local": minor +--- + +Add filesystem-backed snapshot storage (`snapshots.save` / `load` / `delete`) for the new snapshot runtime in `@workflow/core`. Also fixes a race in `events.create()` where concurrent `step_created` / `wait_created` writes with the same `correlationId` would both succeed. diff --git a/.changeset/snapshot-runtime-world-postgres.md b/.changeset/snapshot-runtime-world-postgres.md new file mode 100644 index 0000000000..3982920c55 --- /dev/null +++ b/.changeset/snapshot-runtime-world-postgres.md @@ -0,0 +1,5 @@ +--- +"@workflow/world-postgres": minor +--- + +Add a `workflow_snapshots` table and `snapshots.save` / `load` / `delete` storage for the new snapshot runtime in `@workflow/core`. Also fixes a race in `events.create()` where concurrent `step_created` / `hook_created` / `wait_created` writes with the same `correlationId` would persist duplicate event rows. diff --git a/.changeset/snapshot-runtime-world-vercel.md b/.changeset/snapshot-runtime-world-vercel.md new file mode 100644 index 0000000000..9f73cc93bb --- /dev/null +++ b/.changeset/snapshot-runtime-world-vercel.md @@ -0,0 +1,5 @@ +--- +"@workflow/world-vercel": minor +--- + +Add snapshot storage (PUT/GET/DELETE `/v2/runs/:runId/snapshot`) for the new snapshot runtime in `@workflow/core`. Switches the save path from `fetch()` to `undici.request()` so the `RetryAgent` can replay multi-MB snapshot bodies on transient errors. diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 273da9c088..a2cc8bf513 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -190,8 +190,12 @@ jobs: APP_NAME: "nextjs-turbopack" vitest-plugin: - name: Vitest Plugin Tests + name: Vitest Plugin Tests (${{ matrix.runtime }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + runtime: [snapshot, replay] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} @@ -212,9 +216,11 @@ jobs: - name: Run Vitest Plugin Tests run: pnpm test working-directory: workbench/vitest + env: + WORKFLOW_RUNTIME: ${{ matrix.runtime }} e2e-vercel-prod: - name: E2E Vercel Prod Tests (${{ matrix.app.name }}) + name: E2E Vercel Prod Tests (${{ matrix.app.name }} - ${{ matrix.runtime }}) runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -224,6 +230,7 @@ jobs: strategy: fail-fast: false matrix: + runtime: [snapshot, replay] app: - name: "example" project-id: "prj_xWq20Dd860HHAfzMjK2Mb6TPVxMa" @@ -290,12 +297,13 @@ jobs: environment: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} - name: Run E2E Tests - run: pnpm run test:e2e --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-prod-$APP_NAME.json" + run: pnpm run test:e2e --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts "--outputFile=e2e-vercel-prod-$APP_NAME-$WORKFLOW_RUNTIME.json" env: NODE_OPTIONS: "--enable-source-maps" DEPLOYMENT_URL: ${{ steps.waitForDeployment.outputs.deployment-url }} VERCEL_DEPLOYMENT_ID: ${{ steps.waitForDeployment.outputs.deployment-id }} APP_NAME: ${{ matrix.app.name }} + WORKFLOW_RUNTIME: ${{ matrix.runtime }} WORKFLOW_VERCEL_ENV: ${{ github.ref == 'refs/heads/main' && 'production' || 'preview' }} WORKFLOW_VERCEL_AUTH_TOKEN: ${{ secrets.VERCEL_LABS_TOKEN }} WORKFLOW_VERCEL_TEAM: "team_nO2mCG4W8IxPIeKoSsqwAxxB" @@ -317,15 +325,16 @@ jobs: if: always() env: APP_NAME: ${{ matrix.app.name }} - run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Vercel Prod ($APP_NAME)" >> $GITHUB_STEP_SUMMARY || true + WORKFLOW_RUNTIME: ${{ matrix.runtime }} + run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Vercel Prod ($APP_NAME - $WORKFLOW_RUNTIME)" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@v4 with: - name: e2e-results-vercel-prod-${{ matrix.app.name }} + name: e2e-results-vercel-prod-${{ matrix.app.name }}-${{ matrix.runtime }} path: | - e2e-vercel-prod-${{ matrix.app.name }}.json + e2e-vercel-prod-${{ matrix.app.name }}-${{ matrix.runtime }}.json e2e-metadata-${{ matrix.app.name }}-vercel.json e2e-failures-${{ matrix.app.name }}-vercel.json e2e-diagnostics-${{ matrix.app.name }}-vercel.json @@ -415,6 +424,7 @@ jobs: env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: ${{ matrix.app.name }} + WORKFLOW_RUNTIME: ${{ matrix.app.runtime }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '5173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" DEV_TEST_CONFIG: ${{ toJSON(matrix.app) }} @@ -496,6 +506,7 @@ jobs: env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: ${{ matrix.app.name }} + WORKFLOW_RUNTIME: ${{ matrix.app.runtime }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} @@ -596,6 +607,7 @@ jobs: env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: ${{ matrix.app.name }} + WORKFLOW_RUNTIME: ${{ matrix.app.runtime }} WORKBENCH_APP_PATH: ${{ steps.prepare-workbench.outputs.workbench_app_path }} DEPLOYMENT_URL: "http://localhost:${{ matrix.app.name == 'sveltekit' && '4173' || (matrix.app.name == 'astro' && '4321' || '3000') }}" NEXT_CANARY: ${{ matrix.app.canary && '1' || '' }} @@ -614,10 +626,14 @@ jobs: if-no-files-found: ignore e2e-windows: - name: E2E Windows Tests + name: E2E Windows Tests (${{ matrix.runtime }}) runs-on: windows-latest timeout-minutes: 30 if: ${{ !contains(github.event.pull_request.labels.*.name, 'workflow-server-test') }} + strategy: + fail-fast: false + matrix: + runtime: [snapshot, replay] env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} TURBO_TEAM: ${{ vars.TURBO_TEAM }} @@ -654,7 +670,10 @@ jobs: run: | cd workbench/nextjs-turbopack $logFile = "$env:GITHUB_WORKSPACE/nextjs-server.log" - $job = Start-Job -ScriptBlock { Set-Location $using:PWD; pnpm dev *>&1 | Tee-Object -FilePath $using:logFile } + # `$using:` only resolves PowerShell variables, not env vars, so + # copy MATRIX_RUNTIME into a session variable before Start-Job. + $matrixRuntime = $env:MATRIX_RUNTIME + $job = Start-Job -ScriptBlock { Set-Location $using:PWD; $env:WORKFLOW_RUNTIME = $using:matrixRuntime; pnpm dev *>&1 | Tee-Object -FilePath $using:logFile } Start-Sleep -Seconds 15 cd ../.. @@ -708,7 +727,7 @@ jobs: exit 1 } - pnpm run test:e2e --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-windows-nextjs-turbopack.json + pnpm run test:e2e --reporter=default --reporter=json --reporter=./packages/core/e2e/github-reporter.ts --outputFile=e2e-windows-nextjs-turbopack-$env:MATRIX_RUNTIME.json $e2eExit = $LASTEXITCODE Stop-Job $job -ErrorAction SilentlyContinue exit $e2eExit @@ -716,6 +735,8 @@ jobs: env: NODE_OPTIONS: "--enable-source-maps" APP_NAME: "nextjs-turbopack" + WORKFLOW_RUNTIME: ${{ matrix.runtime }} + MATRIX_RUNTIME: ${{ matrix.runtime }} DEPLOYMENT_URL: "http://localhost:3000" DEV_TEST_CONFIG: '{"generatedStepPath":"app/.well-known/workflow/v1/flow/__step_registrations.js","generatedWorkflowPath":"app/.well-known/workflow/v1/flow/route.js","apiFilePath":"app/api/chat/route.ts","apiFileImportPath":"../../..","port":3000}' @@ -735,14 +756,14 @@ jobs: - name: Generate E2E summary if: always() shell: bash - run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Windows (nextjs-turbopack)" >> $GITHUB_STEP_SUMMARY || true + run: node .github/scripts/aggregate-e2e-results.js . --job-name "E2E Windows (nextjs-turbopack - ${{ matrix.runtime }})" >> $GITHUB_STEP_SUMMARY || true - name: Upload E2E results if: always() uses: actions/upload-artifact@v4 with: - name: e2e-results-windows-nextjs-turbopack - path: e2e-windows-nextjs-turbopack.json + name: e2e-results-windows-nextjs-turbopack-${{ matrix.runtime }} + path: e2e-windows-nextjs-turbopack-${{ matrix.runtime }}.json retention-days: 7 if-no-files-found: ignore @@ -750,7 +771,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: nextjs-server-logs-windows + name: nextjs-server-logs-windows-${{ matrix.runtime }} path: nextjs-server.log retention-days: 7 if-no-files-found: ignore @@ -844,6 +865,7 @@ jobs: echo "postgres=$POSTGRES_STATUS" >> $GITHUB_OUTPUT echo "windows=$WINDOWS_STATUS" >> $GITHUB_OUTPUT + # Community world failures are warnings; everything else is a hard failure if [[ "$VERCEL_STATUS" == "failure" || "$LOCAL_DEV_STATUS" == "failure" || "$LOCAL_PROD_STATUS" == "failure" || "$POSTGRES_STATUS" == "failure" || "$WINDOWS_STATUS" == "failure" ]]; then echo "has_failures=true" >> $GITHUB_OUTPUT else @@ -875,6 +897,21 @@ jobs: Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + - name: Append community warning to PR comment + if: github.event_name == 'pull_request' && steps.check-status.outputs.has_warnings == 'true' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: e2e-test-results + append: true + message: | + + --- + ⚠️ **Community world tests failed** (non-blocking): + - Community Worlds: ${{ needs.e2e-community.result }} + + Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + + # Final required check: passes only when unit + all E2E jobs succeed. # Community worlds are intentionally excluded — they are disabled on main. e2e-required-check: diff --git a/packages/core/.gitignore b/packages/core/.gitignore index 7b6d0b4576..51bcd14a90 100644 --- a/packages/core/.gitignore +++ b/packages/core/.gitignore @@ -1,2 +1,9 @@ # Auto-generated version file src/version.ts + +# Auto-generated quickjs-wasi binary assets (base64-encoded WASM + .so files) +src/runtime/quickjs-assets.generated.ts + +# Auto-generated VM serde bundle (devalue + format-prefix + reducers, +# packaged as an ES-module string for evaluation inside the QuickJS VM) +src/runtime/vm-serde-bundle.generated.ts diff --git a/packages/core/package.json b/packages/core/package.json index 20f09eb728..f90e7881cf 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -71,7 +71,7 @@ "./_workflow": "./dist/workflow/index.js" }, "scripts": { - "build": "genversion --es6 src/version.ts && tsc", + "build": "genversion --es6 src/version.ts && node scripts/build-vm-serde-bundle.js && node scripts/build-quickjs-assets.js && tsc", "dev": "genversion --es6 src/version.ts && tsc --watch", "clean": "tsc --build --clean && rm -rf dist src/version.ts docs ||:", "test": "cross-env WORKFLOW_TARGET_WORLD=local vitest run src", @@ -96,6 +96,7 @@ "devalue": "5.6.3", "ms": "2.1.3", "nanoid": "5.1.6", + "quickjs-wasi": "2.0.0", "seedrandom": "3.0.5", "semver": "catalog:", "ulid": "catalog:", diff --git a/packages/core/scripts/build-quickjs-assets.js b/packages/core/scripts/build-quickjs-assets.js new file mode 100644 index 0000000000..67764ccae5 --- /dev/null +++ b/packages/core/scripts/build-quickjs-assets.js @@ -0,0 +1,73 @@ +/** + * Build script: generates quickjs-assets.generated.ts + * + * Reads the quickjs-wasi WASM binary and native C extension .so files, + * base64-encodes them, and writes a TypeScript module that exports the + * decoded Buffer/Uint8Array values. This embeds the binaries directly + * in JavaScript, bypassing all bundler/framework/deployment issues with + * import.meta.url, require.resolve, and file tracing. + */ + +import { readFileSync, writeFileSync } from 'fs'; +import { createRequire } from 'module'; +import { dirname, join, resolve } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const srcDir = resolve(__dirname, '../src'); + +const require_ = createRequire(import.meta.url); +const quickjsDir = dirname(dirname(require_.resolve('quickjs-wasi'))); + +const files = { + quickjsWasm: join(quickjsDir, 'quickjs.wasm'), + encodingSo: join(quickjsDir, 'extensions/encoding/encoding.so'), + base64So: join(quickjsDir, 'extensions/base64/base64.so'), + headersSo: join(quickjsDir, 'extensions/headers/headers.so'), + urlSo: join(quickjsDir, 'extensions/url/url.so'), + structuredCloneSo: join( + quickjsDir, + 'extensions/structured-clone/structured-clone.so' + ), +}; + +let output = `/** + * Auto-generated by scripts/build-quickjs-assets.js + * Do not edit manually. + * + * Contains base64-encoded quickjs-wasi WASM binary and native C extension + * .so files. Decoded at import time so they can be passed directly to + * QuickJS.create() and QuickJS.restore() without any filesystem access, + * import.meta.url resolution, or require.resolve calls. + */ +import type { ExtensionDescriptor } from 'quickjs-wasi'; + +`; + +let totalSize = 0; + +for (const [name, filePath] of Object.entries(files)) { + const buf = readFileSync(filePath); + const b64 = buf.toString('base64'); + totalSize += buf.length; + output += `const ${name} = Buffer.from('${b64}', 'base64');\n\n`; +} + +output += `export { quickjsWasm };\n\n`; + +output += `export const quickjsExtensions: ExtensionDescriptor[] = [ + { name: 'encoding', wasm: encodingSo }, + { name: 'base64', wasm: base64So }, + { name: 'headers', wasm: headersSo }, + { name: 'url', wasm: urlSo }, + { name: 'structured-clone', wasm: structuredCloneSo, initFn: 'qjs_ext_structured_clone_init' }, +];\n`; + +const outPath = resolve(srcDir, 'runtime/quickjs-assets.generated.ts'); +writeFileSync(outPath, output); + +const sizeKB = (totalSize / 1024).toFixed(0); +const b64SizeKB = (Buffer.byteLength(output) / 1024).toFixed(0); +console.log( + `Generated quickjs-assets.generated.ts (${sizeKB} KB binary → ${b64SizeKB} KB base64)` +); diff --git a/packages/core/scripts/build-vm-serde-bundle.js b/packages/core/scripts/build-vm-serde-bundle.js new file mode 100644 index 0000000000..f738853b9a --- /dev/null +++ b/packages/core/scripts/build-vm-serde-bundle.js @@ -0,0 +1,65 @@ +/** + * Build script: generates the VM serialization bundle. + * + * Uses esbuild to bundle workflow-vm.ts into a self-contained IIFE. + * The output is written as a TypeScript file containing the bundle as + * a string constant, which can be imported by the snapshot runtime. + * + * TextEncoder, TextDecoder, and Headers are provided by native C + * extensions in quickjs-wasi, so no JS polyfills are needed. + */ + +import { buildSync } from 'esbuild'; +import { writeFileSync } from 'fs'; +import { dirname, resolve } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const srcDir = resolve(__dirname, '../src'); + +const result = buildSync({ + entryPoints: [resolve(srcDir, 'serialization/vm-bundle-entry.ts')], + // NOTE: TextEncoder, TextDecoder, and Headers are provided by native + // C extensions (encoding, headers) in quickjs-wasi, so the polyfill + // injection that was previously here has been removed. + bundle: true, + format: 'iife', + platform: 'neutral', + target: 'es2020', + write: false, + minify: true, +}); + +const bundleCode = result.outputFiles[0].text; + +// Write as a TS module using a template literal. Template literals avoid +// the escaping issues that occur with regular string literals — esbuild's +// minifier produces patterns like `typeof x<"u"` whose escaped quotes +// inside a JSON-stringified string break when downstream esbuild (e.g., +// Nitro) re-processes the compiled JS output. Template literals don't +// have this problem since backticks don't conflict with inner quotes. +const escaped = bundleCode + .replace(/\\/g, '\\\\') + .replace(/`/g, '\\`') + .replace(/\$\{/g, '\\${'); + +const outPath = resolve(srcDir, 'runtime/vm-serde-bundle.generated.ts'); +writeFileSync( + outPath, + `/** + * Auto-generated by scripts/build-vm-serde-bundle.js + * Do not edit manually. + * + * This is the VM serialization bundle — a self-contained IIFE that sets up + * serialize/deserialize + TextEncoder/TextDecoder polyfills inside the + * QuickJS WASM VM. It includes devalue and all workflow-mode reducers. + * + * Size: ${(bundleCode.length / 1024).toFixed(1)} KB minified + */ +export const VM_SERDE_BUNDLE: string = \`${escaped}\`; +` +); + +console.log( + `Generated vm-serde-bundle.generated.ts (${(bundleCode.length / 1024).toFixed(1)} KB)` +); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 3c2bfb8f01..c3799902ad 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -32,6 +32,11 @@ import { queueMessage, withHealthCheck, } from './runtime/helpers.js'; +import { + getWorkflowRuntimeFromEnv, + WORKFLOW_RUNTIMES, +} from './runtime/runtime-mode.js'; +import { runWorkflowWithSnapshots } from './runtime/snapshot-entrypoint.js'; import { executeStep } from './runtime/step-executor.js'; import { handleSuspension } from './runtime/suspension-handler.js'; import { @@ -53,6 +58,32 @@ import { getErrorName, getErrorStack, normalizeUnknownError } from './types.js'; import { buildWorkflowSuspensionMessage } from './util.js'; import { runWorkflow } from './workflow.js'; +/** + * Whether to use the snapshot-based workflow runtime for a given run. + * + * The snapshot runtime is the default. It can be disabled globally via + * WORKFLOW_RUNTIME=replay env var, or per-run via + * executionContext.workflowRuntime = 'replay' (set by the SDK at start()). + * The per-run setting allows the same deployment to serve both runtimes. + * + * Throws if `WORKFLOW_RUNTIME` is set to an unknown value, or if the run's + * `executionContext.workflowRuntime` is set to an unknown value. + */ +function useSnapshotRuntime(workflowRun: WorkflowRun): boolean { + if (getWorkflowRuntimeFromEnv() === 'replay') return false; + const runtimeFromRun = workflowRun.executionContext?.workflowRuntime; + if (runtimeFromRun !== undefined) { + if (!(WORKFLOW_RUNTIMES as readonly string[]).includes(runtimeFromRun)) { + throw new WorkflowRuntimeError( + `Invalid executionContext.workflowRuntime value: "${runtimeFromRun}". ` + + `Expected one of: ${WORKFLOW_RUNTIMES.join(', ')}.` + ); + } + if (runtimeFromRun === 'replay') return false; + } + return true; +} + export type { Event, WorkflowRun }; export { WorkflowSuspension } from './global.js'; export { @@ -576,6 +607,33 @@ export function workflowEntrypoint( } } // end if (!workflowRun) + // --- Snapshot runtime dispatch --- + // The snapshot runtime is a self-contained alternative to + // the V2 inline-replay loop below. It runs the workflow in + // a QuickJS VM, persists snapshots between suspensions, + // queues steps via the same combined route (so they hit + // executeStep below on re-entry), and manages its own + // run_completed / run_failed lifecycle. When snapshot + // mode is in effect, return immediately after dispatch. + if (useSnapshotRuntime(workflowRun)) { + runtimeLogger.debug('Using snapshot runtime', { + workflowRunId: runId, + loopIteration, + }); + const snapshotResult = await runWorkflowWithSnapshots({ + workflowCode, + workflowName, + workflowRun, + preloadedEvents, + runInput, + parentSpan: span, + }); + if (snapshotResult?.timeoutSeconds !== undefined) { + return { timeoutSeconds: snapshotResult.timeoutSeconds }; + } + return; + } + // Resolve the encryption key for this run's deployment. // Used eagerly here since both runWorkflow (input // hydration / hook payload decryption) and the run_failed diff --git a/packages/core/src/runtime/get-port-lazy.ts b/packages/core/src/runtime/get-port-lazy.ts index e284df1228..7bb071dc3c 100644 --- a/packages/core/src/runtime/get-port-lazy.ts +++ b/packages/core/src/runtime/get-port-lazy.ts @@ -14,20 +14,39 @@ let _getPort: (() => Promise) | undefined; export async function getPortLazy(): Promise { if (!_getPort) { + // Construct specifier at runtime to defeat bundler static analysis. + const spec = ['@workflow/utils', 'get-port'].join('/'); + // Two resolution attempts so this works in both pnpm-strict app + // bundles (where the app's package.json doesn't list + // @workflow/utils as a direct dep) and in re-bundled CJS outputs: + // + // 1) Resolve from process.cwd() — works for hoisted-node_modules + // layouts where @workflow/utils is reachable from the app root. + // 2) Fall back to this module's own location — works when the + // consumer is pnpm-strict (transitive deps invisible from cwd) + // but @workflow/utils IS available as a peer of @workflow/core. + // + // Mirrors the dual-resolution pattern in `world.ts:getRuntimeRequire`. + let mod: { getPort?: () => Promise } | undefined; try { - // Construct specifier at runtime to defeat bundler static analysis. - const spec = ['@workflow/utils', 'get-port'].join('/'); - // Use process.cwd()-based createRequire for CJS/ESM compatibility. - // import.meta.url is unavailable in CJS re-bundled outputs. const _require = createRequire( pathToFileURL(process.cwd() + '/package.json').href ); - const mod = _require(spec); - _getPort = mod.getPort; + mod = _require(spec); } catch { - // Module not available (e.g., in a browser or minimal bundle) - _getPort = async () => undefined; + try { + // import.meta.url is undefined in CJS re-bundled outputs, but + // when it's present it points at @workflow/core's own location + // where @workflow/utils is always installed as a dep. + if (typeof import.meta?.url === 'string') { + const _require = createRequire(import.meta.url); + mod = _require(spec); + } + } catch { + // Fall through to undefined-getPort fallback + } } + _getPort = mod?.getPort ?? (async () => undefined); } return _getPort!(); } diff --git a/packages/core/src/runtime/runtime-mode.test.ts b/packages/core/src/runtime/runtime-mode.test.ts new file mode 100644 index 0000000000..ff0698e79f --- /dev/null +++ b/packages/core/src/runtime/runtime-mode.test.ts @@ -0,0 +1,67 @@ +import { WorkflowRuntimeError } from '@workflow/errors'; +import { describe, expect, it } from 'vitest'; +import { + getWorkflowRuntimeFromEnv, + WORKFLOW_RUNTIMES, +} from './runtime-mode.js'; + +describe('getWorkflowRuntimeFromEnv', () => { + it('returns undefined when WORKFLOW_RUNTIME is not set', () => { + expect(getWorkflowRuntimeFromEnv({})).toBeUndefined(); + }); + + it('returns undefined when WORKFLOW_RUNTIME is empty', () => { + expect(getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: '' })).toBeUndefined(); + }); + + it('returns "snapshot" when WORKFLOW_RUNTIME=snapshot', () => { + expect(getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'snapshot' })).toBe( + 'snapshot' + ); + }); + + it('returns "replay" when WORKFLOW_RUNTIME=replay', () => { + expect(getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'replay' })).toBe( + 'replay' + ); + }); + + it('throws WorkflowRuntimeError on unknown values', () => { + expect(() => + getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'bogus' }) + ).toThrow(WorkflowRuntimeError); + expect(() => + getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'bogus' }) + ).toThrow(/Invalid WORKFLOW_RUNTIME value: "bogus"/); + }); + + it('is case-sensitive: uppercase values are rejected', () => { + expect(() => + getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'SNAPSHOT' }) + ).toThrow(WorkflowRuntimeError); + expect(() => + getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'Replay' }) + ).toThrow(WorkflowRuntimeError); + }); + + it('rejects leading/trailing whitespace', () => { + expect(() => + getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: ' snapshot' }) + ).toThrow(WorkflowRuntimeError); + expect(() => + getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'replay ' }) + ).toThrow(WorkflowRuntimeError); + }); + + it('error message lists valid options', () => { + try { + getWorkflowRuntimeFromEnv({ WORKFLOW_RUNTIME: 'bogus' }); + expect.fail('expected to throw'); + } catch (err) { + expect(err).toBeInstanceOf(WorkflowRuntimeError); + for (const mode of WORKFLOW_RUNTIMES) { + expect((err as Error).message).toContain(mode); + } + } + }); +}); diff --git a/packages/core/src/runtime/runtime-mode.ts b/packages/core/src/runtime/runtime-mode.ts new file mode 100644 index 0000000000..b51c2a97dd --- /dev/null +++ b/packages/core/src/runtime/runtime-mode.ts @@ -0,0 +1,38 @@ +/** + * Runtime mode selection for workflows. + * + * The snapshot runtime is the default. The event-replay runtime is opt-in + * via the `WORKFLOW_RUNTIME` env var or `executionContext.workflowRuntime`. + */ + +import { WorkflowRuntimeError } from '@workflow/errors'; + +/** + * Known workflow runtime modes. Any other `WORKFLOW_RUNTIME` value is + * treated as a misconfiguration and rejected at startup. + */ +export const WORKFLOW_RUNTIMES = ['snapshot', 'replay'] as const; + +export type WorkflowRuntimeMode = (typeof WORKFLOW_RUNTIMES)[number]; + +/** + * Read and validate the `WORKFLOW_RUNTIME` env var. + * + * Returns the configured mode, or `undefined` if unset/empty. + * Throws {@link WorkflowRuntimeError} if the value is set but not one of + * the known modes — catching misconfiguration early is better than + * silently falling back to the default. + */ +export function getWorkflowRuntimeFromEnv( + env: NodeJS.ProcessEnv = process.env +): WorkflowRuntimeMode | undefined { + const raw = env.WORKFLOW_RUNTIME; + if (raw === undefined || raw === '') return undefined; + if ((WORKFLOW_RUNTIMES as readonly string[]).includes(raw)) { + return raw as WorkflowRuntimeMode; + } + throw new WorkflowRuntimeError( + `Invalid WORKFLOW_RUNTIME value: "${raw}". ` + + `Expected one of: ${WORKFLOW_RUNTIMES.join(', ')}.` + ); +} diff --git a/packages/core/src/runtime/snapshot-encryption.test.ts b/packages/core/src/runtime/snapshot-encryption.test.ts new file mode 100644 index 0000000000..d11f3fc4e6 --- /dev/null +++ b/packages/core/src/runtime/snapshot-encryption.test.ts @@ -0,0 +1,216 @@ +/** + * Verifies the contract the snapshot runtime relies on when wrapping + * `world.snapshots.save()` and `world.snapshots.load()` with encryption. + */ + +import { WorkflowRuntimeError } from '@workflow/errors'; +import { describe, expect, it } from 'vitest'; +import { importKey } from '../encryption.js'; +import { + compress, + decompress, + PREFERRED_CODEC, +} from '../serialization/compression.js'; +import { + decrypt as decryptSerializedData, + encrypt as encryptSerializedData, +} from '../serialization/encryption.js'; +import { + decodeFormatPrefix, + peekFormatPrefix, +} from '../serialization/format.js'; +import { SerializationFormat } from '../serialization/types.js'; + +async function makeKey() { + const raw = new Uint8Array(32); + for (let i = 0; i < raw.length; i++) raw[i] = (i * 7 + 3) & 0xff; + return importKey(raw); +} + +function bytesOf(str: string): Uint8Array { + return new TextEncoder().encode(str); +} + +describe('snapshot encryption', () => { + it('round-trips with a key', async () => { + const key = await makeKey(); + const plaintext = bytesOf('pretend this is a QuickJS VM snapshot'); + const encrypted = (await encryptSerializedData( + plaintext, + key + )) as Uint8Array; + expect(peekFormatPrefix(encrypted)).toBe(SerializationFormat.ENCRYPTED); + const decrypted = (await decryptSerializedData( + encrypted, + key + )) as Uint8Array; + expect(decrypted.length).toBe(plaintext.length); + for (let i = 0; i < plaintext.length; i++) { + expect(decrypted[i]).toBe(plaintext[i]); + } + }); + + it('passes bytes through unchanged when no key is provided (save)', async () => { + const plaintext = bytesOf('unencrypted snapshot'); + const result = await encryptSerializedData(plaintext, undefined); + // Same reference — no wrapping happened. + expect(result).toBe(plaintext); + }); + + it('does not mark unencrypted bytes with the "encr" prefix', async () => { + // Contract: peekFormatPrefix() returns "encr" only for encrypted data. + // Binary QuickJS snapshots start with arbitrary bytes that may + // coincidentally match [a-z0-9]{4}, but never "encr" unless we actually + // encrypted. + const plaintext = bytesOf('plaintext'); + const result = (await encryptSerializedData( + plaintext, + undefined + )) as Uint8Array; + expect(peekFormatPrefix(result)).not.toBe(SerializationFormat.ENCRYPTED); + }); + + it('passes plaintext bytes through unchanged on load (legacy compat)', async () => { + const plaintext = bytesOf('pre-encryption snapshot from an older run'); + const result = await decryptSerializedData(plaintext, undefined); + expect(result).toBe(plaintext); + + const key = await makeKey(); + const resultWithKey = await decryptSerializedData(plaintext, key); + expect(resultWithKey).toBe(plaintext); + }); + + it('fails loud when loading encrypted data without a key', async () => { + const key = await makeKey(); + const encrypted = (await encryptSerializedData( + bytesOf('encrypted'), + key + )) as Uint8Array; + + await expect( + decryptSerializedData(encrypted, undefined) + ).rejects.toBeInstanceOf(WorkflowRuntimeError); + await expect(decryptSerializedData(encrypted, undefined)).rejects.toThrow( + /no encryption key is available/ + ); + }); + + it('decrypt with the wrong key fails', async () => { + const keyA = await makeKey(); + const rawB = new Uint8Array(32).fill(0x99); + const keyB = await importKey(rawB); + const encrypted = (await encryptSerializedData( + bytesOf('confidential'), + keyA + )) as Uint8Array; + + await expect(decryptSerializedData(encrypted, keyB)).rejects.toThrow(); + }); +}); + +describe('snapshot save/load pipeline (compress → encrypt → decrypt → decompress)', () => { + // Generate a payload large and redundant enough that compression + // observably shrinks it. Bytes are deterministic so the test is + // reproducible; the pattern mimics the kind of redundant string-table + // / AST data that QuickJS heaps contain. + function fakeSnapshot(sizeBytes: number): Uint8Array { + const out = new Uint8Array(sizeBytes); + const pattern = new TextEncoder().encode( + 'function workflow() { return { name: "test" }; }\n' + ); + for (let i = 0; i < sizeBytes; i++) { + out[i] = pattern[i % pattern.length]!; + } + return out; + } + + it('full save → load round-trip preserves snapshot bytes (with key)', async () => { + const key = await makeKey(); + const snapshot = fakeSnapshot(64 * 1024); // 64 KB + + // SAVE pipeline: compress → encrypt + const compressed = compress(snapshot) as Uint8Array; + const encrypted = (await encryptSerializedData( + compressed, + key + )) as Uint8Array; + expect(peekFormatPrefix(encrypted)).toBe(SerializationFormat.ENCRYPTED); + + // LOAD pipeline: decrypt → decompress + const decrypted = (await decryptSerializedData( + encrypted, + key + )) as Uint8Array; + const decompressed = decompress(decrypted) as Uint8Array; + + expect(decompressed.byteLength).toBe(snapshot.byteLength); + // Spot-check the content (full deepEqual is slow on large + // Uint8Arrays). + expect(decompressed[0]).toBe(snapshot[0]); + expect(decompressed[snapshot.byteLength - 1]).toBe( + snapshot[snapshot.byteLength - 1] + ); + }); + + it('full save → load round-trip preserves snapshot bytes (no key)', async () => { + // No-encryption path: we still compress, but encrypt() is a + // pass-through. decrypt() likewise sees no `encr` prefix and + // returns the bytes as-is for decompress() to handle. + const snapshot = fakeSnapshot(32 * 1024); + + const compressed = compress(snapshot) as Uint8Array; + const encrypted = (await encryptSerializedData( + compressed, + undefined + )) as Uint8Array; + // No-key encrypt is a pass-through — same reference, no `encr` wrapper. + expect(encrypted).toBe(compressed); + expect(peekFormatPrefix(encrypted)).not.toBe(SerializationFormat.ENCRYPTED); + + const decrypted = (await decryptSerializedData( + encrypted, + undefined + )) as Uint8Array; + const decompressed = decompress(decrypted) as Uint8Array; + expect(decompressed.byteLength).toBe(snapshot.byteLength); + }); + + it('compressed-then-encrypted bytes are smaller than encrypt-only', async () => { + // The whole point of this layering: encryption produces ~random + // ciphertext that doesn't compress, so doing it the OTHER way + // around (encrypt-then-compress) is wasted work. Verify with a + // redundant payload that compress-first wins. + const key = await makeKey(); + const snapshot = fakeSnapshot(128 * 1024); // 128 KB of repeated string + + // compress-then-encrypt + const compressedThenEncrypted = (await encryptSerializedData( + compress(snapshot), + key + )) as Uint8Array; + + // encrypt-only (the "wrong" baseline) + const encryptedOnly = (await encryptSerializedData( + snapshot, + key + )) as Uint8Array; + + // The compressed pipeline should be a fraction of the size. + // The exact ratio depends on the codec; even gzip-default beats + // 4x on this redundant content. + expect(compressedThenEncrypted.byteLength).toBeLessThan( + encryptedOnly.byteLength / 3 + ); + }); + + it('saves use the preferred codec format prefix', () => { + const snapshot = fakeSnapshot(8 * 1024); + const compressed = compress(snapshot) as Uint8Array; + const { format } = decodeFormatPrefix(compressed); + if (PREFERRED_CODEC === 'zstd') { + expect(format).toBe(SerializationFormat.ZSTD); + } else { + expect(format).toBe(SerializationFormat.GZIP); + } + }); +}); diff --git a/packages/core/src/runtime/snapshot-entrypoint.test.ts b/packages/core/src/runtime/snapshot-entrypoint.test.ts new file mode 100644 index 0000000000..1f5322eb6c --- /dev/null +++ b/packages/core/src/runtime/snapshot-entrypoint.test.ts @@ -0,0 +1,84 @@ +import type { Event } from '@workflow/world'; +import { describe, expect, it } from 'vitest'; +import { canSkipSnapshotLoad } from './snapshot-entrypoint.js'; + +/** + * Helper to build a minimally-shaped Event for tests. Only `eventType` + * is read by `canSkipSnapshotLoad`, the rest are placeholders. + */ +function ev(eventType: Event['eventType']): Event { + return { + eventId: `evnt_test_${eventType}`, + runId: 'wrun_test', + correlationId: undefined, + eventType, + eventData: undefined, + createdAt: new Date(), + specVersion: 2, + // biome-ignore lint/suspicious/noExplicitAny: minimal test fixture + } as any; +} + +describe('canSkipSnapshotLoad', () => { + it('returns false when preloadedEvents is undefined', () => { + expect(canSkipSnapshotLoad(undefined)).toBe(false); + }); + + it('returns false when preloadedEvents is an empty array', () => { + expect(canSkipSnapshotLoad([])).toBe(false); + }); + + it('returns true for run_created + run_started only (very first invocation)', () => { + expect(canSkipSnapshotLoad([ev('run_created'), ev('run_started')])).toBe( + true + ); + }); + + it('returns true for run_started only (resilient-start path with no run_created replayed)', () => { + expect(canSkipSnapshotLoad([ev('run_started')])).toBe(true); + }); + + it('returns false when a step_created event is present', () => { + expect( + canSkipSnapshotLoad([ + ev('run_created'), + ev('run_started'), + ev('step_created'), + ]) + ).toBe(false); + }); + + it('returns false when a step_completed event is present', () => { + expect( + canSkipSnapshotLoad([ + ev('run_created'), + ev('run_started'), + ev('step_created'), + ev('step_started'), + ev('step_completed'), + ]) + ).toBe(false); + }); + + it('returns false when a hook_received event is present (hook resume)', () => { + expect( + canSkipSnapshotLoad([ + ev('run_created'), + ev('run_started'), + ev('hook_created'), + ev('hook_received'), + ]) + ).toBe(false); + }); + + it('returns false when a wait_completed event is present (wait elapsed)', () => { + expect( + canSkipSnapshotLoad([ + ev('run_created'), + ev('run_started'), + ev('wait_created'), + ev('wait_completed'), + ]) + ).toBe(false); + }); +}); diff --git a/packages/core/src/runtime/snapshot-entrypoint.ts b/packages/core/src/runtime/snapshot-entrypoint.ts new file mode 100644 index 0000000000..7d4109cdb7 --- /dev/null +++ b/packages/core/src/runtime/snapshot-entrypoint.ts @@ -0,0 +1,1060 @@ +/** + * Snapshot runtime integration with the Workflow DevKit. + * + * This module provides the entry point for running workflows using the + * snapshot-based runtime instead of the event-replay runtime. + */ + +import type { Span } from '@opentelemetry/api'; +import { + EntityConflictError, + RunExpiredError, + WorkflowNotRegisteredError, +} from '@workflow/errors'; +import { getPort } from '@workflow/utils/get-port'; +import { parseWorkflowName } from '@workflow/utils/parse-name'; +import { + type Event, + type RunInput, + SPEC_VERSION_CURRENT, + type WorkflowRun, +} from '@workflow/world'; +import { classifyRunError } from '../classify-error.js'; +import { importKey } from '../encryption.js'; +import { runtimeLogger } from '../logger.js'; +import { + compress, + decompress, + PREFERRED_CODEC, +} from '../serialization/compression.js'; +import { + decrypt as decryptSerializedData, + encrypt as encryptSerializedData, +} from '../serialization/encryption.js'; +import { + dehydrateRunError, + hydrateRunError, + maybeEncrypt, +} from '../serialization.js'; +import { remapErrorStack, stripInlineSourceMap } from '../source-map.js'; +import * as Attribute from '../telemetry/semantic-conventions.js'; +import { serializeTraceCarrier, trace } from '../telemetry.js'; +import { getWorkflowQueueName, queueMessage } from './helpers.js'; +import { + type PendingHook, + type PendingStep, + type PendingWait, + runSnapshotWorkflow, +} from './snapshot-runtime.js'; +import { getWorld } from './world.js'; + +/** Tiny ms timer using performance.now() — already monotonic on Node. */ +function tick(): number { + return performance.now(); +} + +/** + * Returns true when the supplied events indicate the workflow handler + * has not yet completed a suspension cycle for this run, meaning a + * `snapshots.load` call would 404 and can be skipped entirely. + * + * The suspension handler always writes the snapshot BEFORE any + * `step_created` / `hook_created` / `wait_created` events + * (`await trace('snapshot.save', ...)` then `Promise.all(opsPromises)` + * in this file). So the presence of any non-initial event implies a + * save attempt has at least started, and we should still try to load + * to potentially restore from it. The contrapositive: if we only see + * `run_created` / `run_started`, the handler has never reached its + * first suspension and no snapshot exists. + * + * Returns false when `preloadedEvents` is missing/empty so the caller + * falls back to the normal load path (the world may simply not have + * preloaded events for this invocation). + * + * Exported for unit testing. + */ +export function canSkipSnapshotLoad( + preloadedEvents: readonly Event[] | undefined +): boolean { + if (!Array.isArray(preloadedEvents) || preloadedEvents.length === 0) { + return false; + } + return preloadedEvents.every( + (e) => e.eventType === 'run_created' || e.eventType === 'run_started' + ); +} + +/** + * Run a workflow using the snapshot runtime. + * + * This replaces the event-replay path (runWorkflow + EventsConsumer) with: + * 1. Check for existing snapshot + * 2. If snapshot exists: restore + process delta events + * 3. If no snapshot: first run with full event log + * 4. On suspension: save snapshot + create events + queue steps + * 5. On completion: create run_completed + delete snapshot + * 6. On failure: create run_failed + delete snapshot + */ +export async function runWorkflowWithSnapshots(params: { + workflowCode: string; + workflowName: string; + workflowRun: WorkflowRun; + /** + * Events returned inline by `events.create('run_started', ...)`. When + * present, they are used as the initial event log instead of fetching + * via `events.list`, matching the replay runtime's fast path. Crucially, + * if the world backfilled a missing `run_created` via the resilient + * start path, `preloadedEvents` contains it even when a fresh + * `events.list` might not (eventual consistency). + */ + preloadedEvents?: Event[]; + /** + * Run input carried through the queue message on first delivery. Used + * as a last-resort fallback for `run_created.eventData.input` when + * the event log is incomplete. + */ + runInput?: RunInput; + /** + * The parent OTel span (the outer `WORKFLOW {workflowName}` span from + * `runtime.ts`). When supplied, snapshot lifecycle attributes are + * attached to it for end-to-end visibility. + */ + parentSpan?: Span; +}): Promise<{ timeoutSeconds?: number } | void> { + const { + workflowCode, + workflowName, + workflowRun, + preloadedEvents, + runInput, + parentSpan, + } = params; + const world = await getWorld(); + const runId = workflowRun.runId; + const invocationStart = tick(); + + // Strip the inline source map comment before evaluating the bundle in + // the QuickJS VM. The map is purely host-side metadata for + // `remapErrorStack` (called below on workflow failures, against the + // ORIGINAL `workflowCode`). QuickJS retains source text for + // stack-trace line lookups, so the few-MB base64 comment bloats the + // VM heap and therefore every snapshot save+load round-trip. + // Empirically: ~32% snapshot-bytes reduction on the example + // workbench's bundle (11.9 MB → 8.0 MB plaintext snapshot). + const workflowCodeForVM = stripInlineSourceMap(workflowCode); + // Per-invocation diagnostic id so checkpoint logs can be correlated even + // if the same runId is processed by overlapping invocations on different + // function instances. + const invocationId = `inv_${Math.random().toString(36).slice(2, 10)}`; + + // Single high-volume diagnostic helper: emits a single-line structured + // record to stderr that survives Vercel function-log collection and is + // grep-friendly by runId. Always-on (warn level) so it shows up in + // production logs without DEBUG. Use sparingly — one record per + // invocation checkpoint. + const wfdiag = (checkpoint: string, fields: Record) => { + runtimeLogger.warn('SNAPSHOT_DIAG', { + checkpoint, + runId, + invocationId, + tElapsedMs: Math.round(tick() - invocationStart), + ...fields, + }); + }; + + parentSpan?.setAttributes({ + ...Attribute.SnapshotRuntime('snapshot'), + }); + + wfdiag('enter', { + workflowName, + hasPreloadedEvents: + Array.isArray(preloadedEvents) && preloadedEvents.length > 0, + preloadedEventCount: preloadedEvents?.length ?? 0, + hasRunInput: !!runInput, + }); + + // The workflowName from the queue topic is already the full workflow ID + // (e.g. "workflow//./workflows/1_simple//simple") + const workflowId = workflowName; + + // Resolve the encryption key up front. Needed before loading the + // snapshot (to decrypt it) and before saving (to encrypt it). + const rawKey = await world.getEncryptionKeyForRun?.(workflowRun); + const encryptionKey = rawKey ? await importKey(rawKey) : undefined; + + // Fast path: if the events we already have indicate the workflow + // handler has not yet completed a suspension cycle for this run, + // skip the `snapshots.load` round-trip (which would 404 anyway). + const isFirstInvocation = canSkipSnapshotLoad(preloadedEvents); + + // Per-load timing/byte breakdown carried back out of the trace + // closure so we can fold it into the `snapshot_loaded` diagnostic. + let loadDurationMs: number | undefined; + let loadDecryptDurationMs: number | undefined; + let loadDecompressDurationMs: number | undefined; + let loadReturnedBytes: number | undefined; + let loadDecompressedBytes: number | undefined; + + // Load + decrypt + decompress is wrapped in a child span so + // operators can see snapshot-restore latency in waterfall views. + // Pipeline order on load (inverse of save): + // world.snapshots.load → decrypt → decompress → deserialize. + const existingSnapshot = isFirstInvocation + ? null + : await trace<{ + data: Uint8Array; + metadata: import('@workflow/world').SnapshotMetadata; + } | null>('snapshot.load', async (loadSpan) => { + const t0 = tick(); + const loadedSnapshot = await world.snapshots.load(runId); + loadDurationMs = tick() - t0; + + loadSpan?.setAttributes({ + ...Attribute.SnapshotLoadDurationMs(Math.round(loadDurationMs)), + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotLoadDurationMs(Math.round(loadDurationMs)), + }); + + if (!loadedSnapshot) return null; + + loadReturnedBytes = loadedSnapshot.data.byteLength; + loadSpan?.setAttributes({ + ...Attribute.SnapshotLoadBytes(loadReturnedBytes), + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotLoadBytes(loadReturnedBytes), + }); + + // Decrypt if the snapshot was written with encryption. Plaintext + // snapshots (written before this change, or on runs without + // encryption configured) pass through unchanged. + const decryptStart = tick(); + const decrypted = (await decryptSerializedData( + loadedSnapshot.data, + encryptionKey + )) as Uint8Array; + if (encryptionKey) { + loadDecryptDurationMs = tick() - decryptStart; + loadSpan?.setAttributes({ + ...Attribute.SnapshotDecryptDurationMs( + Math.round(loadDecryptDurationMs) + ), + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotDecryptDurationMs( + Math.round(loadDecryptDurationMs) + ), + }); + } + + // Decompress if the snapshot was written with a compression + // prefix (gzip/zstd). Snapshots written before the + // compress-then-encrypt rollout are bare plaintext and pass + // through unchanged via the format-prefix dispatch in + // `decompress()`. + const decompressStart = tick(); + const decompressed = decompress(decrypted) as Uint8Array; + loadDecompressDurationMs = tick() - decompressStart; + loadDecompressedBytes = decompressed.byteLength; + + return { data: decompressed, metadata: loadedSnapshot.metadata }; + }); + + parentSpan?.setAttributes({ + ...Attribute.SnapshotInvocationKind(existingSnapshot ? 'restore' : 'first'), + }); + + wfdiag('snapshot_loaded', { + invocationKind: existingSnapshot ? 'restore' : 'first', + // Plaintext bytes after decrypt + decompress — what gets handed to + // QuickJS.deserializeSnapshot. + snapshotBytes: existingSnapshot?.data.byteLength ?? 0, + // Bytes returned by `world.snapshots.load()` — after the world has + // done its own transport-level decompression (if any). With the + // compress-then-encrypt pipeline and world-vercel's gzip layer + // removed, this should equal the (encrypted, compressed) bytes + // that came off the wire. + returnedBytes: loadReturnedBytes ?? 0, + // Bytes after our own decompress() (pre-deserialize). When equal + // to returnedBytes, the load was a no-op decompression (no + // compression prefix on the stored blob — old format). + decompressedBytes: loadDecompressedBytes ?? 0, + loadDurationMs: + loadDurationMs !== undefined ? Math.round(loadDurationMs) : undefined, + decompressDurationMs: + loadDecompressDurationMs !== undefined + ? Math.round(loadDecompressDurationMs) + : undefined, + decryptDurationMs: + loadDecryptDurationMs !== undefined + ? Math.round(loadDecryptDurationMs) + : undefined, + eventsCursor: existingSnapshot?.metadata.eventsCursor ?? null, + // True when we skipped the snapshots.load call entirely because + // preloadedEvents indicated this is the first handler invocation. + skippedLoad: isFirstInvocation, + }); + + // On first invocation (no snapshot), prefer preloadedEvents from the + // run_started response — they're guaranteed to include run_created + // even if the world's event log is eventually consistent. On restore, + // we always fetch delta events via the cursor. + let events: Event[]; + let lastEventsCursor: string | null = + existingSnapshot?.metadata.eventsCursor ?? null; + + let eventsFetchedPages = 0; + if (!existingSnapshot && preloadedEvents && preloadedEvents.length > 0) { + events = preloadedEvents; + } else { + const allEvents: Event[] = []; + let cursor: string | null = lastEventsCursor; + let hasMore = true; + + while (hasMore) { + const response = await world.events.list({ + runId, + pagination: { + sortOrder: 'asc', + cursor: cursor ?? undefined, + limit: 1000, + }, + }); + eventsFetchedPages++; + allEvents.push(...response.data); + // Update the cursor to the last successfully fetched page's cursor. + // Only update when we got results — the final empty-page response + // returns cursor=null which we must NOT use (it would reset the cursor). + if (response.cursor) { + cursor = response.cursor; + } + hasMore = response.data.length > 0 && response.cursor != null; + } + + events = allEvents; + // Capture the final cursor position (after all fetched events) + if (cursor) lastEventsCursor = cursor; + } + + parentSpan?.setAttributes({ + ...Attribute.SnapshotEventsPreloaded( + !existingSnapshot && !!preloadedEvents && preloadedEvents.length > 0 + ), + ...Attribute.SnapshotEventsFetchedCount(events.length), + ...Attribute.SnapshotEventsFetchedPages(eventsFetchedPages), + }); + + runtimeLogger.info('Snapshot runtime: fetched events', { + workflowRunId: runId, + eventCount: events.length, + isRestore: !!existingSnapshot, + eventsCursor: lastEventsCursor, + }); + + wfdiag('events_fetched', { + eventCount: events.length, + eventsFetchedPages, + eventsCursor: lastEventsCursor, + eventTypes: events.reduce>((acc, e) => { + acc[e.eventType] = (acc[e.eventType] ?? 0) + 1; + return acc; + }, {}), + }); + + // Check for elapsed waits + const now = Date.now(); + const completedWaitIds = new Set( + events + .filter((e) => e.eventType === 'wait_completed') + .map((e) => e.correlationId) + ); + for (const event of events) { + if ( + event.eventType === 'wait_created' && + event.correlationId && + !completedWaitIds.has(event.correlationId) + ) { + const eventData = + 'eventData' in event + ? (event.eventData as Record) + : undefined; + const resumeAt = eventData?.resumeAt; + if (resumeAt && now >= new Date(resumeAt as string).getTime()) { + try { + const result = await world.events.create(runId, { + eventType: 'wait_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: event.correlationId, + }); + if (result.event) events.push(result.event); + } catch (err) { + if (EntityConflictError.is(err)) continue; + throw err; + } + } + } + } + + // Resolve the workflow server port so `getWorkflowMetadata().url` inside + // the VM matches what the step-side handler reports. Skipped on Vercel — + // the VM reads VERCEL_URL directly in that environment. + const isVercel = process.env.VERCEL_URL !== undefined; + const port = isVercel ? undefined : await getPort(); + + // Run the snapshot runtime + runtimeLogger.debug('Snapshot runtime: invoking VM', { + workflowRunId: runId, + workflowId, + eventCount: events.length, + hasSnapshot: !!existingSnapshot, + }); + + const result = await runSnapshotWorkflow({ + // Pass the STRIPPED bundle to the VM so the inline source map + // doesn't end up in the QuickJS heap or the resulting snapshot. + // The original (unstripped) `workflowCode` is still kept in this + // host-side scope and is used by `remapErrorStack` on workflow + // failures below. + workflowCode: workflowCodeForVM, + workflowId, + workflowRun, + events, + existingSnapshot, + encryptionKey, + port, + runInput, + parentSpan, + }); + + runtimeLogger.debug('Snapshot runtime: VM returned', { + workflowRunId: runId, + completed: !!result.completed, + suspended: !!result.suspended, + failed: !!result.failed, + pendingOpsCount: result.suspended?.pendingOperations?.length, + }); + + wfdiag('vm_returned', { + outcome: result.completed + ? 'completed' + : result.suspended + ? 'suspended' + : result.failed + ? 'failed' + : 'unknown', + pendingOpsCount: result.suspended?.pendingOperations?.length ?? 0, + pendingOpSummary: result.suspended?.pendingOperations?.map((p) => ({ + type: p.type, + correlationId: p.correlationId, + hasCreatedEvent: p.hasCreatedEvent, + ...(p.type === 'step' ? { stepId: (p as PendingStep).stepId } : {}), + })), + failureMessage: result.failed?.message, + failureName: result.failed?.name, + }); + + if (result.completed) { + // Workflow completed + runtimeLogger.info('Snapshot runtime: workflow completed', { + workflowRunId: runId, + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotOutcome('completed'), + }); + + // Delete the snapshot + { + const t0 = tick(); + await world.snapshots.delete(runId); + parentSpan?.setAttributes({ + ...Attribute.SnapshotDeleteDurationMs(Math.round(tick() - t0)), + }); + } + + // Create run_completed event. + // The VM serializes the workflow result as format-prefixed devalue bytes + // ("devl" + devalue) with no encryption (the VM has no access to the + // CryptoKey). Host-side encryption is applied here so that `run_completed` + // events have the same `encr`-prefixed payload shape that the replay + // runtime's `dehydrateWorkflowReturnValue` produces. + try { + await world.events.create(runId, { + eventType: 'run_completed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + output: await encryptSerializedData( + result.completed.result, + encryptionKey + ), + }, + }); + wfdiag('exit_completed', { result: 'run_completed_written' }); + } catch (err) { + if (EntityConflictError.is(err) || RunExpiredError.is(err)) { + runtimeLogger.warn( + 'Workflow already finished, skipping run_completed', + { workflowRunId: runId } + ); + wfdiag('exit_completed', { result: 'already_finished' }); + return; + } + wfdiag('exit_completed_error', { + errorName: (err as Error)?.name, + errorMessage: (err as Error)?.message, + }); + throw err; + } + } else if (result.suspended) { + // Workflow suspended + const { pendingOperations, snapshot } = result.suspended; + + runtimeLogger.info('Snapshot runtime: workflow suspended', { + workflowRunId: runId, + pendingSteps: pendingOperations.filter((p) => p.type === 'step').length, + pendingWaits: pendingOperations.filter((p) => p.type === 'wait').length, + pendingOps: pendingOperations.map((p) => ({ + type: p.type, + correlationId: p.correlationId, + hasCreatedEvent: p.hasCreatedEvent, + ...(p.type === 'step' + ? { + stepId: (p as PendingStep).stepId, + inputType: typeof (p as PendingStep).input, + inputIsUint8Array: (p as PendingStep).input instanceof Uint8Array, + } + : {}), + })), + }); + + parentSpan?.setAttributes({ + ...Attribute.SnapshotOutcome('suspended'), + ...Attribute.SnapshotPendingOpsCount(pendingOperations.length), + ...(lastEventsCursor + ? Attribute.SnapshotEventsCursor(lastEventsCursor) + : {}), + }); + + // Save the snapshot, encrypting if a key is available. The save + // must complete before any step is queued so that subsequent + // workflow invocations always observe a snapshot at-or-newer-than + // the events they will process — pipelining save with queueMessage + // creates a window where a step can complete and re-invoke the + // workflow handler, which then loads a stale (or missing) snapshot + // and replays a coroutine state that doesn't match the latest + // events. Per-pending-op events.create + queueMessage calls below + // ARE parallelized via Promise.all, which gives the bulk of the + // wall-clock reduction without the ordering hazard. Wrapped in a + // child span so operators can drill into serialize / encrypt / + // persist latency separately. + // + // Per-stage timings/byte counts are captured here and reported in + // the `snapshot_saved` wfdiag below so the breakdown shows up in + // CI-fetched function logs (not just OTel spans). + // + // Pipeline order: serialize → compress → encrypt → store. + // Compression goes BEFORE encryption because encrypted bytes are + // ~random and don't compress (gzip on ciphertext is wasted CPU). + // For QuickJS heaps the compression ratio is ~4x with zstd or + // gzip, so the bytes that get encrypted (and uploaded) are + // already much smaller than the raw heap. + let saveCompressedBytes = 0; + let saveCompressDurationMs = 0; + let saveHandedToWorldBytes = 0; + let saveEncryptDurationMs: number | undefined; + let saveStoreDurationMs = 0; + await trace('snapshot.save', async (saveSpan) => { + const plaintextBytes = snapshot.byteLength; + saveSpan?.setAttributes({ + ...Attribute.SnapshotSavePlaintextBytes(plaintextBytes), + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotSavePlaintextBytes(plaintextBytes), + }); + + // Compress before encrypt — see comment above. + const compressStart = tick(); + const compressed = compress(snapshot) as Uint8Array; + saveCompressDurationMs = Math.round(tick() - compressStart); + saveCompressedBytes = compressed.byteLength; + + const encryptStart = tick(); + const snapshotToStore = (await encryptSerializedData( + compressed, + encryptionKey + )) as Uint8Array; + if (encryptionKey) { + saveEncryptDurationMs = Math.round(tick() - encryptStart); + saveSpan?.setAttributes({ + ...Attribute.SnapshotEncryptDurationMs(saveEncryptDurationMs), + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotEncryptDurationMs(saveEncryptDurationMs), + }); + } + saveHandedToWorldBytes = snapshotToStore.byteLength; + + runtimeLogger.debug('Snapshot runtime: saving snapshot', { + workflowRunId: runId, + snapshotType: typeof snapshotToStore, + snapshotIsUint8Array: snapshotToStore instanceof Uint8Array, + snapshotLength: snapshotToStore?.length, + snapshotByteLength: snapshotToStore?.byteLength, + encrypted: !!encryptionKey, + eventsCursor: lastEventsCursor, + }); + + const saveStart = tick(); + await world.snapshots.save(runId, snapshotToStore, { + eventsCursor: lastEventsCursor, + createdAt: new Date(), + }); + saveStoreDurationMs = Math.round(tick() - saveStart); + + saveSpan?.setAttributes({ + ...Attribute.SnapshotSaveBytes(snapshotToStore.byteLength), + ...Attribute.SnapshotSaveDurationMs(saveStoreDurationMs), + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotSaveBytes(snapshotToStore.byteLength), + ...Attribute.SnapshotSaveDurationMs(saveStoreDurationMs), + }); + }); + + wfdiag('snapshot_saved', { + // Plaintext bytes — QuickJS serializeSnapshot output, before + // compression / encryption. What the host actually generated. + plaintextBytes: snapshot.byteLength, + // After compression but before encryption. The codec used + // (zstd vs gzip) is reflected in the format prefix on the bytes; + // PREFERRED_CODEC reports which one this process is using. + compressedBytes: saveCompressedBytes, + compressionRatio: + snapshot.byteLength > 0 && saveCompressedBytes > 0 + ? +(snapshot.byteLength / saveCompressedBytes).toFixed(2) + : 0, + compressionCodec: PREFERRED_CODEC, + // Bytes handed to `world.snapshots.save()` — after both + // compression and encryption. This is what the world transports. + // The world should NOT add its own compression layer (encrypted + // bytes are not compressible). + handedToWorldBytes: saveHandedToWorldBytes, + // Per-stage timings. + compressDurationMs: saveCompressDurationMs, + encryptDurationMs: saveEncryptDurationMs, + storeDurationMs: saveStoreDurationMs, + eventsCursor: lastEventsCursor, + }); + + // Build per-pending-op promises so events.create + queueMessage + // calls fan out in parallel rather than serially. This mirrors + // the replay runtime's `Promise.all(ops)` pattern in + // suspension-handler.ts and significantly reduces wall-clock time + // on cloud worlds (e.g. Vercel) where each storage call is a + // network round-trip. + let minTimeoutSeconds: number | undefined; + const opsPromises: Promise[] = []; + + for (const op of pendingOperations) { + if (op.type === 'step' && !op.hasCreatedEvent) { + const step = op as PendingStep; + opsPromises.push( + (async () => { + // Create step_created event. `step.input` is the + // format-prefixed devalue bytes ("devl" + devalue) produced + // by `globalThis[Symbol.for('workflow-serialize')]({args, + // closureVars, thisVal})` inside the VM. The VM has no + // access to the CryptoKey, so encryption is applied here + // on the host side — matching what + // `dehydrateStepArguments` does in the replay runtime. + try { + await world.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: step.correlationId, + eventData: { + stepName: step.stepId, + input: await encryptSerializedData(step.input, encryptionKey), + }, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + + // Queue the step execution via the unified workflow queue + // (V2 architecture). The combined handler in runtime.ts + // dispatches messages with `stepId` to executeStep, which + // works for both replay and snapshot modes — so snapshot + // mode reuses the same step execution path as V2 replay + // instead of needing a separate step route. + const traceCarrier = await serializeTraceCarrier(); + await queueMessage( + world, + getWorkflowQueueName(workflowRun.workflowName), + { + runId, + stepId: step.correlationId, + stepName: step.stepId, + traceCarrier, + requestedAt: new Date(), + }, + { + idempotencyKey: step.correlationId, + } + ); + wfdiag('step_queued', { + stepId: step.stepId, + correlationId: step.correlationId, + }); + })() + ); + } else if (op.type === 'hook' && !op.hasCreatedEvent) { + const hook = op as PendingHook; + runtimeLogger.debug('Snapshot runtime: creating hook_created event', { + workflowRunId: runId, + correlationId: hook.correlationId, + token: hook.token, + tokenType: typeof hook.token, + isWebhook: hook.isWebhook, + }); + + opsPromises.push( + (async () => { + // `hook.metadata` is the format-prefixed devalue bytes + // produced by `globalThis[Symbol.for('workflow-serialize')] + // (options.metadata)` inside the VM. Encrypt on the host + // side before writing — matches the replay runtime's + // `dehydrateStepArguments` flow. + // + // No pre-check via hooks.list: with deterministic correlationIds + // (same VM seed across replays) and per-(runId, correlationId) + // uniqueness in worlds, the storage layer rejects duplicates as + // EntityConflictError, which we swallow below. This drops one + // network round-trip per pending hook. + try { + const encryptedMetadata = + typeof hook.metadata === 'undefined' + ? undefined + : await encryptSerializedData(hook.metadata, encryptionKey); + const result = await world.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hook.correlationId, + eventData: { + token: hook.token, + metadata: encryptedMetadata, + // Always include isWebhook explicitly. Worlds default it to + // `true` when absent, which would break the public webhook + // endpoint's 404 guard for hooks created via createHook(). + isWebhook: hook.isWebhook, + } as any, + }); + + // If storage detected a real token conflict with another + // workflow's hook, re-queue so the snapshot runtime can + // process the conflict event and fail gracefully. + if (result.event?.eventType === 'hook_conflict') { + await queueMessage( + world, + `__wkf_workflow_${workflowRun.workflowName}`, + { + runId, + }, + { idempotencyKey: `hook_conflict_${hook.correlationId}` } + ); + } + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } else if (op.type === 'hook_dispose' && !op.hasCreatedEvent) { + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'hook_disposed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: op.correlationId, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } else if (op.type === 'wait' && !op.hasCreatedEvent) { + const wait = op as PendingWait; + opsPromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'wait_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: wait.correlationId, + eventData: { + resumeAt: new Date(wait.resumeAt), + }, + }); + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } + } + + // Per-op dispatch runs in parallel; snapshot.save above already + // completed. + await Promise.all(opsPromises); + + // Handle pending waits — both newly created and pre-existing from the + // snapshot. For each wait, either create a wait_completed event (if + // elapsed) or schedule a timeout for re-queuing. + let needsRequeue = false; + const waitCompletePromises: Promise[] = []; + for (const op of pendingOperations) { + if (op.type !== 'wait') continue; + const wait = op as PendingWait; + const resumeMs = new Date(wait.resumeAt).getTime() - Date.now(); + + if (resumeMs <= 0) { + // Wait has elapsed — create wait_completed and re-queue. + waitCompletePromises.push( + (async () => { + try { + await world.events.create(runId, { + eventType: 'wait_completed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: wait.correlationId, + }); + needsRequeue = true; + } catch (err) { + if (EntityConflictError.is(err)) return; + throw err; + } + })() + ); + } else { + // Wait hasn't elapsed yet — schedule a timeout + const timeoutSeconds = Math.max(1, Math.ceil(resumeMs / 1000)); + if ( + minTimeoutSeconds === undefined || + timeoutSeconds < minTimeoutSeconds + ) { + minTimeoutSeconds = timeoutSeconds; + } + } + } + if (waitCompletePromises.length > 0) { + await Promise.all(waitCompletePromises); + } + + if (needsRequeue) { + // An elapsed wait was completed — re-queue immediately so the + // snapshot runtime can process the wait_completed event. + wfdiag('exit_suspended', { + action: 'wait_elapsed_requeue', + timeoutSeconds: 0, + }); + return { timeoutSeconds: 0 }; + } + + if (minTimeoutSeconds !== undefined) { + wfdiag('exit_suspended', { + action: 'schedule_wait_timeout', + timeoutSeconds: minTimeoutSeconds, + }); + return { timeoutSeconds: minTimeoutSeconds }; + } + + wfdiag('exit_suspended', { + action: 'awaiting_external', + pendingOpsCount: pendingOperations.length, + }); + } else if (result.failed) { + // Workflow failed — remap stack trace using inline source maps + let errorStack = result.failed.stack; + if (errorStack) { + const parsedName = parseWorkflowName(workflowName); + const filename = parsedName?.moduleSpecifier || workflowName; + errorStack = remapErrorStack(errorStack, filename, workflowCode); + } + + // Classify the error so consumers (`run.returnValue`, observability) + // get `USER_ERROR` / `RUNTIME_ERROR` on `error.cause.code`, matching + // what the replay runtime already does in runtime.ts. + // + // The VM serializes errors as `{ name, message, stack }`, so we + // reconstruct a host-side Error of the correct class based on the + // VM-side `name` — specific WorkflowRuntimeError subclasses need + // to be preserved so classifyRunError() tags them as RUNTIME_ERROR. + const reconstructed: Error = + result.failed.name === 'WorkflowNotRegisteredError' + ? new WorkflowNotRegisteredError(workflowName) + : result.failed.name === 'Error' + ? new Error(result.failed.message) + : Object.assign(new Error(result.failed.message), { + name: result.failed.name, + }); + const errorCode = classifyRunError(reconstructed); + + runtimeLogger.error('Snapshot runtime: workflow failed', { + workflowRunId: runId, + errorName: result.failed.name, + errorMessage: result.failed.message, + errorStack, + errorCode, + }); + parentSpan?.setAttributes({ + ...Attribute.SnapshotOutcome('failed'), + }); + + // Delete the snapshot + { + const t0 = tick(); + await world.snapshots.delete(runId); + parentSpan?.setAttributes({ + ...Attribute.SnapshotDeleteDurationMs(Math.round(tick() - t0)), + }); + } + + // Create run_failed event. Serialize the error through the + // first-class dehydration pipeline so consumers (CLI, observability, + // run.returnValue) get the same hydrated value shape as the replay + // runtime emits. Two paths: + // * Modern (valueBytes present): the VM-side rejection handler + // serialized the original thrown value (Error subclass with + // cause chain, plain object, primitive, etc.) using the VM's + // workflow-serialize. Pass those bytes through directly so + // type identity, cause chains, and non-Error throws survive. + // We just need to apply encryption if configured (the VM's + // serializer doesn't have access to the encryption key). + // * Legacy fallback: reconstruct an Error from the host-visible + // {name, message, stack} fields and run it through + // `dehydrateRunError`. Used when valueBytes is absent (e.g. + // extractError pseudo-failures from VM bootstrap). + let dehydratedError: Uint8Array; + if (result.failed.valueBytes) { + // Hydrate the VM-side bytes, remap the error stack with the + // host-side source map (the VM can't do this — it lacks both the + // source map and `remapErrorStack`), and re-dehydrate. This + // preserves the original value's type identity / cause chain + // while fixing up frames to point at the user's source files. + try { + const hydrated = await hydrateRunError( + result.failed.valueBytes, + runId, + undefined // VM bytes are unencrypted + ); + if ( + hydrated && + typeof hydrated === 'object' && + 'stack' in (hydrated as object) && + typeof (hydrated as { stack?: unknown }).stack === 'string' + ) { + const parsedName = parseWorkflowName(workflowName); + const filename = parsedName?.moduleSpecifier || workflowName; + (hydrated as { stack?: string }).stack = remapErrorStack( + (hydrated as { stack: string }).stack, + filename, + workflowCode + ); + } + // Walk the cause chain and remap nested stacks too. + const seen = new WeakSet(); + let node = (hydrated as { cause?: unknown })?.cause; + while (node && typeof node === 'object' && !seen.has(node as object)) { + seen.add(node as object); + const nodeStack = (node as { stack?: unknown }).stack; + if (typeof nodeStack === 'string') { + const parsedName = parseWorkflowName(workflowName); + const filename = parsedName?.moduleSpecifier || workflowName; + (node as { stack?: string }).stack = remapErrorStack( + nodeStack, + filename, + workflowCode + ); + } + node = (node as { cause?: unknown }).cause; + } + dehydratedError = await dehydrateRunError( + hydrated, + runId, + encryptionKey + ); + } catch (rehydrateErr) { + // If hydration / re-dehydration fails for any reason, fall + // back to passing through the original VM bytes (just apply + // encryption if configured). Better to lose source-mapped + // frames than to lose the error entirely. + runtimeLogger.warn( + 'Snapshot runtime: failed to remap workflow error stack, passing VM bytes through', + { + workflowRunId: runId, + message: (rehydrateErr as Error)?.message, + } + ); + dehydratedError = (await maybeEncrypt( + result.failed.valueBytes, + encryptionKey + )) as Uint8Array; + } + } else { + if (errorStack) { + reconstructed.stack = errorStack; + } + try { + dehydratedError = await dehydrateRunError( + reconstructed, + runId, + encryptionKey + ); + } catch (serErr) { + // Fall back to a minimal payload so the run still terminates + // even when the error itself contains unserializable values. + runtimeLogger.warn( + 'Snapshot runtime: failed to dehydrate run error, falling back to bare Error', + { workflowRunId: runId, message: (serErr as Error)?.message } + ); + dehydratedError = await dehydrateRunError( + Object.assign(new Error(result.failed.message), { + name: result.failed.name, + }), + runId, + encryptionKey + ); + } + } + try { + await world.events.create(runId, { + eventType: 'run_failed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + error: dehydratedError, + errorCode, + }, + }); + } catch (err) { + if (EntityConflictError.is(err) || RunExpiredError.is(err)) { + runtimeLogger.warn('Workflow already finished, skipping run_failed', { + workflowRunId: runId, + }); + wfdiag('exit_failed', { result: 'already_finished' }); + return; + } + wfdiag('exit_failed_error', { + errorName: (err as Error)?.name, + errorMessage: (err as Error)?.message, + }); + throw err; + } + wfdiag('exit_failed', { result: 'run_failed_written' }); + } +} + +// ---- Helpers ---- diff --git a/packages/core/src/runtime/snapshot-runtime.test.ts b/packages/core/src/runtime/snapshot-runtime.test.ts new file mode 100644 index 0000000000..04b647c8d0 --- /dev/null +++ b/packages/core/src/runtime/snapshot-runtime.test.ts @@ -0,0 +1,487 @@ +import { QuickJS } from 'quickjs-wasi'; +import { describe, expect, it } from 'vitest'; +import { deserialize } from '../serialization/workflow-vm.js'; +import { runSnapshotWorkflow } from './snapshot-runtime.js'; + +/** Helper to deserialize the format-prefixed result bytes */ +function unwrapResult(result: Uint8Array): unknown { + return deserialize(result); +} + +function makeRun(overrides: Record = {}) { + return { + runId: 'wrun_test123', + deploymentId: 'dpl_test', + workflowName: 'test-workflow', + input: undefined, + status: 'running' as const, + output: undefined, + error: undefined, + completedAt: undefined, + startedAt: new Date('2025-01-01T00:00:00Z'), + createdAt: new Date('2025-01-01T00:00:00Z'), + updatedAt: new Date('2025-01-01T00:00:00Z'), + specVersion: 2, + ...overrides, + }; +} + +describe('runSnapshotWorkflow', () => { + it('should run a simple workflow with no steps to completion', async () => { + const result = await runSnapshotWorkflow({ + workflowCode: ` + globalThis.__private_workflows = new Map(); + async function hello() { return 42; } + hello.workflowId = "workflow//test//hello"; + globalThis.__private_workflows.set("workflow//test//hello", hello); + `, + workflowId: 'workflow//test//hello', + workflowRun: makeRun(), + events: [], + existingSnapshot: null, + }); + + expect(result.completed).toBeDefined(); + expect(unwrapResult(result.completed!.result)).toBe(42); + }); + + it('should suspend on first step and return pending operations', async () => { + const result = await runSnapshotWorkflow({ + workflowCode: ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { + var a = await add(10, 7); + return a; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + existingSnapshot: null, + }); + + expect(result.suspended).toBeDefined(); + expect(result.suspended?.pendingOperations).toHaveLength(1); + expect(result.suspended?.pendingOperations[0]).toMatchObject({ + type: 'step', + stepId: 'step//test//add', + }); + expect(result.suspended?.pendingOperations[0].correlationId).toMatch( + /^step_[0-9A-Z]{26}$/ + ); + expect(result.suspended?.snapshot).toBeInstanceOf(Uint8Array); + }); + + it('should restore from snapshot and complete after step resolves', async () => { + const r1 = await runSnapshotWorkflow({ + workflowCode: ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { return await add(10, 7); } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + existingSnapshot: null, + }); + expect(r1.suspended).toBeDefined(); + const stepCid = r1.suspended!.pendingOperations[0].correlationId; + + const r2 = await runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [ + { + eventId: 'evnt_001', + runId: 'wrun_test123', + eventType: 'step_completed', + correlationId: stepCid, + eventData: { result: 17 }, + createdAt: new Date(), + }, + ], + existingSnapshot: { + data: r1.suspended!.snapshot, + metadata: { eventsCursor: null, createdAt: new Date() }, + }, + }); + + expect(unwrapResult(r2.completed!.result)).toBe(17); + }); + + it('should handle multi-step workflows across multiple snapshots', async () => { + const code = ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { + var a = await add(10, 7); + var b = await add(a, 8); + return b; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + const run = makeRun(); + + const r1 = await runSnapshotWorkflow({ + workflowCode: code, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + existingSnapshot: null, + }); + const step1Cid = r1.suspended?.pendingOperations[0]?.correlationId; + expect(step1Cid).toMatch(/^step_[0-9A-Z]{26}$/); + + const r2 = await runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_completed', + correlationId: step1Cid!, + eventData: { result: 17 }, + createdAt: new Date(), + }, + ], + existingSnapshot: { + data: r1.suspended!.snapshot, + metadata: { eventsCursor: null, createdAt: new Date() }, + }, + }); + const step2Cid = r2.suspended?.pendingOperations[0]?.correlationId; + expect(step2Cid).toMatch(/^step_[0-9A-Z]{26}$/); + expect(step2Cid).not.toBe(step1Cid); + + const r3 = await runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + { + eventId: 'evnt_002', + runId: run.runId, + eventType: 'step_completed', + correlationId: step2Cid!, + eventData: { result: 25 }, + createdAt: new Date(), + }, + ], + existingSnapshot: { + data: r2.suspended!.snapshot, + metadata: { eventsCursor: 'evnt_001', createdAt: new Date() }, + }, + }); + expect(unwrapResult(r3.completed!.result)).toBe(25); + }); + + it('should handle sleep suspension and wake', async () => { + const r1 = await runSnapshotWorkflow({ + workflowCode: ` + async function workflow() { + await globalThis[Symbol.for("WORKFLOW_SLEEP")]("5s"); + return "woke up"; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + existingSnapshot: null, + }); + expect(r1.suspended).toBeDefined(); + expect(r1.suspended?.pendingOperations[0]).toMatchObject({ + type: 'wait', + }); + const waitCid = r1.suspended!.pendingOperations[0].correlationId; + expect(waitCid).toMatch(/^wait_[0-9A-Z]{26}$/); + + const r2 = await runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [ + { + eventId: 'evnt_001', + runId: 'wrun_test123', + eventType: 'wait_completed', + correlationId: waitCid, + createdAt: new Date(), + }, + ], + existingSnapshot: { + data: r1.suspended!.snapshot, + metadata: { eventsCursor: null, createdAt: new Date() }, + }, + }); + expect(unwrapResult(r2.completed!.result)).toBe('woke up'); + }); + + it('should handle step failure with try/catch in workflow', async () => { + const r1 = await runSnapshotWorkflow({ + workflowCode: ` + var fail = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//fail"); + async function workflow() { + try { await fail(); return "nope"; } + catch (e) { return "caught: " + e.message; } + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [], + existingSnapshot: null, + }); + expect(r1.suspended).toBeDefined(); + + const failStepCid = r1.suspended!.pendingOperations[0].correlationId; + + const r2 = await runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: makeRun(), + events: [ + { + eventId: 'evnt_001', + runId: 'wrun_test123', + eventType: 'step_failed', + correlationId: failStepCid, + eventData: { error: { message: 'boom' } }, + createdAt: new Date(), + }, + ], + existingSnapshot: { + data: r1.suspended!.snapshot, + metadata: { eventsCursor: null, createdAt: new Date() }, + }, + }); + expect(unwrapResult(r2.completed!.result)).toBe('caught: boom'); + }); +}); + +describe('correlationId determinism', () => { + // The snapshot runtime must produce identical correlationIds for the + // same logical workflow operation across concurrent invocations of the + // same resumption — otherwise two queue messages for the same runId + // can each generate "fresh" pending step ops, the world has no + // EntityConflictError to dedup them, and a single logical step + // becomes 2 step_created events (and only one of them ever has a + // matching step_completed handler in the running VM, so the others + // hang). + // + // Determinism boundary: + // 1. Same `workflowRun` (runId, name, startedAt) + same starting + // state (no snapshot, OR same snapshot+events) → IDENTICAL ids. + // 2. Different starting state (different cursor) → DIFFERENT ids. + // + // The fix injects a deterministic `__ulidTimestamp` (workflowRun.startedAt) + // into the VM so the ULID timestamp portion is stable across concurrent + // invocations, and seeds the PRNG with `runId:name:startedAt:cursor` + // so the random portion advances across resumptions but is stable + // within a resumption. + + const stepWorkflow = ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { return await add(10, 7); } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `; + + it('produces identical correlationIds for two concurrent first-run invocations', async () => { + const run = makeRun(); + + // Two independent VM invocations of the same fresh workflow run. + // These could be two queue messages for the same runId being + // processed in parallel by two workflow handler instances. + const [a, b] = await Promise.all([ + runSnapshotWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + existingSnapshot: null, + }), + runSnapshotWorkflow({ + workflowCode: stepWorkflow, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + existingSnapshot: null, + }), + ]); + + expect(a.suspended).toBeDefined(); + expect(b.suspended).toBeDefined(); + const aCid = a.suspended!.pendingOperations[0].correlationId; + const bCid = b.suspended!.pendingOperations[0].correlationId; + expect(aCid).toBe(bCid); + }); + + it('produces identical correlationIds for two concurrent restore invocations', async () => { + const run = makeRun(); + + // Drive the workflow to a suspension point so we have a snapshot. + const r1 = await runSnapshotWorkflow({ + workflowCode: ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { + var a = await add(10, 7); + var b = await add(a, 8); + return b; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + existingSnapshot: null, + }); + const step1Cid = r1.suspended!.pendingOperations[0].correlationId; + + // Two concurrent resumes from the same snapshot, both processing + // the same step_completed event. Each independently runs the + // workflow body forward to the next suspension point. + const events = [ + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_completed' as const, + correlationId: step1Cid, + eventData: { result: 17 }, + createdAt: new Date(), + }, + ]; + const existingSnapshot = { + data: r1.suspended!.snapshot, + metadata: { eventsCursor: 'evnt_001', createdAt: new Date() }, + }; + + const [a, b] = await Promise.all([ + runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: run, + events, + existingSnapshot, + }), + runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: run, + events, + existingSnapshot, + }), + ]); + + expect(a.suspended).toBeDefined(); + expect(b.suspended).toBeDefined(); + const aCid = a.suspended!.pendingOperations[0].correlationId; + const bCid = b.suspended!.pendingOperations[0].correlationId; + expect(aCid).toBe(bCid); + }); + + it('produces a different correlationId across resumes (different cursor)', async () => { + const run = makeRun(); + + const r1 = await runSnapshotWorkflow({ + workflowCode: ` + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function workflow() { + var a = await add(10, 7); + var b = await add(a, 8); + return b; + } + workflow.workflowId = "workflow//test//workflow"; + globalThis.__private_workflows.set("workflow//test//workflow", workflow); + `, + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [], + existingSnapshot: null, + }); + const step1Cid = r1.suspended!.pendingOperations[0].correlationId; + + const r2 = await runSnapshotWorkflow({ + workflowCode: '', + workflowId: 'workflow//test//workflow', + workflowRun: run, + events: [ + { + eventId: 'evnt_001', + runId: run.runId, + eventType: 'step_completed', + correlationId: step1Cid, + eventData: { result: 17 }, + createdAt: new Date(), + }, + ], + existingSnapshot: { + data: r1.suspended!.snapshot, + metadata: { eventsCursor: 'evnt_001', createdAt: new Date() }, + }, + }); + const step2Cid = r2.suspended!.pendingOperations[0].correlationId; + + // The second step's correlationId must be distinct from the first — + // different resume, different position in the workflow body, + // different PRNG state, different cursor. Otherwise EntityConflictError + // would falsely dedup it as a duplicate. + expect(step2Cid).not.toBe(step1Cid); + }); +}); + +describe('raw QuickJS proof of concept', () => { + it('should run, snapshot, restore, and complete', async () => { + const vm = await QuickJS.create(); + + vm.evalCode(` + globalThis.__private_workflows = new Map(); + globalThis.__resolvers = {}; + globalThis.__pending = []; + globalThis.__stepCounter = 0; + globalThis.__workflowResult = undefined; + + globalThis[Symbol.for("WORKFLOW_USE_STEP")] = function(stepId) { + return function() { + var args = Array.prototype.slice.call(arguments); + var cid = "step_" + (globalThis.__stepCounter++); + globalThis.__pending.push({ type: "step", correlationId: cid, stepId: stepId }); + return new Promise(function(resolve, reject) { + globalThis.__resolvers[cid] = { resolve: resolve, reject: reject }; + }); + }; + }; + + var add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step//test//add"); + async function simple(i) { var a = await add(i, 7); var b = await add(a, 8); return b; } + globalThis.__private_workflows.set("test", simple); + globalThis.__private_workflows.get("test")(10).then(function(r) { globalThis.__workflowResult = r; }); + `).dispose(); + vm.executePendingJobs(); + + const snap1 = vm.snapshot(); + vm.dispose(); + + const vm2 = await QuickJS.restore(snap1); + vm2.evalCode('globalThis.__resolvers["step_0"].resolve(17);').dispose(); + vm2.executePendingJobs(); + const snap2 = vm2.snapshot(); + vm2.dispose(); + + const vm3 = await QuickJS.restore(snap2); + vm3.evalCode('globalThis.__resolvers["step_1"].resolve(25);').dispose(); + vm3.executePendingJobs(); + + expect(vm3.dump(vm3.evalCode('globalThis.__workflowResult'))).toBe(25); + vm3.dispose(); + }); +}); diff --git a/packages/core/src/runtime/snapshot-runtime.ts b/packages/core/src/runtime/snapshot-runtime.ts new file mode 100644 index 0000000000..619c1f6f1a --- /dev/null +++ b/packages/core/src/runtime/snapshot-runtime.ts @@ -0,0 +1,1189 @@ +/** + * Snapshot-based workflow runtime. + * + * Instead of replaying the full event log on every invocation, this runtime: + * 1. Runs workflow code in a QuickJS WASM VM (via quickjs-wasi) + * 2. Snapshots the VM state when the workflow suspends + * 3. Restores the VM from the snapshot on resumption + * 4. Only fetches events since the last snapshot + * + * The workflow primitives (useStep, sleep, createHook) are implemented as + * JavaScript code running inside the QuickJS VM. The host communicates with + * the VM by evaluating small JS snippets to read pending operations and + * resolve/reject promises. + */ + +import type { Span } from '@opentelemetry/api'; +import type { + Event, + RunInput, + SnapshotMetadata, + WorkflowRun, +} from '@workflow/world'; +import * as nanoid from 'nanoid'; +import { JSException, QuickJS } from 'quickjs-wasi'; +import seedrandom from 'seedrandom'; +import type { CryptoKey } from '../encryption.js'; +import { runtimeLogger } from '../logger.js'; +import { decrypt as decryptData } from '../serialization/encryption.js'; +import * as Attribute from '../telemetry/semantic-conventions.js'; +import { quickjsExtensions, quickjsWasm } from './quickjs-assets.generated.js'; +import { VM_SERDE_BUNDLE } from './vm-serde-bundle.generated.js'; + +// ---- Types ---- + +export interface PendingStep { + type: 'step'; + correlationId: string; + stepId: string; + /** Format-prefixed devalue-serialized step input (args + closureVars) */ + input: Uint8Array; + /** Whether a step_created event already exists for this step */ + hasCreatedEvent: boolean; +} + +export interface PendingWait { + type: 'wait'; + correlationId: string; + /** ISO string of when to resume */ + resumeAt: string; + /** Whether a wait_created event already exists for this wait */ + hasCreatedEvent: boolean; +} + +export interface PendingHook { + type: 'hook'; + correlationId: string; + token: string; + isWebhook: boolean; + metadata?: unknown; + hasCreatedEvent: boolean; +} + +export interface PendingHookDispose { + type: 'hook_dispose'; + correlationId: string; + hasCreatedEvent: boolean; +} + +export type PendingOperation = + | PendingStep + | PendingWait + | PendingHook + | PendingHookDispose; + +export interface SnapshotRuntimeResult { + /** The workflow completed — result is format-prefixed devalue bytes */ + completed?: { result: Uint8Array }; + /** The workflow suspended with pending operations */ + suspended?: { + pendingOperations: PendingOperation[]; + snapshot: Uint8Array; + }; + /** The workflow failed */ + failed?: { + message: string; + stack?: string; + name?: string; + /** + * Format-prefixed devalue bytes of the original thrown value + * (Error subclass with cause chain, plain object, primitive, etc.). + * Set when the VM-side rejection handler successfully serializes + * the thrown value. The host uses these bytes to reconstruct the + * original value through the standard error hydration pipeline, + * preserving type identity (TypeError, FatalError) and non-Error + * throws verbatim. Falls back to the message/stack/name fields + * when this is undefined (e.g. extractError pseudo-failures). + */ + valueBytes?: Uint8Array; + }; +} + +export interface SnapshotRuntimeOptions { + /** The compiled workflow bundle code (workflow mode output from SWC) */ + workflowCode: string; + /** The workflow ID (e.g. "workflow//./workflows/1_simple//simple") */ + workflowId: string; + /** The workflow run entity */ + workflowRun: WorkflowRun; + /** Events to process: all events for first run, delta events for subsequent */ + events: Event[]; + /** Existing snapshot to restore from, or null for first invocation */ + existingSnapshot: { + data: Uint8Array; + metadata: SnapshotMetadata; + } | null; + /** Encryption key for decrypting event payloads (undefined if unencrypted) */ + encryptionKey?: CryptoKey; + /** + * The local port the workflow server is listening on, used to populate + * `workflowMetadata.url`. Resolved at call time on the host side so the + * VM doesn't have to probe the filesystem. Ignored on Vercel — VERCEL_URL + * takes precedence there. + */ + port?: number; + /** + * Fallback workflow input from the queue message's resilient-start + * payload. Used when the fetched event log lacks a `run_created` event + * (eventually-consistent read after the parent's start() wrote it). + */ + runInput?: RunInput; + /** + * Parent OTel span (the outer `WORKFLOW {workflowName}` span). When + * provided, VM serialize / deserialize timing attributes are attached + * to it for end-to-end visibility. + */ + parentSpan?: Span; +} + +// ---- VM Bootstrap Code ---- + +/** + * JavaScript code that runs inside the QuickJS VM to set up the workflow + * primitives. This sets up: + * - globalThis.__private_workflows (Map) - workflow registry + * - globalThis.__resolvers (Object) - pending promise resolve/reject functions + * - globalThis.__pending (Array) - metadata about pending operations + * - globalThis[Symbol.for("WORKFLOW_USE_STEP")] - step proxy factory + * - globalThis[Symbol.for("WORKFLOW_SLEEP")] - sleep function + */ +const VM_BOOTSTRAP = ` +// Symbol.dispose / Symbol.asyncDispose polyfills for QuickJS +if (typeof Symbol.dispose === "undefined") { + Symbol.dispose = Symbol.for("Symbol.dispose"); +} +if (typeof Symbol.asyncDispose === "undefined") { + Symbol.asyncDispose = Symbol.for("Symbol.asyncDispose"); +} + +globalThis.__private_workflows = new Map(); +globalThis.__resolvers = {}; +globalThis.__pending = []; +globalThis.__workflowResult = undefined; +globalThis.__workflowError = undefined; +// Buffer for hook_received payloads that arrive before the hook is awaited. +// Keyed by correlationId → array of payloads (preserves delivery order). +// This mirrors the event-replay runtime's payloadsQueue in hook.ts. +globalThis.__hookPayloadBuffer = {}; + +// Stubs for Web APIs that the workflow bundle may reference but are not +// available in QuickJS. Native C extensions (encoding, base64, headers, +// url, structuredClone) provide the real implementations; these are +// minimal stubs for APIs that don't have native extensions yet. + +if (typeof ReadableStream === "undefined") { + // Minimal ReadableStream that stores body data for Response.json()/text() + globalThis.ReadableStream = function() {}; + globalThis.ReadableStream.prototype.__bodyData = null; +} + +if (typeof WritableStream === "undefined") { + globalThis.WritableStream = function() {}; +} + +if (typeof TransformStream === "undefined") { + globalThis.TransformStream = function() {}; +} + +if (typeof console === "undefined") { + globalThis.console = { log: function(){}, error: function(){}, warn: function(){}, info: function(){} }; +} +// Stub exports/module for CJS bundle format +globalThis.exports = {}; +globalThis.module = { exports: globalThis.exports }; +// NOTE: TextEncoder/TextDecoder are provided by the native encoding extension. + +globalThis[Symbol.for("WORKFLOW_USE_STEP")] = function(stepId, closureVarsFn) { + var fn = function() { + var args = Array.prototype.slice.call(arguments); + var correlationId = "step_" + globalThis.__generateUlid(); + // Capture 'this' for method invocations (e.g., MyClass.method()) + var thisVal = (this !== undefined && this !== null && this !== globalThis) ? this : undefined; + // Serialize step input using the host-provided devalue serializer. + // This produces a format-prefixed Uint8Array ("devl" + devalue.stringify). + var input = globalThis[Symbol.for("workflow-serialize")]({ + args: args, + closureVars: closureVarsFn ? closureVarsFn() : undefined, + thisVal: thisVal, + }); + globalThis.__pending.push({ + type: "step", + correlationId: correlationId, + stepId: stepId, + input: input, + hasCreatedEvent: false, + }); + return new Promise(function(resolve, reject) { + globalThis.__resolvers[correlationId] = { resolve: resolve, reject: reject }; + }); + }; + // Set stepId on the proxy so the StepFunction reducer can detect and + // serialize step function references (e.g. when passed as arguments). + fn.stepId = stepId; + if (closureVarsFn) fn.__closureVarsFn = closureVarsFn; + return fn; +}; + +// Parses an "ms" library style duration string into milliseconds. +// Supports the same units as the replay runtime (which uses the "ms" +// package): ms / s / m / h / d / w / y, with verbose aliases +// (seconds, minutes, ...). +globalThis.__parseDurationMs = function(str) { + str = String(str); + if (str.length > 100) return undefined; + var match = str.match( + /^(-?(?:\\d+)?\\.?\\d+) *(milliseconds?|msecs?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d|weeks?|w|years?|yrs?|y)?$/i + ); + if (!match) return undefined; + var n = parseFloat(match[1]); + var type = (match[2] || "ms").toLowerCase(); + var s = 1000, m = 60 * s, h = 60 * m, d = 24 * h, w = 7 * d, y = 365.25 * d; + switch (type) { + case "years": case "year": case "yrs": case "yr": case "y": return n * y; + case "weeks": case "week": case "w": return n * w; + case "days": case "day": case "d": return n * d; + case "hours": case "hour": case "hrs": case "hr": case "h": return n * h; + case "minutes": case "minute": case "mins": case "min": case "m": return n * m; + case "seconds": case "second": case "secs": case "sec": case "s": return n * s; + case "milliseconds": case "millisecond": case "msecs": case "msec": case "ms": return n; + default: return undefined; + } +}; + +globalThis[Symbol.for("WORKFLOW_SLEEP")] = function(param) { + var correlationId = "wait_" + globalThis.__generateUlid(); + var resumeAt; + if (typeof param === "number") { + resumeAt = new Date(Date.now() + param).toISOString(); + } else if (typeof param === "string") { + var ms = globalThis.__parseDurationMs(param); + if (typeof ms === "number" && isFinite(ms)) { + resumeAt = new Date(Date.now() + ms).toISOString(); + } else { + // Not a duration string — try as an absolute date string. + var date = new Date(param); + if (isNaN(date.getTime())) { + throw new Error("Invalid sleep parameter: " + param); + } + resumeAt = date.toISOString(); + } + } else if (param instanceof Date) { + if (isNaN(param.getTime())) { + throw new Error("Invalid sleep parameter: " + param); + } + resumeAt = param.toISOString(); + } else { + throw new Error("Invalid sleep parameter: " + param); + } + globalThis.__pending.push({ + type: "wait", + correlationId: correlationId, + resumeAt: resumeAt, + hasCreatedEvent: false, + }); + return new Promise(function(resolve, reject) { + globalThis.__resolvers[correlationId] = { resolve: resolve, reject: reject }; + }); +}; + +// Response/Request polyfills — .json()/.text()/.arrayBuffer() are useStep +// proxies that execute on the host side. The proxies are assigned directly +// to the prototypes so that 'this' (the Response/Request instance) is +// serialized as thisVal by WORKFLOW_USE_STEP, matching the event-replay +// runtime's approach (commit dcb0761). +if (typeof Response === "undefined") { + var __BODY_INIT = Symbol.for("BODY_INIT"); + + globalThis.Response = function(body, init) { + init = init || {}; + this.status = init.status || 200; + this.statusText = init.statusText || ""; + this.headers = new globalThis.Headers(init.headers || []); + this.type = "default"; + this.url = ""; + this.redirected = false; + if (body !== null && body !== undefined) { + this.body = Object.create(globalThis.ReadableStream.prototype); + this.body[__BODY_INIT] = body; + } else { + this.body = null; + } + }; + Object.defineProperty(globalThis.Response.prototype, "ok", { + get: function() { return this.status >= 200 && this.status < 300; } + }); + Object.defineProperty(globalThis.Response.prototype, "bodyUsed", { + get: function() { return false; } + }); + // Assign useStep proxies directly — 'this' binding provides the + // Response instance, which gets serialized as thisVal by the proxy. + Object.defineProperties(globalThis.Response.prototype, { + arrayBuffer: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_array_buffer"), writable: true, configurable: true }, + json: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_json"), writable: true, configurable: true }, + text: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_text"), writable: true, configurable: true }, + }); + globalThis.Response.prototype.bytes = function() { + return this.arrayBuffer().then(function(buf) { return new Uint8Array(buf); }); + }; + globalThis.Response.prototype.clone = function() { + var r = Object.create(globalThis.Response.prototype); + r.status = this.status; r.statusText = this.statusText; + r.headers = this.headers; r.type = this.type; + r.url = this.url; r.redirected = this.redirected; r.body = this.body; + return r; + }; + globalThis.Response.json = function(data, init) { + var body = JSON.stringify(data); + var headers = new globalThis.Headers(init ? init.headers : []); + if (!headers.has("content-type")) { headers.set("content-type", "application/json"); } + return new globalThis.Response(body, { status: (init && init.status) || 200, statusText: (init && init.statusText) || "", headers: headers }); + }; +} +if (typeof Request === "undefined") { + globalThis.Request = function(input, init) { + init = init || {}; + if (typeof input === "string") { this.url = input; } + else if (input && typeof input === "object") { + this.url = input.url || ""; this.method = input.method; + this.headers = input.headers; this.body = input.body; + } + if (init.method) this.method = init.method.toUpperCase(); + if (!this.method) this.method = "GET"; + if (init.headers) this.headers = new globalThis.Headers(init.headers); + if (!this.headers) this.headers = new globalThis.Headers(); + if (init.body !== undefined) this.body = init.body; + if (!this.body) this.body = null; + this.duplex = init.duplex || "half"; + }; + Object.defineProperty(globalThis.Request.prototype, "bodyUsed", { + get: function() { return false; } + }); + Object.defineProperties(globalThis.Request.prototype, { + arrayBuffer: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_array_buffer"), writable: true, configurable: true }, + json: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_json"), writable: true, configurable: true }, + text: { value: globalThis[Symbol.for("WORKFLOW_USE_STEP")]("__builtin_response_text"), writable: true, configurable: true }, + }); +} + +// createHook — returns a Hook object that is both a Thenable and AsyncIterable. +// Each await/yield creates a new promise keyed by the same correlationId. +// The promise is resolved when a hook_received event arrives. +globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")] = function(options) { + options = options || {}; + var token = options.token || globalThis.__generateNanoid(); + var correlationId = "hook_" + globalThis.__generateUlid(); + var isDisposed = false; + var hasCreatedEvent = false; + + // Register in pending operations. + // Serialize metadata inside the VM so Response/Request objects are + // properly handled by the devalue reducers before crossing the boundary. + globalThis.__pending.push({ + type: "hook", + correlationId: correlationId, + token: token, + isWebhook: !!options.isWebhook, + metadata: options.metadata ? globalThis[Symbol.for("workflow-serialize")](options.metadata) : undefined, + hasCreatedEvent: false, + }); + + // Each await creates a new promise for the next payload. + // The correlationId stays the same — the resolver is replaced each time. + function createHookPromise() { + // Check the payload buffer first — if a hook_received event arrived + // before this hook was awaited, the payload was buffered in the VM + // heap. Drain it immediately (matching event-replay payloadsQueue). + var buf = globalThis.__hookPayloadBuffer[correlationId]; + if (buf && buf.length > 0) { + return Promise.resolve(buf.shift()); + } + return new Promise(function(resolve, reject) { + globalThis.__resolvers[correlationId] = { resolve: resolve, reject: reject }; + }); + } + + function disposeHook() { + if (isDisposed) return; + isDisposed = true; + // Signal to the entrypoint to create a hook_disposed event + globalThis.__pending.push({ + type: "hook_dispose", + correlationId: correlationId, + hasCreatedEvent: false, + }); + // If there's a pending resolver, resolve it with undefined to break the iterator + if (globalThis.__resolvers[correlationId]) { + globalThis.__resolvers[correlationId].resolve(undefined); + delete globalThis.__resolvers[correlationId]; + } + } + + var hook = { + token: token, + then: function(onFulfilled, onRejected) { + return createHookPromise().then(onFulfilled, onRejected); + }, + dispose: disposeHook, + }; + + // Symbol.dispose for explicit resource management + hook[Symbol.dispose] = disposeHook; + + // AsyncIterable — yields payloads until disposed + hook[Symbol.asyncIterator] = function() { + return { + next: function() { + if (isDisposed) { + return Promise.resolve({ done: true, value: undefined }); + } + return createHookPromise().then(function(value) { + // If disposed while waiting, signal done + if (isDisposed) return { done: true, value: undefined }; + return { done: false, value: value }; + }); + }, + return: function() { + disposeHook(); + return Promise.resolve({ done: true, value: undefined }); + }, + }; + }; + + return hook; +}; + +// WORKFLOW_GET_STREAM_ID — generates a stream ID for a workflow run. +// Replicates getWorkflowRunStreamId() from util.ts inside the QuickJS VM. +// Uses native btoa() from the base64 extension for base64url encoding. +globalThis[Symbol.for("WORKFLOW_GET_STREAM_ID")] = function(namespace) { + var runId = globalThis[Symbol.for("WORKFLOW_CONTEXT")] + ? globalThis[Symbol.for("WORKFLOW_CONTEXT")].workflowRunId + : ""; + var streamId = runId.replace("wrun_", "strm_") + "_user"; + if (!namespace) return streamId; + // base64url: btoa then replace + with -, / with _, strip = + var b64 = btoa(namespace).replace(/\\+/g, "-").replace(/\\//g, "_").replace(/=+$/, ""); + return streamId + "_" + b64; +}; +`; + +// ---- Runtime ---- + +export async function runSnapshotWorkflow( + options: SnapshotRuntimeOptions +): Promise { + const { workflowCode, workflowId, workflowRun, events, existingSnapshot } = + options; + + const startedAt = workflowRun.startedAt ? +workflowRun.startedAt : Date.now(); + + // Mix the snapshot's events cursor into the PRNG seed so that each + // resumption draws from a different point in the sequence. Without this, + // every restore re-initialized the RNG from the same `runId:name:startedAt` + // seed and replayed the first-N draws, producing identical correlationIds + // across resumptions and breaking the hasCreatedEvent dedup guard. + // The cursor is stable for retries of the same resumption (idempotent + // within a single resume) but advances across resumes — exactly the + // determinism boundary we want. + const seedParts = [ + workflowRun.runId, + workflowRun.workflowName, + String(startedAt), + ]; + if (existingSnapshot?.metadata.eventsCursor) { + seedParts.push(existingSnapshot.metadata.eventsCursor); + } + const seed = seedParts.join(':'); + const rng = seedrandom(seed); + + let vm: QuickJS; + + // Seeded nanoid generator — uses the same nanoid package and seeded PRNG + // as the event-replay runtime for consistent token generation. + const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => + new Uint8Array(size).map(() => 256 * rng()) + ); + + if (existingSnapshot) { + // ---- RESTORE from snapshot ---- + const deserializeStart = performance.now(); + const snapshot = QuickJS.deserializeSnapshot(existingSnapshot.data); + const deserializeDurationMs = Math.round( + performance.now() - deserializeStart + ); + options.parentSpan?.setAttributes({ + ...Attribute.SnapshotDeserializeDurationMs(deserializeDurationMs), + }); + vm = await QuickJS.restore(snapshot, { + wasm: quickjsWasm, + // Use real time for Date.now() — determinism is handled by seeded Math.random + memoryLimit: 256 * 1024 * 1024, + interruptHandler: createInterruptHandler(), + extensions: quickjsExtensions, + }); + + // Re-register host callbacks after restore. Host functions are stored + // in the WASM heap by name. After restore, the host callback registry + // is empty — we must re-register each callback with the same name + // used during newFunction() in the first-run path. + vm.registerHostCallback('random', () => vm.newNumber(rng())); + vm.registerHostCallback('__generateNanoid', () => + vm.newString(generateNanoid()) + ); + + // Note: globalThis[Symbol.for('workflow-serialize')] and + // globalThis[Symbol.for('workflow-deserialize')] are JS functions + // in the VM (set by the serde bundle), so they survive + // snapshot/restore as part of the QuickJS heap. No re-registration + // needed. + + // Process events and drain jobs in a loop. Events may resolve promises + // that unblock workflow code, which then creates NEW resolvers for + // subsequent events. Re-processing events matches these new resolvers + // against events that were already delivered. + let maxIterations = 100; + let madeProgress: boolean; + do { + madeProgress = await processEvents(vm, events, options.encryptionKey); + let batch: number; + do { + batch = vm.executePendingJobs(); + if (batch > 0) madeProgress = true; + } while (batch > 0); + } while (madeProgress && --maxIterations > 0); + } else { + // ---- FIRST RUN ---- + vm = await QuickJS.create({ + wasm: quickjsWasm, + // Use real time for Date.now() — determinism is handled by seeded Math.random + memoryLimit: 256 * 1024 * 1024, + interruptHandler: createInterruptHandler(), + extensions: quickjsExtensions, + }); + + // Seeded Math.random — host callback ID = baseId + { + using randomFn = vm.newFunction('random', () => vm.newNumber(rng())); + using math = vm.global.getProp('Math'); + math.setProp('random', randomFn); + } + + // Seeded nanoid generator — host callback ID = baseId + 1 + { + using nanoidFn = vm.newFunction('__generateNanoid', () => + vm.newString(generateNanoid()) + ); + vm.setProp(vm.global, '__generateNanoid', nanoidFn); + } + + // Inject a deterministic timestamp for the VM's ULID factory. ULIDs + // produced inside the VM use this as their time prefix instead of + // Date.now(), so two concurrent workflow invocations of the same + // resumption produce IDENTICAL correlationIds (the random portion + // also matches because the PRNG is seeded the same way) and the + // world's EntityConflictError on `events.create` dedups one of each + // pair. Use `startedAt` (constant per-run) — distinctness across + // resumptions comes from the cursor mixed into the seedrandom seed, + // which advances the PRNG sequence between resumes. + vm.evalCode(`globalThis.__ulidTimestamp = ${startedAt};`).dispose(); + + // Evaluate the VM serde bundle + vm.evalCode(VM_SERDE_BUNDLE, 'vm-serde.js').dispose(); + + // Bootstrap workflow primitives + vm.evalCode(VM_BOOTSTRAP, 'bootstrap.js').dispose(); + + // Execute the workflow bundle — use the workflowId as the eval filename + // so QuickJS stack traces reference the workflow name, enabling source map + // remapping by remapErrorStack (which matches frames by filename). + try { + vm.evalCode(workflowCode, workflowId || 'workflow.js').dispose(); + } catch (err) { + return extractError(vm, err, 'Workflow evaluation failed'); + } + + // Extract workflow arguments. Prefer the run_created event; fall back + // to the queue message's runInput if the event log is incomplete + // (eventually-consistent read after start()). Failing to find input + // for a first invocation is fatal — running the workflow function + // with no args would silently turn typed arguments into `undefined` + // and, for recursive workflows, produce exponential fan-out. + const runCreatedEvent = events.find((e) => e.eventType === 'run_created'); + const runCreatedInput = + runCreatedEvent && 'eventData' in runCreatedEvent + ? (runCreatedEvent.eventData as Record)?.input + : undefined; + const runInput: unknown = + runCreatedInput ?? (options.runInput?.input as unknown); + + if (runInput instanceof Uint8Array) { + const decryptedInput = (await decryptData( + runInput, + options.encryptionKey + )) as Uint8Array; + runtimeLogger.debug('Snapshot runtime: run input format', { + prefix: new TextDecoder().decode(decryptedInput.subarray(0, 4)), + byteLength: decryptedInput.byteLength, + source: runCreatedInput ? 'run_created' : 'queueMessage.runInput', + }); + const inputHandle = vm.newUint8Array(decryptedInput); + vm.setProp(vm.global, '__wdk_input', inputHandle); + inputHandle.dispose(); + } else if (runInput === undefined && events.length > 0) { + // The event log is non-empty (we got run_started or similar) but + // no run_created event was found and no queue-provided runInput is + // available. This is the race condition observed during the fib + // incident — silently dropping arguments would turn `n` into + // `undefined` and, for recursive workflows, cause exponential + // fan-out. Fail loud so the run goes to `run_failed` and the queue + // can retry. Empty `events` is allowed because tests that bootstrap + // a workflow with no arguments rely on the old permissive behavior. + throw new Error( + `Cannot start workflow run "${workflowRun.runId}": no run_created event found and no runInput in the queue payload, but other events are present (likely a read-after-write race during start()).` + ); + } + + // Set workflow context metadata (for getWorkflowMetadata()). + // Must match the shape that the replay runtime produces (see + // packages/core/src/workflow.ts: runWorkflow → ctx) so user code + // that compares `getWorkflowMetadata()` values between a step + // (server-side) and the workflow (VM-side) sees identical objects. + { + const metadata = { + workflowName: workflowRun.workflowName, + workflowRunId: workflowRun.runId, + workflowStartedAt: workflowRun.startedAt + ? new Date(+workflowRun.startedAt) + : new Date(), + url: process.env.VERCEL_URL + ? `https://${process.env.VERCEL_URL}` + : `http://localhost:${options.port ?? 3000}`, + features: { encryption: !!options.encryptionKey }, + }; + vm.evalCode( + `globalThis[Symbol.for("WORKFLOW_CONTEXT")] = ${JSON.stringify(metadata)};` + + `globalThis[Symbol.for("WORKFLOW_CONTEXT")].workflowStartedAt = new Date(${JSON.stringify(metadata.workflowStartedAt.toISOString())});` + ).dispose(); + } + + // Start the workflow function. If the workflow isn't registered, + // throw an error tagged with `name = "WorkflowNotRegisteredError"` + // so the host-side entrypoint can reconstruct a real + // WorkflowNotRegisteredError (a WorkflowRuntimeError subclass that + // classifies as RUNTIME_ERROR) rather than a generic user error. + // See snapshot-entrypoint.ts's run_failed branch. + try { + vm.evalCode(` + var __wfn = globalThis.__private_workflows.get(${JSON.stringify(workflowId)}); + if (!__wfn) { + var __wfnErr = new Error("Workflow \\"" + ${JSON.stringify(workflowId)} + "\\" is not registered in the current deployment."); + __wfnErr.name = "WorkflowNotRegisteredError"; + throw __wfnErr; + } + var __args = globalThis.__wdk_input + ? globalThis[Symbol.for("workflow-deserialize")](globalThis.__wdk_input) + : []; + delete globalThis.__wdk_input; + if (!Array.isArray(__args)) __args = [__args]; + __wfn.apply(null, __args).then( + function(result) { globalThis.__workflowResult = globalThis[Symbol.for("workflow-serialize")](result); }, + function(error) { + // Preserve display info on the host-side failed object + // (matches the legacy host-visible shape) AND serialize the + // entire thrown value so the host can dehydrate the original + // type-identity, cause chain, or non-Error throws verbatim + // through the standard error pipeline. + globalThis.__workflowError = { + message: error && error.message != null ? String(error.message) : String(error), + stack: error && error.stack ? error.stack : "", + name: error && error.name ? error.name : (error instanceof Error ? "Error" : typeof error), + valueBytes: globalThis[Symbol.for("workflow-serialize")](error), + }; + } + ); + `).dispose(); + } catch (err) { + return extractError(vm, err, 'Failed to start workflow'); + } + + // Process events and drain jobs in a loop (same as restore path) + { + let maxIterations = 100; + let madeProgress: boolean; + do { + madeProgress = await processEvents(vm, events, options.encryptionKey); + let batch: number; + do { + batch = vm.executePendingJobs(); + if (batch > 0) madeProgress = true; + } while (batch > 0); + } while (madeProgress && --maxIterations > 0); + } + } + + // ---- Check result ---- + return checkWorkflowState(vm, options.parentSpan); +} + +// ---- Event Processing ---- + +async function processEvents( + vm: QuickJS, + events: Event[], + encryptionKey?: CryptoKey +): Promise { + let resolved = false; + for (const event of events) { + const cid = event.correlationId; + if (!cid) continue; + + const escapedCid = cid.replace(/"/g, '\\"'); + const eventData = + 'eventData' in event + ? (event.eventData as Record) + : undefined; + + // Log the event and whether the resolver exists + switch (event.eventType) { + case 'step_completed': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + const rawOutput = eventData?.result ?? eventData?.output; + if (hasResolver) { + if (rawOutput instanceof Uint8Array) { + // Decrypt if encrypted — the VM only understands 'devl' format + runtimeLogger.debug('Snapshot runtime: step result raw', { + correlationId: escapedCid, + rawPrefix: new TextDecoder().decode(rawOutput.subarray(0, 4)), + rawByteLength: rawOutput.byteLength, + isBuffer: Buffer.isBuffer(rawOutput), + }); + const decryptedOutput = (await decryptData( + rawOutput, + encryptionKey + )) as Uint8Array; + runtimeLogger.debug('Snapshot runtime: step result decrypted', { + correlationId: escapedCid, + prefix: new TextDecoder().decode(decryptedOutput.subarray(0, 4)), + byteLength: decryptedOutput.byteLength, + }); + const bytesHandle = vm.newUint8Array(decryptedOutput); + vm.setProp(vm.global, '__tmp_result', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + + `delete globalThis.__resolvers["${escapedCid}"];` + + `delete globalThis.__tmp_result;` + ).dispose(); + } else { + runtimeLogger.debug('Snapshot runtime: step result non-binary', { + correlationId: escapedCid, + type: typeof rawOutput, + isNull: rawOutput === null, + isUndefined: rawOutput === undefined, + constructor: rawOutput?.constructor?.name, + }); + const serialized = + rawOutput !== undefined ? JSON.stringify(rawOutput) : 'undefined'; + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(${serialized});` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + } + // Drain ALL microtasks after resolve + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'step_failed': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + if (hasResolver) { + const errorData = eventData?.error; + if (errorData instanceof Uint8Array) { + // Modern path (post-#1851): the step handler dehydrated the + // thrown value through the first-class error pipeline. Decrypt + // (if encrypted) and pass the bytes to the VM-side deserializer + // so the workflow catch sees a properly typed Error subclass + // (TypeError, FatalError with original cause chain, etc.) with + // the original message and stack preserved. + const decrypted = (await decryptData( + errorData, + encryptionKey + )) as Uint8Array; + const bytesHandle = vm.newUint8Array(decrypted); + vm.setProp(vm.global, '__tmp_error', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + `(function(){` + + `var e=globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_error);` + + `globalThis.__resolvers["${escapedCid}"].reject(e);` + + `delete globalThis.__resolvers["${escapedCid}"];` + + `delete globalThis.__tmp_error;` + + `})()` + ).dispose(); + } else { + // Legacy path: pre-pipeline events stored error as + // `{ message, stack, code }`. Reconstruct a FatalError so + // workflow catch can detect it via FatalError.is(), matching + // the original V1 step handler behavior. + const isErrorObject = + typeof errorData === 'object' && errorData !== null; + const msg = isErrorObject + ? (((errorData as Record).message as string) ?? + 'Step failed') + : typeof errorData === 'string' + ? errorData + : 'Step failed'; + const errorStack = + (isErrorObject + ? (errorData as Record).stack + : undefined) ?? (eventData?.stack as string | undefined); + const stackAssignment = errorStack + ? `e.stack=${JSON.stringify(errorStack)};` + : ''; + vm.evalCode( + `(function(){var e=new Error(${JSON.stringify(msg)});e.name="FatalError";e.fatal=true;${stackAssignment}` + + `globalThis.__resolvers["${escapedCid}"].reject(e);` + + `delete globalThis.__resolvers["${escapedCid}"];})()` + ).dispose(); + } + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'wait_completed': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + if (hasResolver) { + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve();` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'hook_received': { + // Check if this event was already processed (delivered or buffered) + // in this invocation or a prior one (tracked in the VM heap so it + // survives snapshot/restore). Prevents double-delivery when the + // outer loop re-scans events. + const alreadyProcessed = event.eventId + ? vm.dump( + vm.evalCode( + `!!(globalThis.__hookPayloadBuffer.__processedEventIds && globalThis.__hookPayloadBuffer.__processedEventIds[${JSON.stringify(event.eventId)}])` + ) + ) + : false; + if (alreadyProcessed) { + runtimeLogger.debug( + 'Snapshot runtime: hook_received already processed', + { + correlationId: cid, + eventId: event.eventId, + } + ); + markCreated(vm, escapedCid); + break; + } + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + const rawPayload = eventData?.payload ?? eventData?.result; + runtimeLogger.debug('Snapshot runtime: processing hook_received', { + correlationId: cid, + eventId: event.eventId, + hasResolver, + payloadType: typeof rawPayload, + payloadIsUint8Array: rawPayload instanceof Uint8Array, + payloadKeys: + rawPayload && typeof rawPayload === 'object' + ? Object.keys(rawPayload) + : undefined, + }); + if (hasResolver) { + if (rawPayload instanceof Uint8Array) { + // Decrypt if encrypted — the VM only understands 'devl' format + const decryptedPayload = (await decryptData( + rawPayload, + encryptionKey + )) as Uint8Array; + const bytesHandle = vm.newUint8Array(decryptedPayload); + vm.setProp(vm.global, '__tmp_result', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result));` + + `delete globalThis.__resolvers["${escapedCid}"];` + + `delete globalThis.__tmp_result;` + ).dispose(); + } else { + const serialized = + rawPayload !== undefined + ? JSON.stringify(rawPayload) + : 'undefined'; + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].resolve(${serialized});` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + } + // Mark this event as processed in the VM heap to prevent + // double-delivery on re-scan or snapshot restore. + if (event.eventId) { + vm.evalCode( + `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${JSON.stringify(event.eventId)}] = true;` + ).dispose(); + } + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } else { + // No resolver yet — buffer the payload in the VM heap so it + // survives snapshot/restore. When createHookPromise() is called + // later, it will drain this buffer first (matching the event- + // replay runtime's payloadsQueue behavior). + const eventIdJs = event.eventId + ? JSON.stringify(event.eventId) + : 'null'; + const bufferAndTrack = + `(globalThis.__hookPayloadBuffer["${escapedCid}"] = globalThis.__hookPayloadBuffer["${escapedCid}"] || [])` + + `.push(%PAYLOAD%);` + + (event.eventId + ? `(globalThis.__hookPayloadBuffer.__processedEventIds = globalThis.__hookPayloadBuffer.__processedEventIds || {})[${eventIdJs}] = true;` + : ''); + if (rawPayload instanceof Uint8Array) { + // Decrypt if encrypted — the VM only understands 'devl' format + const decryptedPayload = (await decryptData( + rawPayload, + encryptionKey + )) as Uint8Array; + const bytesHandle = vm.newUint8Array(decryptedPayload); + vm.setProp(vm.global, '__tmp_result', bytesHandle); + bytesHandle.dispose(); + vm.evalCode( + bufferAndTrack.replace( + '%PAYLOAD%', + 'globalThis[Symbol.for("workflow-deserialize")](globalThis.__tmp_result)' + ) + 'delete globalThis.__tmp_result;' + ).dispose(); + } else { + const serialized = + rawPayload !== undefined + ? JSON.stringify(rawPayload) + : 'undefined'; + vm.evalCode( + bufferAndTrack.replace('%PAYLOAD%', serialized) + ).dispose(); + } + } + markCreated(vm, escapedCid); + break; + } + case 'hook_conflict': { + const hasResolver = vm.dump( + vm.evalCode(`!!globalThis.__resolvers["${escapedCid}"]`) + ); + if (hasResolver) { + const conflictToken = (eventData?.token as string) ?? 'unknown'; + vm.evalCode( + `globalThis.__resolvers["${escapedCid}"].reject(new Error(${JSON.stringify(`Hook token "${conflictToken}" is already in use by another workflow`)}));` + + `delete globalThis.__resolvers["${escapedCid}"];` + ).dispose(); + { + resolved = true; + let b: number; + do { + b = vm.executePendingJobs(); + } while (b > 0); + } + } + markCreated(vm, escapedCid); + break; + } + case 'step_created': + case 'step_started': + case 'step_retrying': + case 'wait_created': + case 'hook_created': { + markCreated(vm, escapedCid); + break; + } + case 'hook_disposed': { + // Disambiguate from the `hook` pending op with the same + // correlationId — we want to mark the `hook_dispose` entry. + markCreated(vm, escapedCid, 'hook_dispose'); + break; + } + } + } + return resolved; +} + +function markCreated(vm: QuickJS, escapedCid: string, opType?: string): void { + // `hook` and `hook_dispose` pending ops share the same correlationId, + // so when processing `hook_disposed` events we must disambiguate by + // type — otherwise `.find()` returns the original `hook` op and the + // `hook_dispose` op is never marked, causing the entrypoint to keep + // retrying a hook_disposed for an already-deleted entity. + const predicate = opType + ? `function(p){return p.correlationId==="${escapedCid}"&&p.type==="${opType}";}` + : `function(p){return p.correlationId==="${escapedCid}";}`; + vm.evalCode( + `var __p=globalThis.__pending.find(${predicate});` + + `if(__p)__p.hasCreatedEvent=true;` + ).dispose(); +} + +// ---- State Checking ---- + +function checkWorkflowState( + vm: QuickJS, + parentSpan?: Span +): SnapshotRuntimeResult { + // Check completed — __workflowResult is a format-prefixed Uint8Array + { + using h = vm.evalCode('globalThis.__workflowResult'); + if (!h.isUndefined) { + const resultBytes = h.toUint8Array(); + vm.dispose(); + return { completed: { result: resultBytes } }; + } + } + + // Check failed + { + using h = vm.evalCode('globalThis.__workflowError'); + if (!h.isUndefined) { + const errorObj = vm.dump(h) as + | { + message: string; + stack?: string; + name?: string; + valueBytes?: Uint8Array; + } + | string; + const failed = + typeof errorObj === 'string' + ? { message: errorObj } + : { + message: errorObj.message, + stack: errorObj.stack || undefined, + name: errorObj.name || undefined, + valueBytes: errorObj.valueBytes, + }; + runtimeLogger.error('Snapshot runtime: workflow failed in VM', { + errorMessage: failed.message, + errorName: failed.name, + errorStack: failed.stack, + }); + vm.dispose(); + return { failed }; + } + } + + // Check suspended — the workflow is suspended if there are active resolvers + // OR pending operations that haven't been created yet (e.g. hooks created + // upfront but not yet awaited) + { + using h = vm.evalCode( + 'Object.keys(globalThis.__resolvers).length > 0 || globalThis.__pending.some(function(p){return!p.hasCreatedEvent;})' + ); + if (vm.dump(h)) { + using pendingH = vm.evalCode( + `globalThis.__pending.filter(function(p){return!!globalThis.__resolvers[p.correlationId] || !p.hasCreatedEvent;})` + ); + const pendingOps = vm.dump(pendingH) as PendingOperation[]; + + const serializeStart = performance.now(); + const snapshot = vm.snapshot(); + const serialized = QuickJS.serializeSnapshot(snapshot); + const serializeDurationMs = Math.round( + performance.now() - serializeStart + ); + parentSpan?.setAttributes({ + ...Attribute.SnapshotSerializeDurationMs(serializeDurationMs), + }); + vm.dispose(); + + runtimeLogger.debug('Snapshot runtime: serialized snapshot', { + type: typeof serialized, + byteLength: serialized?.byteLength, + length: serialized?.length, + durationMs: serializeDurationMs, + }); + + return { + suspended: { + pendingOperations: pendingOps, + snapshot: serialized, + }, + }; + } + } + + vm.dispose(); + return { failed: { message: 'Workflow ended in unknown state' } }; +} + +// ---- Helpers ---- + +function extractError( + vm: QuickJS, + err: unknown, + fallbackMessage: string +): SnapshotRuntimeResult { + let message = fallbackMessage; + let stack: string | undefined; + let name: string | undefined; + + if (err instanceof JSException) { + const error = vm.dump(err.handle) as Record | null; + err.handle.dispose(); + message = (error?.message as string) ?? err.message ?? fallbackMessage; + stack = (error?.stack as string) ?? err.stack; + name = (error?.name as string) ?? err.name; + } else if (err instanceof Error) { + message = err.message ?? fallbackMessage; + stack = err.stack; + name = err.name; + } + + vm.dispose(); + return { + failed: { message, stack, name }, + }; +} + +function createInterruptHandler(): () => boolean { + const start = Date.now(); + const timeout = 30_000; + return () => Date.now() - start > timeout; +} diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index 6f20e81c2a..2051cf4e33 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -20,9 +20,10 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { serializeTraceCarrier, trace } from '../telemetry.js'; import { waitedUntil } from '../util.js'; import { version as workflowCoreVersion } from '../version.js'; +import { getWorldLazy } from './get-world-lazy.js'; import { getWorkflowQueueName } from './helpers.js'; import { Run } from './run.js'; -import { getWorldLazy } from './get-world-lazy.js'; +import { getWorkflowRuntimeFromEnv } from './runtime-mode.js'; /** ULID generator for client-side runId generation */ const ulid = monotonicFactory(); @@ -207,10 +208,16 @@ export async function start( v1Compat ); + // If WORKFLOW_RUNTIME is set on the client starting the run, propagate + // that choice through to the runtime so the same deployment can serve + // both runtimes. Unknown values throw — see getWorkflowRuntimeFromEnv(). + const workflowRuntime = getWorkflowRuntimeFromEnv(); + const executionContext = { traceCarrier, workflowCoreVersion, features: { encryption: !!encryptionKey }, + ...(workflowRuntime ? { workflowRuntime } : {}), }; // Call events.create (run_created) and queue in parallel. diff --git a/packages/core/src/serialization/codec-devalue-vm.ts b/packages/core/src/serialization/codec-devalue-vm.ts new file mode 100644 index 0000000000..3713132058 --- /dev/null +++ b/packages/core/src/serialization/codec-devalue-vm.ts @@ -0,0 +1,93 @@ +/** + * VM-compatible devalue codec. + * + * Same as codec-devalue.ts but uses VM-compatible reducers/revivers + * (no Node.js Buffer, no node:util). Safe to bundle into the QuickJS VM. + */ + +import { parse, stringify, unflatten } from 'devalue'; +import { SerializationFormat, type Reducers, type Revivers } from './types.js'; +import type { Codec, SerializationMode } from './codec.js'; +import { getClassReducers, getClassRevivers } from './reducers/class.js'; +import { getCommonReducers, getCommonRevivers } from './reducers/common-vm.js'; +import { + getStepFunctionReducer, + getStepFunctionReviver, +} from './reducers/step-function.js'; + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +function getReducersForMode(mode: SerializationMode): Partial { + switch (mode) { + case 'workflow': + return { + ...getClassReducers(), + ...getStepFunctionReducer(), + ...getCommonReducers(), + }; + case 'step': + return { + ...getClassReducers(), + ...getCommonReducers(), + }; + case 'client': + return { + ...getClassReducers(), + ...getCommonReducers(), + }; + } +} + +function getReviversForMode(mode: SerializationMode): Partial { + switch (mode) { + case 'workflow': + return { + ...getClassRevivers(), + ...getStepFunctionReviver(), + ...getCommonRevivers(), + }; + case 'step': + return { + ...getClassRevivers(), + ...getCommonRevivers(), + }; + case 'client': + return { + ...getClassRevivers(), + ...getCommonRevivers(), + StepFunction: () => { + throw new Error( + 'Step functions cannot be deserialized in client context.' + ); + }, + }; + } +} + +export const devalueVmCodec: Codec = { + formatPrefix: SerializationFormat.DEVALUE_V1, + + serialize(value: unknown, mode: SerializationMode): Uint8Array { + const reducers = getReducersForMode(mode); + const str = stringify( + value, + reducers as Record any> + ); + return encoder.encode(str); + }, + + deserialize(data: Uint8Array, mode: SerializationMode): unknown { + const revivers = getReviversForMode(mode); + const str = decoder.decode(data); + return parse(str, revivers as Record any>); + }, + + deserializeLegacy(data: unknown, mode: SerializationMode): unknown { + const revivers = getReviversForMode(mode); + return unflatten( + data as any[], + revivers as Record any> + ); + }, +}; diff --git a/packages/core/src/serialization/compat.test.ts b/packages/core/src/serialization/compat.test.ts new file mode 100644 index 0000000000..b8e2580d1c --- /dev/null +++ b/packages/core/src/serialization/compat.test.ts @@ -0,0 +1,185 @@ +/** + * Compatibility tests: verify that data serialized by the new modules + * can be deserialized by the old serialization.ts functions, and vice versa. + * + * This ensures the new modules are safe to use alongside the old code + * during the migration period. + */ + +import { describe, it, expect } from 'vitest'; +import * as workflow from './workflow.js'; +import * as step from './step.js'; +import * as client from './client.js'; +import { + dehydrateWorkflowArguments, + hydrateWorkflowArguments, + dehydrateWorkflowReturnValue, + hydrateWorkflowReturnValue, + dehydrateStepArguments, + hydrateStepArguments, + dehydrateStepReturnValue, + hydrateStepReturnValue, +} from '../serialization.js'; +import { importKey } from '../encryption.js'; + +const testData = { + primitives: [42, 'hello', true, null], + date: new Date('2025-06-15T12:00:00Z'), + error: Object.assign(new Error('test'), { name: 'TypeError' }), + map: new Map([ + ['a', 1], + ['b', 2], + ]), + set: new Set([1, 2, 3]), + bigint: 9007199254740993n, + uint8: new Uint8Array([1, 2, 3]), + url: new URL('https://example.com'), + regexp: /foo.*bar/gi, + nested: { + items: [1, 'two', new Date('2025-01-01')], + inner: { x: 42 }, + }, +}; + +describe('new workflow.serialize → old hydrateStepReturnValue', () => { + it('should round-trip primitives', async () => { + for (const val of testData.primitives) { + const serialized = workflow.serialize(val); + const hydrated = await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + ); + expect(hydrated).toEqual(val); + } + }); + + it('should round-trip Date', async () => { + const serialized = workflow.serialize(testData.date); + const hydrated = (await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + )) as Date; + expect(hydrated).toBeInstanceOf(Date); + expect(hydrated.toISOString()).toBe('2025-06-15T12:00:00.000Z'); + }); + + it('should round-trip Map', async () => { + const serialized = workflow.serialize(testData.map); + const hydrated = (await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + )) as Map; + expect(hydrated).toBeInstanceOf(Map); + expect(hydrated.get('a')).toBe(1); + }); + + it('should round-trip nested objects', async () => { + const serialized = workflow.serialize(testData.nested); + const hydrated = (await hydrateStepReturnValue( + serialized, + 'run-123', + undefined + )) as any; + expect(hydrated.items[0]).toBe(1); + expect(hydrated.items[2]).toBeInstanceOf(Date); + expect(hydrated.inner.x).toBe(42); + }); +}); + +describe('old dehydrateStepReturnValue → new workflow.deserialize', () => { + it('should round-trip primitives', async () => { + for (const val of testData.primitives) { + const dehydrated = await dehydrateStepReturnValue( + val, + 'run-123', + undefined, + [] + ); + const deserialized = workflow.deserialize(dehydrated); + expect(deserialized).toEqual(val); + } + }); + + it('should round-trip Date', async () => { + const dehydrated = await dehydrateStepReturnValue( + testData.date, + 'run-123', + undefined, + [] + ); + const deserialized = workflow.deserialize(dehydrated) as Date; + expect(deserialized).toBeInstanceOf(Date); + expect(deserialized.toISOString()).toBe('2025-06-15T12:00:00.000Z'); + }); + + it('should round-trip nested objects', async () => { + const dehydrated = await dehydrateStepReturnValue( + testData.nested, + 'run-123', + undefined, + [] + ); + const deserialized = workflow.deserialize(dehydrated) as any; + expect(deserialized.items[0]).toBe(1); + expect(deserialized.items[2]).toBeInstanceOf(Date); + }); +}); + +describe('old dehydrateWorkflowArguments → new workflow.deserialize', () => { + it('should round-trip when unencrypted', async () => { + const dehydrated = await dehydrateWorkflowArguments( + [42, 'hello'], + 'run-123', + undefined + ); + const deserialized = workflow.deserialize(dehydrated); + expect(deserialized).toEqual([42, 'hello']); + }); +}); + +describe('new client.serialize → old hydrateWorkflowArguments', () => { + it('should round-trip when unencrypted', async () => { + const serialized = await client.serialize([42, 'hello']); + const hydrated = await hydrateWorkflowArguments( + serialized, + 'run-123', + undefined + ); + expect(hydrated).toEqual([42, 'hello']); + }); +}); + +describe('encryption compat: new step.serialize → old hydrateStepArguments', () => { + it('should round-trip with encryption', async () => { + const rawKey = new Uint8Array(32); + rawKey.fill(0x42); + const key = await importKey(rawKey); + + const value = { x: 42, date: new Date('2025-01-01') }; + const serialized = await step.serialize(value, key); + const hydrated = (await hydrateStepArguments( + serialized, + 'run-123', + key + )) as any; + expect(hydrated.x).toBe(42); + expect(hydrated.date).toBeInstanceOf(Date); + }); +}); + +describe('encryption compat: old dehydrateStepArguments → new step.deserialize', () => { + it('should round-trip with encryption', async () => { + const rawKey = new Uint8Array(32); + rawKey.fill(0x42); + const key = await importKey(rawKey); + + const value = { x: 42, date: new Date('2025-01-01') }; + const dehydrated = await dehydrateStepArguments(value, 'run-123', key); + const deserialized = (await step.deserialize(dehydrated, key)) as any; + expect(deserialized.x).toBe(42); + expect(deserialized.date).toBeInstanceOf(Date); + }); +}); diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts new file mode 100644 index 0000000000..ec3eed7043 --- /dev/null +++ b/packages/core/src/serialization/compression.test.ts @@ -0,0 +1,105 @@ +import { gzipSync } from 'node:zlib'; +import { describe, expect, it } from 'vitest'; +import { + compress, + decompress, + isCompressed, + PREFERRED_CODEC, +} from './compression.js'; +import { decodeFormatPrefix, peekFormatPrefix } from './format.js'; +import { SerializationFormat } from './types.js'; + +describe('compress / decompress', () => { + it('round-trips a small payload', () => { + const input = new TextEncoder().encode('hello world'); + const compressed = compress(input) as Uint8Array; + expect(compressed).toBeInstanceOf(Uint8Array); + expect(compressed).not.toEqual(input); + const decompressed = decompress(compressed) as Uint8Array; + expect(Array.from(decompressed)).toEqual(Array.from(input)); + }); + + it('round-trips a highly-redundant 1MB payload (compresses well)', () => { + const input = new Uint8Array(1024 * 1024).fill(0x41); // all 'A' + const compressed = compress(input) as Uint8Array; + // Should compress massively — >100x + expect(compressed.byteLength).toBeLessThan(input.byteLength / 100); + const decompressed = decompress(compressed) as Uint8Array; + expect(decompressed.byteLength).toBe(input.byteLength); + // Spot-check first/last bytes (full deepEqual on 1MB Uint8Array is slow) + expect(decompressed[0]).toBe(0x41); + expect(decompressed[decompressed.byteLength - 1]).toBe(0x41); + }); + + it('uses the preferred codec format prefix', () => { + const input = new TextEncoder().encode('test data'); + const compressed = compress(input); + const prefix = peekFormatPrefix(compressed); + if (PREFERRED_CODEC === 'zstd') { + expect(prefix).toBe(SerializationFormat.ZSTD); + } else { + expect(prefix).toBe(SerializationFormat.GZIP); + } + }); + + it('returns non-binary inputs unchanged', () => { + expect(compress('a string' as unknown)).toBe('a string'); + expect(compress(42 as unknown)).toBe(42); + expect(compress(null as unknown)).toBe(null); + expect(compress(undefined as unknown)).toBe(undefined); + expect(decompress('a string' as unknown)).toBe('a string'); + }); + + it('is idempotent on already-compressed payloads', () => { + const input = new TextEncoder().encode('data to compress'); + const compressed = compress(input) as Uint8Array; + const reCompressed = compress(compressed) as Uint8Array; + // Second call must short-circuit and return the same Uint8Array, not + // double-wrap it. Identity check: same reference. + expect(reCompressed).toBe(compressed); + }); + + it('decompress passes through payloads with no compression prefix', () => { + const raw = new TextEncoder().encode('not compressed, no prefix'); + expect(decompress(raw)).toBe(raw); + }); + + it('decompress can read gzip-prefixed blobs even when zstd is preferred', () => { + // Construct a gzip blob manually so we always have one regardless of + // PREFERRED_CODEC. The decoder side must always handle gzip — older + // deployments may have written gzip even when newer ones write zstd. + const innerPayload = new TextEncoder().encode('round trip me'); + const gzipPayload = gzipSync(innerPayload); + const prefix = new TextEncoder().encode('gzip'); + const blob = new Uint8Array(prefix.length + gzipPayload.length); + blob.set(prefix, 0); + blob.set(gzipPayload, prefix.length); + + expect(peekFormatPrefix(blob)).toBe(SerializationFormat.GZIP); + const out = decompress(blob) as Uint8Array; + expect(Array.from(out)).toEqual(Array.from(innerPayload)); + }); + + it('isCompressed identifies compressed payloads', () => { + expect(isCompressed(compress(new Uint8Array([1, 2, 3])))).toBe(true); + expect(isCompressed(new Uint8Array([1, 2, 3]))).toBe(false); + expect(isCompressed('a string' as unknown)).toBe(false); + expect(isCompressed(undefined as unknown)).toBe(false); + }); +}); + +describe('PREFERRED_CODEC feature detection', () => { + it('reports a known codec', () => { + expect(['zstd', 'gzip']).toContain(PREFERRED_CODEC); + }); + + it('matches the codec actually emitted by compress()', () => { + const compressed = compress(new TextEncoder().encode('abc')) as Uint8Array; + const { format } = decodeFormatPrefix(compressed); + if (PREFERRED_CODEC === 'zstd') { + expect(format).toBe(SerializationFormat.ZSTD); + } else { + expect(format).toBe(SerializationFormat.GZIP); + } + }); +}); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts new file mode 100644 index 0000000000..19fb3a5002 --- /dev/null +++ b/packages/core/src/serialization/compression.ts @@ -0,0 +1,151 @@ +/** + * Composable compression layer for serialized data. + * + * Wraps/unwraps payloads with gzip or zstd (Node 22.15+) compression, + * using the format-prefix system to mark compressed data. + * + * Why is compression a separate, opt-in layer (not in + * `serialization/encryption.ts`)? Compression only pays off for + * larger payloads — gzip/zstd headers (~10-20 bytes) and a CPU pass + * are wasted on KB-scale CBOR/devalue payloads. The snapshot save + * path is the only call site today; small payloads (events, hook + * metadata) skip compression entirely. + * + * For the QuickJS heap snapshots produced by `runSnapshotWorkflow`, + * compression dominates encrypt() in the wire-bytes equation — + * encryption produces ~random ciphertext that doesn't compress, so + * `gzip(encrypt(plain))` is wasted work. The intended composition is + * `encrypt(compress(plain))`: compress first while data is still + * compressible, then encrypt the (small) result. + * + * Codec choice (benchmarked against an 8 MB QuickJS heap snapshot): + * + * | codec | ratio | compress | decompress | + * |--------|-------|----------|------------| + * | zstd-3 | 4.29x | 18 ms | 6 ms | + * | gzip-6 | 4.02x | 127 ms | 11 ms | + * + * zstd wins on ratio AND speed, but `node:zlib` only exposes it from + * Node 22.15. We feature-detect at module init and fall back to gzip + * on older Node versions. The format prefix on the saved blob marks + * which codec was used, so an in-flight workflow whose snapshot was + * written by one codec remains decodable after a deploy that uses the + * other. + */ + +import * as zlib from 'node:zlib'; +import { + decodeFormatPrefix, + encodeWithFormatPrefix, + peekFormatPrefix, +} from './format.js'; +import { SerializationFormat } from './types.js'; + +interface SyncCodec { + compress: (data: Uint8Array) => Uint8Array; + decompress: (data: Uint8Array) => Uint8Array; +} + +const gzipCodec: SyncCodec = { + compress: (d) => zlib.gzipSync(d), + decompress: (d) => zlib.gunzipSync(d), +}; + +/** + * Detect zstd availability at module init. `node:zlib` exposes + * `zstdCompressSync` / `zstdDecompressSync` starting in v22.15; + * older Node versions don't have these symbols, so guard with a + * typeof check rather than calling them and catching. + */ +const zstdCodec: SyncCodec | null = (() => { + // biome-ignore lint/suspicious/noExplicitAny: optional API surface + const z = zlib as any; + if (typeof z.zstdCompressSync !== 'function') return null; + if (typeof z.zstdDecompressSync !== 'function') return null; + return { + compress: (d) => z.zstdCompressSync(d) as Uint8Array, + decompress: (d) => z.zstdDecompressSync(d) as Uint8Array, + }; +})(); + +/** + * The codec that `compress()` will use for new payloads. Exposed so + * tests / diagnostics can confirm which codec is in effect. + * + * - `'zstd'` on Node >= 22.15 + * - `'gzip'` on older Node versions + */ +export const PREFERRED_CODEC: 'zstd' | 'gzip' = zstdCodec ? 'zstd' : 'gzip'; + +/** + * Compress a binary payload. Picks the best available codec + * (zstd if Node supports it, gzip otherwise) and wraps the result + * with the corresponding format prefix. + * + * Non-binary inputs are returned unchanged. Already-compressed + * inputs (recognized by their format prefix) are returned unchanged + * to make the helper idempotent. + */ +export function compress(data: Uint8Array | unknown): Uint8Array | unknown { + if (!(data instanceof Uint8Array)) return data; + + const existing = peekFormatPrefix(data); + if ( + existing === SerializationFormat.GZIP || + existing === SerializationFormat.ZSTD + ) { + return data; + } + + if (zstdCodec) { + const compressed = zstdCodec.compress(data); + return encodeWithFormatPrefix(SerializationFormat.ZSTD, compressed); + } + const compressed = gzipCodec.compress(data); + return encodeWithFormatPrefix(SerializationFormat.GZIP, compressed); +} + +/** + * Decompress a format-prefixed payload. Dispatches on the prefix: + * `gzip` → `gunzipSync`, `zstd` → `zstdDecompressSync`. Non-compressed + * inputs (no compression prefix) pass through unchanged so this layer + * composes cleanly with callers that may receive either wrapped or + * already-raw data. + * + * Throws if a `zstd`-prefixed blob is encountered on a Node version + * without zstd support — this can only happen if a deployment running + * a newer Node wrote a snapshot, and a deployment running an older + * Node tries to read it. The error message is explicit so operators + * can diagnose the version skew. + */ +export function decompress(data: Uint8Array | unknown): Uint8Array | unknown { + if (!(data instanceof Uint8Array)) return data; + + const prefix = peekFormatPrefix(data); + if (prefix === SerializationFormat.GZIP) { + const { payload } = decodeFormatPrefix(data); + return gzipCodec.decompress(payload); + } + if (prefix === SerializationFormat.ZSTD) { + if (!zstdCodec) { + throw new Error( + 'Encountered a zstd-compressed payload but zstd is not available on ' + + 'this Node runtime (requires Node 22.15+). This usually means a ' + + 'snapshot was written by a deployment running a newer Node version ' + + 'and is being read by an older one — upgrade the reading side.' + ); + } + const { payload } = decodeFormatPrefix(data); + return zstdCodec.decompress(payload); + } + return data; +} + +/** True when the payload carries a compression format prefix. */ +export function isCompressed(data: Uint8Array | unknown): boolean { + if (!(data instanceof Uint8Array)) return false; + const prefix = peekFormatPrefix(data); + return ( + prefix === SerializationFormat.GZIP || prefix === SerializationFormat.ZSTD + ); +} diff --git a/packages/core/src/serialization/reducers/common-vm.ts b/packages/core/src/serialization/reducers/common-vm.ts new file mode 100644 index 0000000000..2c53032fb1 --- /dev/null +++ b/packages/core/src/serialization/reducers/common-vm.ts @@ -0,0 +1,465 @@ +/** + * VM-compatible common reducers and revivers. + * + * Identical to common.ts but without Node.js dependencies: + * - Uses native `btoa` / `atob` (provided by quickjs-wasi's base64 + * extension, see `quickjs-assets.generated.ts`) instead of Buffer + * or pure-JS base64. + * - Uses `instanceof Error` instead of `types.isNativeError()`. + * + * This module is safe to bundle into the QuickJS WASM VM. + */ + +import type { Reducers, Revivers, SerializableSpecial } from '../types.js'; + +// ---- Base64 helpers (native btoa/atob from the quickjs-wasi base64 extension) ---- + +function arrayBufferToBase64( + value: ArrayBufferLike, + offset: number, + length: number +): string { + if (length === 0) return '.'; + // btoa requires a binary string. Build it from the byte view. + const uint8 = new Uint8Array(value, offset, length); + let binary = ''; + for (let i = 0; i < uint8.length; i++) { + binary += String.fromCharCode(uint8[i]!); + } + return btoa(binary); +} + +function viewToBase64(value: ArrayBufferView): string { + return arrayBufferToBase64(value.buffer, value.byteOffset, value.byteLength); +} + +function reviveArrayBuffer(value: string): ArrayBuffer { + if (value === '.') return new ArrayBuffer(0); + const binary = atob(value); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes.buffer as ArrayBuffer; +} + +// ---- Error subclass helper ---- + +// Creates a reducer for a built-in Error subclass whose serialized shape +// is exactly { message, stack, cause? }. Matches by `value.name` +// (instance property) for cross-realm + bundler-output robustness — see +// the host-side common.ts for full rationale. +function makeNamedErrorSubclassReducer(subclassName: string) { + return ( + value: unknown + ): { message: string; stack?: string; cause?: unknown } | false => { + if (!(value instanceof Error)) return false; + if (value.name !== subclassName) return false; + const reduced: { message: string; stack?: string; cause?: unknown } = { + message: value.message, + stack: value.stack, + }; + if ('cause' in value) reduced.cause = (value as { cause: unknown }).cause; + return reduced; + }; +} + +// Creates a reviver for a built-in Error subclass. Looks up the +// constructor on globalThis so the resulting object passes +// `instanceof TypeError` etc. in the consuming realm. Falls back to +// a base Error with the right `.name` if the constructor is not +// available (defensive — built-ins always exist). +function makeNamedErrorSubclassReviver(subclassName: string) { + return (value: { message: string; stack?: string; cause?: unknown }) => { + const Cls = (globalThis as any)[subclassName]; + let error: Error; + if (typeof Cls === 'function') { + error = new Cls(value.message); + } else { + error = new Error(value.message); + error.name = subclassName; + } + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }; +} + +// ---- Reducers ---- + +export function getCommonReducers(): Partial { + return { + ArrayBuffer: (value) => + value instanceof ArrayBuffer && + arrayBufferToBase64(value, 0, value.byteLength), + BigInt: (value) => typeof value === 'bigint' && value.toString(), + BigInt64Array: (value) => + value instanceof BigInt64Array && viewToBase64(value), + BigUint64Array: (value) => + value instanceof BigUint64Array && viewToBase64(value), + Date: (value) => { + if (!(value instanceof Date)) return false; + const valid = !Number.isNaN(value.getDate()); + return valid ? value.toISOString() : '.'; + }, + // DOMException is checked before Error so that DOMException-specific + // shape (name, message, stack, cause) survives the round-trip. + DOMException: (value) => { + if (!(value instanceof DOMException)) return false; + const reduced: SerializableSpecial['DOMException'] = { + message: value.message, + name: value.name, + stack: value.stack, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + // First-class Error subclass reducers. Order matters: each subclass + // reducer is checked before the generic `Error` catch-all so that + // e.g. a TypeError instance routes through the TypeError reducer + // instead of the base Error reducer. Matching is by `value.name` + // (the instance property) for cross-realm + bundler robustness; + // see common.ts for full rationale. + AggregateError: (value) => { + if (!(value instanceof Error) || value.name !== 'AggregateError') + return false; + const reduced: SerializableSpecial['AggregateError'] = { + message: value.message, + stack: value.stack, + errors: (value as AggregateError).errors, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + EvalError: makeNamedErrorSubclassReducer('EvalError'), + FatalError: makeNamedErrorSubclassReducer('FatalError'), + RangeError: makeNamedErrorSubclassReducer('RangeError'), + ReferenceError: makeNamedErrorSubclassReducer('ReferenceError'), + // RetryableError carries an extra retryAfter; serialize as numeric + // epoch timestamp for cross-realm safety (see host-side common.ts). + RetryableError: (value) => { + if (!(value instanceof Error) || value.name !== 'RetryableError') + return false; + const retryAfterRaw = (value as any).retryAfter; + let retryAfter: number; + if ( + retryAfterRaw && + typeof retryAfterRaw === 'object' && + typeof (retryAfterRaw as { getTime?: unknown }).getTime === 'function' + ) { + const t = (retryAfterRaw as Date).getTime(); + retryAfter = Number.isNaN(t) ? Date.now() + 1000 : t; + } else if ( + typeof retryAfterRaw === 'string' || + typeof retryAfterRaw === 'number' + ) { + const t = new Date(retryAfterRaw).getTime(); + retryAfter = Number.isNaN(t) ? Date.now() + 1000 : t; + } else { + retryAfter = Date.now() + 1000; + } + const reduced: SerializableSpecial['RetryableError'] = { + message: value.message, + stack: value.stack, + retryAfter, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + SyntaxError: makeNamedErrorSubclassReducer('SyntaxError'), + TypeError: makeNamedErrorSubclassReducer('TypeError'), + URIError: makeNamedErrorSubclassReducer('URIError'), + // Base Error reducer — catch-all. Matched LAST after subclass-specific + // reducers above. Preserves `name` so user Error subclasses without + // dedicated reducers retain their identity through the round-trip. + Error: (value) => { + // In the VM, use instanceof Error (no node:util available) + if (!(value instanceof Error)) return false; + const reduced: SerializableSpecial['Error'] = { + name: value.name, + message: value.message, + stack: value.stack, + }; + if ('cause' in value) reduced.cause = (value as any).cause; + return reduced; + }, + Float32Array: (value) => + value instanceof Float32Array && viewToBase64(value), + Float64Array: (value) => + value instanceof Float64Array && viewToBase64(value), + Int8Array: (value) => value instanceof Int8Array && viewToBase64(value), + Int16Array: (value) => value instanceof Int16Array && viewToBase64(value), + Int32Array: (value) => value instanceof Int32Array && viewToBase64(value), + Map: (value) => value instanceof Map && Array.from(value), + RegExp: (value) => + value instanceof RegExp && { + source: value.source, + flags: value.flags, + }, + // Request/Response/Headers — serialize using the polyfill constructors + Headers: (value) => { + const H = (globalThis as any).Headers; + if (!H || !(value instanceof H)) return false; + return Array.from(value as Iterable<[string, string]>); + }, + Request: (value) => { + const R = (globalThis as any).Request; + if (!R) return false; + // Use instanceof OR check for the Request-specific .json method + // (duck-typing on method/url alone would match plain objects and + // cause infinite recursion since the reducer output also has those) + if (!(value instanceof R) && typeof value?.json !== 'function') + return false; + if (typeof value?.method !== 'string') return false; + const data: any = { + method: value.method, + url: value.url, + headers: value.headers, + body: value.body, + duplex: value.duplex, + }; + // Include the webhook response writable stream if present + const responseWritable = value[Symbol.for('WEBHOOK_RESPONSE_WRITABLE')]; + if (responseWritable) { + data.responseWritable = responseWritable; + } + return data; + }, + Response: (value) => { + const R = (globalThis as any).Response; + if (!R) return false; + // Use instanceof OR check for Response-specific .clone method + if (!(value instanceof R) && typeof value?.clone !== 'function') + return false; + if (typeof value?.status !== 'number') return false; + return { + type: value.type, + url: value.url, + status: value.status, + statusText: value.statusText, + headers: value.headers, + body: value.body, + redirected: value.redirected, + }; + }, + ReadableStream: ((value: any) => { + if (value == null) return false; + const RS = (globalThis as any).ReadableStream; + if ( + !RS || + !(value instanceof RS || Object.getPrototypeOf(value) === RS.prototype) + ) + return false; + const bodyInit = value[Symbol.for('BODY_INIT')]; + if (bodyInit !== undefined) { + return { bodyInit }; + } + // Preserve stream name if present (opaque pointer for passing to steps) + const name = value[Symbol.for('WORKFLOW_STREAM_NAME')]; + if (name) { + const s: any = { name }; + const type = value[Symbol.for('WORKFLOW_STREAM_TYPE')]; + if (type) s.type = type; + return s; + } + return { name: '__empty' }; + }) as any, + WritableStream: ((value: any) => { + if (value == null) return false; + const WS = (globalThis as any).WritableStream; + if ( + !WS || + !(value instanceof WS || Object.getPrototypeOf(value) === WS.prototype) + ) + return false; + const name = value[Symbol.for('WORKFLOW_STREAM_NAME')]; + return { name: name || '__empty' }; + }) as any, + Set: (value) => value instanceof Set && Array.from(value), + URL: (value) => value instanceof URL && value.href, + WorkflowFunction: (value) => { + // Only match function references with a workflowId property (set by + // the SWC compiler on workflow functions). Plain { workflowId } objects + // are NOT matched — this prevents infinite recursion since the reduced + // form { workflowId } is a plain object, not a function. + if (typeof value !== 'function') return false; + const workflowId = (value as any).workflowId; + if (typeof workflowId !== 'string') return false; + return { workflowId }; + }, + URLSearchParams: (value) => { + if (!(value instanceof URLSearchParams)) return false; + return value.size === 0 ? '.' : String(value); + }, + Uint8Array: (value) => value instanceof Uint8Array && viewToBase64(value), + Uint8ClampedArray: (value) => + value instanceof Uint8ClampedArray && viewToBase64(value), + Uint16Array: (value) => value instanceof Uint16Array && viewToBase64(value), + Uint32Array: (value) => value instanceof Uint32Array && viewToBase64(value), + }; +} + +// ---- Revivers ---- + +export function getCommonRevivers(): Partial { + return { + ArrayBuffer: (value: string) => reviveArrayBuffer(value), + BigInt: (value: string) => BigInt(value), + BigInt64Array: (value: string) => + new BigInt64Array(reviveArrayBuffer(value)), + BigUint64Array: (value: string) => + new BigUint64Array(reviveArrayBuffer(value)), + Date: (value) => new Date(value), + DOMException: (value) => { + const error = new DOMException(value.message, value.name); + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = value.cause; + return error; + }, + AggregateError: (value) => { + const error = new AggregateError(value.errors ?? [], value.message); + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = value.cause; + return error; + }, + EvalError: makeNamedErrorSubclassReviver('EvalError'), + FatalError: (value) => { + // Prefer the host-registered FatalError class (registered via + // Symbol.for keys by @workflow/errors so `instanceof FatalError` + // works across realms). Fall back to a synthesized Error with + // the right .name when no registration is present. + // FatalError's constructor takes only `message`, so cause is + // attached as a property after construction (matching the host + // reviver in common.ts). + const Cls = (globalThis as any)[ + Symbol.for('@workflow/errors//FatalError') + ]; + let error: Error; + if (typeof Cls === 'function') { + error = new Cls(value.message); + } else { + error = new Error(value.message); + error.name = 'FatalError'; + } + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }, + RangeError: makeNamedErrorSubclassReviver('RangeError'), + ReferenceError: makeNamedErrorSubclassReviver('ReferenceError'), + RetryableError: (value) => { + // RetryableError's constructor accepts (message, { retryAfter }). + // Cause is attached after construction (the constructor does not + // forward it). retryAfter is stored as a Date in the VM realm. + const Cls = (globalThis as any)[ + Symbol.for('@workflow/errors//RetryableError') + ]; + const retryAfter = new Date(value.retryAfter); + let error: Error; + if (typeof Cls === 'function') { + error = new Cls(value.message, { retryAfter }); + } else { + error = new Error(value.message); + error.name = 'RetryableError'; + (error as any).retryAfter = retryAfter; + } + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }, + SyntaxError: makeNamedErrorSubclassReviver('SyntaxError'), + TypeError: makeNamedErrorSubclassReviver('TypeError'), + URIError: makeNamedErrorSubclassReviver('URIError'), + Error: (value) => { + const error = new Error(value.message); + error.name = value.name; + if (value.stack !== undefined) error.stack = value.stack; + if ('cause' in value) (error as any).cause = (value as any).cause; + return error; + }, + Float32Array: (value: string) => new Float32Array(reviveArrayBuffer(value)), + Float64Array: (value: string) => new Float64Array(reviveArrayBuffer(value)), + Int8Array: (value: string) => new Int8Array(reviveArrayBuffer(value)), + Int16Array: (value: string) => new Int16Array(reviveArrayBuffer(value)), + Int32Array: (value: string) => new Int32Array(reviveArrayBuffer(value)), + Map: (value) => new Map(value), + RegExp: (value) => new RegExp(value.source, value.flags), + Set: (value) => new Set(value), + URL: (value) => new URL(value), + WorkflowFunction: (value) => + Object.assign( + () => { + throw new Error( + 'Workflow functions cannot be called directly. Use start() to invoke them.' + ); + }, + { workflowId: value.workflowId } + ), + URLSearchParams: (value) => new URLSearchParams(value === '.' ? '' : value), + Uint8Array: (value: string) => new Uint8Array(reviveArrayBuffer(value)), + Uint8ClampedArray: (value: string) => + new Uint8ClampedArray(reviveArrayBuffer(value)), + Uint16Array: (value: string) => new Uint16Array(reviveArrayBuffer(value)), + Uint32Array: (value: string) => new Uint32Array(reviveArrayBuffer(value)), + // Web API types — revived as plain objects in the VM since the real + // constructors (Headers, Request, Response) are not available in QuickJS. + // The workflow code can access the properties but not call Web API methods. + Headers: (value) => { + return new (globalThis as any).Headers(value); + }, + Request: (value: any) => { + const Req = (globalThis as any).Request; + if (Req) { + value.json = Req.prototype.json; + value.text = Req.prototype.text; + value.arrayBuffer = Req.prototype.arrayBuffer; + } + // Carry over the webhook response writable stream to the symbol property + if (value.responseWritable) { + value[Symbol.for('WEBHOOK_RESPONSE_WRITABLE')] = value.responseWritable; + } + return value; + }, + Response: (value: any) => { + // Don't use Object.setPrototypeOf — devalue continues to set properties + // on the object after the reviver runs, and getter-only properties + // (like 'ok') on the prototype would cause "no setter" errors. + // Instead, copy methods directly onto the object. + const Resp = (globalThis as any).Response; + if (Resp) { + value.json = Resp.prototype.json; + value.text = Resp.prototype.text; + value.arrayBuffer = Resp.prototype.arrayBuffer; + if (Resp.prototype.bytes) value.bytes = Resp.prototype.bytes; + if (Resp.prototype.clone) value.clone = Resp.prototype.clone; + } + value._body = value.body; + value.ok = value.status >= 200 && value.status < 300; + value.bodyUsed = false; + return value; + }, + ReadableStream: (value) => { + const RS = (globalThis as any).ReadableStream; + const stream = Object.create(RS ? RS.prototype : {}); + if (value && 'bodyInit' in value) { + // Body from Response/Request constructor — store the raw data + stream[Symbol.for('BODY_INIT')] = value.bodyInit; + } else if (value && 'name' in value) { + // Named stream reference — preserve the name/type for re-serialization. + // Streams are opaque pointers in the VM — they can be passed to steps + // but not consumed directly. + stream[Symbol.for('WORKFLOW_STREAM_NAME')] = value.name; + if (value.type) stream[Symbol.for('WORKFLOW_STREAM_TYPE')] = value.type; + } + return stream; + }, + WritableStream: (value) => { + const WS = (globalThis as any).WritableStream; + const stream = Object.create(WS ? WS.prototype : {}); + if (value && 'name' in value) { + stream[Symbol.for('WORKFLOW_STREAM_NAME')] = value.name; + } + return stream; + }, + }; +} diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 4c14f82bff..41c03f7c3f 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -30,6 +30,10 @@ export const SerializationFormat = { DEVALUE_V1: 'devl' as FormatPrefix, /** Encrypted payload (inner payload has its own format prefix) */ ENCRYPTED: 'encr' as FormatPrefix, + /** gzip-compressed payload (`zlib.gzipSync`); inner is raw bytes */ + GZIP: 'gzip' as FormatPrefix, + /** zstd-compressed payload (`zlib.zstdCompressSync`, Node >= 22.15); inner is raw bytes */ + ZSTD: 'zstd' as FormatPrefix, } as const; // ---- Serializable Types ---- diff --git a/packages/core/src/serialization/vm-bundle-entry.ts b/packages/core/src/serialization/vm-bundle-entry.ts new file mode 100644 index 0000000000..8bdc795900 --- /dev/null +++ b/packages/core/src/serialization/vm-bundle-entry.ts @@ -0,0 +1,57 @@ +/** + * Entry point for the VM serialization bundle. + * + * This file is bundled by esbuild into a self-contained IIFE that + * sets up serialize/deserialize on globalThis. The bundled output + * is evaluated inside the QuickJS VM during bootstrap. + * + * TextEncoder, TextDecoder, and Headers are provided by native C + * extensions in quickjs-wasi, so no polyfills are needed. + */ + +import { monotonicFactory } from 'ulid'; +import { deserialize, serialize } from './workflow-vm.js'; + +// Install on global scope under the public well-known symbols. The +// snapshot runtime's bootstrap (and the various inline-evaluated JS +// strings in `snapshot-runtime.ts`) reach the same functions via +// `globalThis[Symbol.for('workflow-serialize')]` etc. +(globalThis as any)[Symbol.for('workflow-serialize')] = serialize; +(globalThis as any)[Symbol.for('workflow-deserialize')] = deserialize; + +// ULID generator for correlationIds — uses the same monotonicFactory +// as the event-replay runtime. Both inputs MUST be set by the host +// before this bundle is evaluated, otherwise the seeded-ULID +// determinism guarantee is silently broken: +// +// * `Math.random` must already be replaced with the host's seeded +// PRNG via `vm.newFunction('random', …)` (see +// `snapshot-runtime.ts`, the `Seeded Math.random` block). Two +// workflow invocations of the same resumption MUST observe an +// identical random sequence so their correlationIds collide and +// the world's EntityConflictError dedup applies. We pass it +// explicitly to `monotonicFactory` because ULID's auto-detect +// (`detectPRNG`) only knows about `crypto.getRandomValues` / +// `crypto.randomBytes`, neither of which exist in QuickJS. +// * `globalThis.__ulidTimestamp` must be a number (typically +// `workflowRun.startedAt`). It's used in place of `Date.now()` so +// the time portion of the ULID is also stable across concurrent +// invocations of the same resumption. +// +// Both prerequisites are validated below — fail loudly if either is +// missing rather than fall back to `Date.now()` / unseeded +// `Math.random`, which would re-introduce non-determinism that the +// snapshot runtime relies on us NOT having. +const ulid = monotonicFactory(Math.random); +(globalThis as any).__generateUlid = () => { + const t = (globalThis as any).__ulidTimestamp; + if (typeof t !== 'number') { + throw new Error( + '__generateUlid: globalThis.__ulidTimestamp must be a number set by ' + + 'the host before the serde bundle is evaluated. Without it, ULIDs ' + + 'would fall back to Date.now() and concurrent workflow invocations ' + + 'of the same resumption would produce divergent correlationIds.' + ); + } + return ulid(t); +}; diff --git a/packages/core/src/serialization/workflow-vm.test.ts b/packages/core/src/serialization/workflow-vm.test.ts new file mode 100644 index 0000000000..37faba45bd --- /dev/null +++ b/packages/core/src/serialization/workflow-vm.test.ts @@ -0,0 +1,186 @@ +/** + * Tests for the VM-compatible workflow serializer. + * + * Verifies that: + * 1. The VM serializer produces the same wire format as the Node.js serializer. + * 2. Data serialized by the VM can be deserialized by Node.js and vice versa. + */ + +import { describe, expect, it } from 'vitest'; +import { peekFormatPrefix } from './format.js'; +import { + deserialize as nodeDeserialize, + serialize as nodeSerialize, +} from './workflow.js'; +import { + deserialize as vmDeserialize, + serialize as vmSerialize, +} from './workflow-vm.js'; + +describe('VM workflow serializer', () => { + it('should produce format-prefixed output', () => { + const serialized = vmSerialize(42); + expect(serialized).toBeInstanceOf(Uint8Array); + expect(peekFormatPrefix(serialized)).toBe('devl'); + }); + + it('should round-trip primitives', () => { + for (const val of [42, 'hello', true, null, undefined]) { + expect(vmDeserialize(vmSerialize(val))).toEqual(val); + } + }); + + it('should round-trip Date', () => { + const date = new Date('2025-01-01T00:00:00Z'); + const result = vmDeserialize(vmSerialize(date)) as Date; + expect(result).toBeInstanceOf(Date); + expect(result.toISOString()).toBe('2025-01-01T00:00:00.000Z'); + }); + + it('should round-trip Map', () => { + const map = new Map([ + ['a', 1], + ['b', 2], + ]); + const result = vmDeserialize(vmSerialize(map)) as Map; + expect(result).toBeInstanceOf(Map); + expect(result.get('a')).toBe(1); + }); + + it('should round-trip Uint8Array', () => { + const u8 = new Uint8Array([1, 2, 3, 4, 5]); + const result = vmDeserialize(vmSerialize(u8)) as Uint8Array; + expect(result).toBeInstanceOf(Uint8Array); + expect(Array.from(result)).toEqual([1, 2, 3, 4, 5]); + }); + + it('should round-trip nested objects', () => { + const val = { a: 1, b: [2, new Date('2025-01-01')], c: { d: 'e' } }; + const result = vmDeserialize(vmSerialize(val)) as any; + expect(result.a).toBe(1); + expect(result.b[0]).toBe(2); + expect(result.b[1]).toBeInstanceOf(Date); + expect(result.c.d).toBe('e'); + }); + + it('should round-trip WorkflowFunction reference', () => { + // Simulate an SWC-compiled workflow function: a function with a + // `workflowId` property that the runtime treats as an opaque handle. + const fn = Object.assign(() => {}, { + workflowId: 'workflow//./src/foo//myWorkflow', + }); + const revived = vmDeserialize(vmSerialize(fn)) as any; + expect(typeof revived).toBe('function'); + expect(revived.workflowId).toBe('workflow//./src/foo//myWorkflow'); + // Calling the revived stub throws — workflow functions must be invoked + // via start(), not directly. + expect(() => revived()).toThrow(/Use start\(\)/); + }); + + it('should round-trip DOMException', () => { + const ex = new DOMException('boom', 'AbortError'); + const revived = vmDeserialize(vmSerialize(ex)) as Error; + // The revived value is a DOMException (or Error fallback with the same + // name) — either way it should preserve name/message and be instanceof Error. + expect(revived).toBeInstanceOf(Error); + expect(revived.name).toBe('AbortError'); + expect(revived.message).toBe('boom'); + }); +}); + +describe('VM ↔ Node.js cross-compatibility', () => { + it('VM serialize → Node.js deserialize', () => { + const values = [ + 42, + 'hello', + new Date('2025-06-15'), + new Map([['x', 1]]), + new Set([1, 2, 3]), + new Uint8Array([10, 20, 30]), + { nested: { arr: [1, 2, 3] } }, + ]; + for (const val of values) { + const vmBytes = vmSerialize(val); + const nodeResult = nodeDeserialize(vmBytes); + const vmResult = vmDeserialize(vmBytes); + // Both should produce equivalent values + expect(JSON.stringify(nodeResult)).toBe(JSON.stringify(vmResult)); + } + }); + + it('Node.js serialize → VM deserialize', () => { + const values = [ + 42, + 'hello', + new Date('2025-06-15'), + new Map([['x', 1]]), + new Set([1, 2, 3]), + new Uint8Array([10, 20, 30]), + { nested: { arr: [1, 2, 3] } }, + ]; + for (const val of values) { + const nodeBytes = nodeSerialize(val); + const vmResult = vmDeserialize(nodeBytes); + const nodeResult = nodeDeserialize(nodeBytes); + expect(JSON.stringify(vmResult)).toBe(JSON.stringify(nodeResult)); + } + }); + + it('step args format: VM serialize → Node.js hydrateStepArguments', async () => { + // This is the critical path: VM serializes step args, step handler deserializes + const { hydrateStepArguments } = await import('../serialization.js'); + + const stepInput = { args: [10, 7], closureVars: { x: 42 } }; + const vmBytes = vmSerialize(stepInput); + + const hydrated = (await hydrateStepArguments( + vmBytes, + 'run-123', + undefined + )) as any; + expect(hydrated.args).toEqual([10, 7]); + expect(hydrated.closureVars).toEqual({ x: 42 }); + }); + + it('Node.js serialize TypeError → VM deserialize keeps subclass identity + cause', () => { + const cause = new TypeError('underlying'); + const wrapped = new Error('outer'); + (wrapped as any).cause = cause; + const nodeBytes = nodeSerialize(wrapped); + const result = vmDeserialize(nodeBytes) as Error; + expect(result).toBeInstanceOf(Error); + expect(result.message).toBe('outer'); + expect((result as any).cause).toBeInstanceOf(TypeError); + expect(((result as any).cause as Error).message).toBe('underlying'); + }); + + it('Node.js serialize built-in subclasses → VM deserialize preserves type identity', () => { + const cases: Array<[Error, new (...args: any[]) => Error]> = [ + [new TypeError('t'), TypeError], + [new RangeError('r'), RangeError], + [new SyntaxError('s'), SyntaxError], + [new ReferenceError('rf'), ReferenceError], + ]; + for (const [err, ctor] of cases) { + const result = vmDeserialize(nodeSerialize(err)) as Error; + expect(result).toBeInstanceOf(ctor); + expect(result.message).toBe(err.message); + } + }); + + it('step result format: Node.js dehydrateStepReturnValue → VM deserialize', async () => { + // This is the other critical path: step handler serializes result, VM deserializes + const { dehydrateStepReturnValue } = await import('../serialization.js'); + + const result = { sum: 17, computed: true }; + const nodeBytes = await dehydrateStepReturnValue( + result, + 'run-123', + undefined, + [] + ); + const vmResult = vmDeserialize(nodeBytes) as any; + expect(vmResult.sum).toBe(17); + expect(vmResult.computed).toBe(true); + }); +}); diff --git a/packages/core/src/serialization/workflow-vm.ts b/packages/core/src/serialization/workflow-vm.ts new file mode 100644 index 0000000000..61fc9d14a4 --- /dev/null +++ b/packages/core/src/serialization/workflow-vm.ts @@ -0,0 +1,73 @@ +/** + * VM-compatible workflow mode serialization. + * + * This module is designed to be bundled into the QuickJS WASM VM. + * It has NO Node.js dependencies (no Buffer, no node:util). + * + * Produces and consumes the same wire format as the Node.js workflow.ts — + * format-prefixed devalue data ("devl" + devalue.stringify output). + */ + +import { devalueVmCodec } from './codec-devalue-vm.js'; +import { SerializationFormat, isFormatPrefix } from './types.js'; + +const FORMAT_PREFIX_LENGTH = 4; +let _encoder: { encode(s: string): Uint8Array }; +let _decoder: { decode(d: Uint8Array): string }; +function getEncoder() { + if (!_encoder) _encoder = new (globalThis as any).TextEncoder(); + return _encoder; +} +function getDecoder() { + if (!_decoder) _decoder = new (globalThis as any).TextDecoder(); + return _decoder; +} + +/** + * Serialize a value to format-prefixed bytes. + * + * @param value - The value to serialize + * @returns Uint8Array with "devl" prefix + devalue payload + */ +export function serialize(value: unknown): Uint8Array { + const payload = devalueVmCodec.serialize(value, 'workflow'); + const prefix = getEncoder().encode(SerializationFormat.DEVALUE_V1); + const result = new Uint8Array(prefix.length + payload.length); + result.set(prefix, 0); + result.set(payload, prefix.length); + return result; +} + +/** + * Deserialize format-prefixed bytes back to a value. + * + * @param data - Uint8Array with format prefix, or legacy non-binary data + * @returns The deserialized value + */ +export function deserialize(data: Uint8Array | unknown): unknown { + // Legacy: non-binary data + if (!(data instanceof Uint8Array)) { + if (devalueVmCodec.deserializeLegacy) { + return devalueVmCodec.deserializeLegacy(data, 'workflow'); + } + throw new Error( + 'Cannot deserialize non-binary data without legacy support' + ); + } + + if (data.length < FORMAT_PREFIX_LENGTH) { + throw new Error('Data too short to contain format prefix'); + } + + const prefixStr = getDecoder().decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); + if (!isFormatPrefix(prefixStr)) { + throw new Error(`Invalid format prefix: "${prefixStr}"`); + } + + if (prefixStr === SerializationFormat.DEVALUE_V1) { + const payload = data.subarray(FORMAT_PREFIX_LENGTH); + return devalueVmCodec.deserialize(payload, 'workflow'); + } + + throw new Error(`Unsupported serialization format: ${prefixStr}`); +} diff --git a/packages/core/src/source-map.test.ts b/packages/core/src/source-map.test.ts new file mode 100644 index 0000000000..337e7a4079 --- /dev/null +++ b/packages/core/src/source-map.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from 'vitest'; +import { stripInlineSourceMap } from './source-map.js'; + +describe('stripInlineSourceMap', () => { + it('returns the input unchanged when there is no inline map', () => { + const code = 'const x = 1;\nconsole.log(x);\n'; + expect(stripInlineSourceMap(code)).toBe(code); + }); + + it('strips a trailing inline source map comment', () => { + const code = + 'var workflow = { name: "test" };\nconst result = workflow.name;\n' + + '//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozfQ==\n'; + const stripped = stripInlineSourceMap(code); + expect(stripped).not.toMatch(/sourceMappingURL/); + expect(stripped).toContain('var workflow'); + expect(stripped).toContain('workflow.name'); + }); + + it('strips a long source map comment without trailing newline', () => { + // Many bundlers emit the comment as the very last line with no + // trailing newline. The regex must match end-of-input too. + const longBase64 = 'A'.repeat(4 * 1024 * 1024); // 4 MB of payload + const code = `globalThis.x = 1;\n//# sourceMappingURL=data:application/json;base64,${longBase64}`; + const stripped = stripInlineSourceMap(code); + expect(stripped).not.toMatch(/sourceMappingURL/); + expect(stripped.length).toBeLessThan(code.length); + // The bundle proper is preserved — only the trailing comment is gone. + expect(stripped).toContain('globalThis.x = 1;'); + }); + + it('only strips the trailing inline map (not embedded substrings)', () => { + // A workflow could legitimately contain the literal string + // "sourceMappingURL" inside JS code (e.g. inside a string literal + // for an unrelated reason). The regex anchors to end-of-line/end + // and only matches the comment form, so non-comment occurrences + // are preserved. + const code = ` +const literal = "sourceMappingURL=foo"; +console.log(literal); +//# sourceMappingURL=data:application/json;base64,Zm9v +`; + const stripped = stripInlineSourceMap(code); + expect(stripped).toContain(`"sourceMappingURL=foo"`); + expect(stripped).not.toMatch(/\/\/# sourceMappingURL/); + }); +}); diff --git a/packages/core/src/source-map.ts b/packages/core/src/source-map.ts index d121421f6b..94dae54dfa 100644 --- a/packages/core/src/source-map.ts +++ b/packages/core/src/source-map.ts @@ -1,5 +1,31 @@ import { originalPositionFor, TraceMap } from '@jridgewell/trace-mapping'; +/** + * Pattern matching the trailing inline source map comment that bundlers + * (esbuild, etc.) emit. The comment is purely host-side metadata for + * `remapErrorStack` — the VM never needs it. Stripping it before + * passing the bundle to `vm.evalCode` materially reduces the QuickJS + * heap (and therefore snapshot bytes), because QuickJS retains source + * text for stack-trace line lookups. + */ +const INLINE_SOURCE_MAP_COMMENT_RE = + /\/\/# sourceMappingURL=data:application\/json;base64,[A-Za-z0-9+/=]+\s*$/m; + +/** + * Strip the trailing `//# sourceMappingURL=data:…` comment from a JS + * bundle. Returns the input unchanged if no inline map is present. + * + * Use this on the host side before evaluating workflow bundles inside + * the QuickJS VM — the inline map can account for ~30%+ of the + * resulting snapshot bytes (measured 11.9 MB → 8.0 MB on the example + * workbench's bundle), and the VM never needs it; only host-side + * `remapErrorStack` reads the map (and it can do so against the + * original, unstripped string). + */ +export function stripInlineSourceMap(workflowCode: string): string { + return workflowCode.replace(INLINE_SOURCE_MAP_COMMENT_RE, ''); +} + /** * Remaps an error stack trace using inline source maps to show original source locations. * diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index 58cfd9cda0..25a9627462 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -92,6 +92,101 @@ export const WorkflowTracePropagated = SemanticConvention( 'workflow.trace.propagated' ); +// Snapshot runtime attributes + +/** The runtime mode handling this invocation */ +export const SnapshotRuntime = SemanticConvention<'snapshot' | 'replay'>( + 'snapshot.runtime' +); + +/** + * Whether this VM invocation is the first run (no existing snapshot) or a + * restore from a previously-persisted snapshot. + */ +export const SnapshotInvocationKind = SemanticConvention<'first' | 'restore'>( + 'snapshot.invocation_kind' +); + +/** Stored snapshot size on load, including any encryption framing */ +export const SnapshotLoadBytes = SemanticConvention( + 'snapshot.load.bytes' +); + +/** Time spent in `world.snapshots.load()` (ms) */ +export const SnapshotLoadDurationMs = SemanticConvention( + 'snapshot.load.duration_ms' +); + +/** Time spent decrypting the snapshot payload (ms) */ +export const SnapshotDecryptDurationMs = SemanticConvention( + 'snapshot.decrypt.duration_ms' +); + +/** Time spent in QuickJS.deserializeSnapshot() (ms) */ +export const SnapshotDeserializeDurationMs = SemanticConvention( + 'snapshot.deserialize.duration_ms' +); + +/** Whether preloaded events from `events.create('run_started')` were used */ +export const SnapshotEventsPreloaded = SemanticConvention( + 'snapshot.events.preloaded' +); + +/** Total number of events fetched from the world for this invocation */ +export const SnapshotEventsFetchedCount = SemanticConvention( + 'snapshot.events.fetched_count' +); + +/** Number of pages required to fetch all events */ +export const SnapshotEventsFetchedPages = SemanticConvention( + 'snapshot.events.fetched_pages' +); + +/** Number of pending VM operations captured at suspension */ +export const SnapshotPendingOpsCount = SemanticConvention( + 'snapshot.pending_ops_count' +); + +/** Stored snapshot size on save, post-encryption (the bytes the world sees) */ +export const SnapshotSaveBytes = SemanticConvention( + 'snapshot.save.bytes' +); + +/** Snapshot size before encryption (raw QuickJS serializeSnapshot output) */ +export const SnapshotSavePlaintextBytes = SemanticConvention( + 'snapshot.save.plaintext_bytes' +); + +/** Time spent in QuickJS.serializeSnapshot() (ms) */ +export const SnapshotSerializeDurationMs = SemanticConvention( + 'snapshot.serialize.duration_ms' +); + +/** Time spent encrypting the snapshot payload (ms) */ +export const SnapshotEncryptDurationMs = SemanticConvention( + 'snapshot.encrypt.duration_ms' +); + +/** Time spent in `world.snapshots.save()` (ms) */ +export const SnapshotSaveDurationMs = SemanticConvention( + 'snapshot.save.duration_ms' +); + +/** Time spent in `world.snapshots.delete()` (ms) */ +export const SnapshotDeleteDurationMs = SemanticConvention( + 'snapshot.delete.duration_ms' +); + +/** Outcome of this snapshot-runtime VM invocation */ +export const SnapshotOutcome = SemanticConvention< + 'completed' | 'suspended' | 'failed' +>('snapshot.outcome'); + +/** Events cursor written into the saved snapshot's metadata */ +export const SnapshotEventsCursor = SemanticConvention( + 'snapshot.events_cursor' +); + /** Name of the error that caused workflow failure */ export const WorkflowErrorName = SemanticConvention( 'workflow.error.name' diff --git a/packages/core/turbo.json b/packages/core/turbo.json index e503fb6757..aa04cd0e81 100644 --- a/packages/core/turbo.json +++ b/packages/core/turbo.json @@ -3,7 +3,12 @@ "tasks": { "build": { "dependsOn": ["^build"], - "outputs": ["dist", "src/version.ts"] + "outputs": [ + "dist", + "src/version.ts", + "src/runtime/vm-serde-bundle.generated.ts", + "src/runtime/quickjs-assets.generated.ts" + ] } } } diff --git a/packages/world-local/src/queue.ts b/packages/world-local/src/queue.ts index 5a7bf6d7f7..bab53552ac 100644 --- a/packages/world-local/src/queue.ts +++ b/packages/world-local/src/queue.ts @@ -299,6 +299,10 @@ export function createQueue(config: Partial): LocalQueue { return Response.json({ ok: true }); } catch (error) { + console.error( + '[local world] Queue handler error:', + error instanceof Error ? error.stack : String(error) + ); return Response.json(String(error), { status: 500 }); } }; diff --git a/packages/world-local/src/storage/index.ts b/packages/world-local/src/storage/index.ts index 2bdcd85b3e..7d53dbaa47 100644 --- a/packages/world-local/src/storage/index.ts +++ b/packages/world-local/src/storage/index.ts @@ -3,6 +3,7 @@ import { instrumentObject } from '../instrumentObject.js'; import { createEventsStorage } from './events-storage.js'; import { createHooksStorage } from './hooks-storage.js'; import { createRunsStorage, type LocalRunsStorage } from './runs-storage.js'; +import { createSnapshotsStorage } from './snapshots-storage.js'; import { createStepsStorage } from './steps-storage.js'; /** @@ -27,6 +28,7 @@ export function createStorage(basedir: string, tag?: string): LocalStorage { const steps = createStepsStorage(basedir, tag); const events = createEventsStorage(basedir, tag); const hooks = createHooksStorage(basedir, tag); + const snapshots = createSnapshotsStorage(basedir); // Instrument all storage methods with tracing // NOTE: Span names are lowercase per OTEL semantic conventions @@ -35,5 +37,6 @@ export function createStorage(basedir: string, tag?: string): LocalStorage { steps: instrumentObject('world.steps', steps), events: instrumentObject('world.events', events), hooks: instrumentObject('world.hooks', hooks), + snapshots: instrumentObject('world.snapshots', snapshots), }; } diff --git a/packages/world-local/src/storage/snapshots-storage.ts b/packages/world-local/src/storage/snapshots-storage.ts new file mode 100644 index 0000000000..0e4d2e6f33 --- /dev/null +++ b/packages/world-local/src/storage/snapshots-storage.ts @@ -0,0 +1,73 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { SnapshotMetadata } from '@workflow/world'; +import { SnapshotMetadataSchema } from '@workflow/world'; +import { ensureDir, readBuffer, readJSON, write, writeJSON } from '../fs.js'; + +/** + * Create the snapshots sub-storage for a local World implementation. + * + * Snapshots are stored as two files per run: + * {basedir}/snapshots/{runId}.bin — opaque VM snapshot bytes + * {basedir}/snapshots/{runId}.json — metadata (eventsCursor, createdAt) + * + * Compression and encryption are handled by `@workflow/core`'s snapshot + * entrypoint (`compress → encrypt → save`); this world layer stores the + * resulting bytes verbatim. + */ +export function createSnapshotsStorage(basedir: string) { + const snapshotsDir = path.join(basedir, 'snapshots'); + + function dataPath(runId: string): string { + return path.join(snapshotsDir, `${runId}.bin`); + } + function metadataPath(runId: string): string { + return path.join(snapshotsDir, `${runId}.json`); + } + + return { + async save( + runId: string, + data: Uint8Array, + metadata: SnapshotMetadata + ): Promise { + await ensureDir(snapshotsDir); + await Promise.all([ + write(dataPath(runId), Buffer.from(data), { overwrite: true }), + writeJSON(metadataPath(runId), metadata, { overwrite: true }), + ]); + }, + + async load( + runId: string + ): Promise<{ data: Uint8Array; metadata: SnapshotMetadata } | null> { + const metadata = await readJSON( + metadataPath(runId), + SnapshotMetadataSchema + ); + if (!metadata) return null; + + try { + const dataBuf = await readBuffer(dataPath(runId)); + const data = new Uint8Array( + dataBuf.buffer, + dataBuf.byteOffset, + dataBuf.byteLength + ); + return { data, metadata }; + } catch (error: any) { + if (error.code === 'ENOENT') { + return null; + } + throw error; + } + }, + + async delete(runId: string): Promise { + await Promise.all([ + fs.rm(dataPath(runId), { force: true }), + fs.rm(metadataPath(runId), { force: true }), + ]); + }, + }; +} diff --git a/packages/world-postgres/src/drizzle/migrations/0012_add_snapshots_table.sql b/packages/world-postgres/src/drizzle/migrations/0012_add_snapshots_table.sql new file mode 100644 index 0000000000..83c8b55257 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0012_add_snapshots_table.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS "workflow"."workflow_snapshots" ( + "run_id" varchar PRIMARY KEY NOT NULL, + "data" "bytea" NOT NULL, + "events_cursor" varchar, + "created_at" timestamp DEFAULT now() NOT NULL +); diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index 5e99b153c2..2a7b1c4460 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -85,6 +85,13 @@ "when": 1771500000000, "tag": "0011_add_error_code", "breakpoints": true + }, + { + "idx": 12, + "version": "7", + "when": 1772000000000, + "tag": "0012_add_snapshots_table", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 3176f4359c..93e4f1d3ff 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -230,6 +230,23 @@ const bytea = customType<{ data: Buffer; notNull: false; default: false }>({ }, }); +/** + * VM snapshots for the snapshot runtime. + * + * Each row is a 1-to-1 mapping with a workflow run — a snapshot captures + * the QuickJS VM state at a suspension point so execution can resume from + * there without replaying the full event log. + * + * The binary data is stored gzip-compressed in the `data` column. + * Metadata (`eventsCursor`, `createdAt`) lives alongside for cheap loads. + */ +export const snapshots = schema.table('workflow_snapshots', { + runId: varchar('run_id').primaryKey(), + data: bytea('data').notNull(), + eventsCursor: varchar('events_cursor'), + createdAt: timestamp('created_at').defaultNow().notNull(), +}); + export const streams = schema.table( 'workflow_stream_chunks', { diff --git a/packages/world-postgres/src/index.ts b/packages/world-postgres/src/index.ts index 9ad7565e06..f32547df02 100644 --- a/packages/world-postgres/src/index.ts +++ b/packages/world-postgres/src/index.ts @@ -4,6 +4,7 @@ import { Pool } from 'pg'; import type { PostgresWorldConfig } from './config.js'; import { createClient, type Drizzle } from './drizzle/index.js'; import { createQueue } from './queue.js'; +import { createSnapshotsStorage } from './snapshots.js'; import { createEventsStorage, createHooksStorage, @@ -18,6 +19,7 @@ function createStorage(drizzle: Drizzle): Storage { events: createEventsStorage(drizzle), hooks: createHooksStorage(drizzle), steps: createStepsStorage(drizzle), + snapshots: createSnapshotsStorage(drizzle), }; } diff --git a/packages/world-postgres/src/snapshots.ts b/packages/world-postgres/src/snapshots.ts new file mode 100644 index 0000000000..f11ea7ef64 --- /dev/null +++ b/packages/world-postgres/src/snapshots.ts @@ -0,0 +1,73 @@ +import type { SnapshotMetadata, Storage } from '@workflow/world'; +import { eq } from 'drizzle-orm'; +import { type Drizzle, Schema } from './drizzle/index.js'; + +/** + * Snapshot storage for world-postgres. + * + * Compression and encryption are handled by `@workflow/core`'s + * snapshot entrypoint (`compress(snapshot) → encrypt → save`). This + * world layer treats the bytes as opaque — it does NOT add its own + * compression. Blobs are stored verbatim in the `data` column of + * `workflow.workflow_snapshots`. Each run has at most one row; + * `save()` upserts the latest suspension's bytes. + */ +export function createSnapshotsStorage(drizzle: Drizzle): Storage['snapshots'] { + const { snapshots } = Schema; + + return { + async save( + runId: string, + data: Uint8Array, + metadata: SnapshotMetadata + ): Promise { + const blob = Buffer.from(data); + await drizzle + .insert(snapshots) + .values({ + runId, + data: blob, + eventsCursor: metadata.eventsCursor, + createdAt: metadata.createdAt, + }) + .onConflictDoUpdate({ + target: snapshots.runId, + set: { + data: blob, + eventsCursor: metadata.eventsCursor, + createdAt: metadata.createdAt, + }, + }); + }, + + async load( + runId: string + ): Promise<{ data: Uint8Array; metadata: SnapshotMetadata } | null> { + const [row] = await drizzle + .select() + .from(snapshots) + .where(eq(snapshots.runId, runId)) + .limit(1); + + if (!row) return null; + + const data = new Uint8Array( + row.data.buffer, + row.data.byteOffset, + row.data.byteLength + ); + + return { + data, + metadata: { + eventsCursor: row.eventsCursor, + createdAt: row.createdAt, + }, + }; + }, + + async delete(runId: string): Promise { + await drizzle.delete(snapshots).where(eq(snapshots.runId, runId)); + }, + }; +} diff --git a/packages/world-testing/src/inline-batches-debug.mts b/packages/world-testing/src/inline-batches-debug.mts index bf7a356cb3..246aecd782 100644 --- a/packages/world-testing/src/inline-batches-debug.mts +++ b/packages/world-testing/src/inline-batches-debug.mts @@ -1,5 +1,5 @@ -import { expect, test, vi } from 'vitest'; import { hydrateWorkflowReturnValue } from '@workflow/core/serialization'; +import { expect, test, vi } from 'vitest'; import { createFetcher, startServer } from './util.mjs'; /** @@ -23,6 +23,9 @@ export function inlineBatchesDebug(world: string) { const server = await startServer({ world, env: { + // Pin to replay — this debug helper measures V2 inline-execution + // batching behavior, which is replay-runtime-specific. + WORKFLOW_RUNTIME: 'replay', DEBUG: 'workflow:runtime:*', }, }); diff --git a/packages/world-testing/src/inline-execution.mts b/packages/world-testing/src/inline-execution.mts index 34deb70c50..3436902f46 100644 --- a/packages/world-testing/src/inline-execution.mts +++ b/packages/world-testing/src/inline-execution.mts @@ -1,5 +1,5 @@ -import { expect, test, vi } from 'vitest'; import { hydrateWorkflowReturnValue } from '@workflow/core/serialization'; +import { expect, test, vi } from 'vitest'; import { createFetcher, startServer } from './util.mjs'; /** @@ -12,13 +12,20 @@ import { createFetcher, startServer } from './util.mjs'; * - Parallel steps (Promise.all): 1-3 invocations depending on whether the * embedded harness observes the background step and continuation separately * - Hook + resume: 2 invocations (hook requires external resume) + * + * These tests pin to the event-replay runtime — invocation-count assertions + * are V2-replay-specific. Snapshot runtime makes a separate flow invocation + * per resume point, so the same workflow produces a different (larger) + * invocation count under snapshot mode. */ +const INLINE_EXEC_ENV = { WORKFLOW_RUNTIME: 'replay' }; + export function inlineExecution(world: string) { test( 'sequential steps complete in a single flow invocation', { timeout: 30_000 }, async () => { - const server = await startServer({ world }).then(createFetcher); + const server = await startServer({ world, env: INLINE_EXEC_ENV }).then(createFetcher); const result = await server.invoke( 'workflows/inline-execution.ts', 'sequentialStepsWorkflow', @@ -50,7 +57,7 @@ export function inlineExecution(world: string) { 'sequential steps with stream complete in a single flow invocation', { timeout: 30_000 }, async () => { - const server = await startServer({ world }).then(createFetcher); + const server = await startServer({ world, env: INLINE_EXEC_ENV }).then(createFetcher); const result = await server.invoke( 'workflows/inline-execution.ts', 'sequentialStepsWithStreamWorkflow', @@ -82,7 +89,7 @@ export function inlineExecution(world: string) { 'sleep workflow requires exactly 2 flow invocations', { timeout: 30_000 }, async () => { - const server = await startServer({ world }).then(createFetcher); + const server = await startServer({ world, env: INLINE_EXEC_ENV }).then(createFetcher); const result = await server.invoke( 'workflows/inline-execution.ts', 'sleepWorkflow', @@ -116,7 +123,7 @@ export function inlineExecution(world: string) { 'parallel steps (Promise.all) complete in 1-3 flow invocations', { timeout: 30_000 }, async () => { - const server = await startServer({ world }).then(createFetcher); + const server = await startServer({ world, env: INLINE_EXEC_ENV }).then(createFetcher); const result = await server.invoke( 'workflows/inline-execution.ts', 'parallelStepsWorkflow', diff --git a/packages/world-vercel/src/snapshots.test.ts b/packages/world-vercel/src/snapshots.test.ts new file mode 100644 index 0000000000..fb8b27bc78 --- /dev/null +++ b/packages/world-vercel/src/snapshots.test.ts @@ -0,0 +1,180 @@ +import { createServer, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// Mock getHttpConfig to return a localhost URL pointing at the test server. +// Set per-test via setBaseUrl(). +let baseUrl = 'http://127.0.0.1:0'; +vi.mock('./utils.js', () => ({ + getHttpConfig: vi.fn(() => + Promise.resolve({ + baseUrl, + headers: new Headers(), + usingProxy: false, + }) + ), +})); + +// Bypass the OIDC token fetch in getHttpConfig — handled by the mock above. + +import { createSnapshotsStorage } from './snapshots.js'; + +interface RequestRecord { + method: string; + path: string; + contentLength?: string; + bodyBytes: number; + bodyError?: string; +} + +/** + * HTTP test server with programmable response handlers. + * + * Each test installs a handler via `server.handle = (req, res, attempt) => …`. + * The server tracks per-request body sizes and content-length so tests can + * assert that the FULL body was received on every attempt (not 0 bytes, + * which is the symptom of the undici fetch+RetryAgent+Buffer-body bug). + */ +class TestServer { + server!: Server; + url = ''; + records: RequestRecord[] = []; + handle: + | (( + req: import('node:http').IncomingMessage, + res: import('node:http').ServerResponse, + attempt: number + ) => void) + | undefined; + + async start(): Promise { + this.records = []; + this.server = createServer((req, res) => { + const cl = req.headers['content-length'] as string | undefined; + let bodyBytes = 0; + const record: RequestRecord = { + method: req.method ?? '?', + path: req.url ?? '?', + contentLength: cl, + bodyBytes: 0, + }; + req.on('data', (chunk) => { + bodyBytes += chunk.length; + }); + req.on('end', () => { + record.bodyBytes = bodyBytes; + this.records.push(record); + const attempt = this.records.length; + if (this.handle) { + this.handle(req, res, attempt); + } else { + res.writeHead(200); + res.end('ok'); + } + }); + req.on('error', (err) => { + record.bodyError = err.message; + record.bodyBytes = bodyBytes; + this.records.push(record); + }); + }); + await new Promise((resolve) => this.server.listen(0, resolve)); + const { port } = this.server.address() as AddressInfo; + this.url = `http://127.0.0.1:${port}`; + } + + async stop(): Promise { + if (this.server) { + await new Promise((resolve) => { + this.server.close(() => resolve()); + }); + } + } +} + +describe('snapshots storage', () => { + let server: TestServer; + + beforeEach(async () => { + server = new TestServer(); + await server.start(); + baseUrl = server.url; + }); + + afterEach(async () => { + await server.stop(); + }); + + describe('save', () => { + it('sends a single PUT with the full body when the server responds 200', async () => { + server.handle = (_req, res) => { + res.writeHead(200); + res.end('ok'); + }; + const storage = createSnapshotsStorage(); + const data = new Uint8Array(1024).fill(7); + await storage.save('wrun_test', data, { + eventsCursor: 'eid:test', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }); + + expect(server.records).toHaveLength(1); + const r = server.records[0]!; + expect(r.method).toBe('PUT'); + expect(r.path).toBe('/v2/runs/wrun_test/snapshot'); + expect(r.bodyBytes).toBeGreaterThan(0); + expect(r.bodyBytes).toBe(Number(r.contentLength)); + }); + + it('retries on transient 503 and sends the full body on every attempt (regression: undici fetch+RetryAgent loses Buffer body on retry)', async () => { + // First attempt: 503 (transient). Second: 200. + // The undici fetch() + RetryAgent combo wraps Buffer bodies in a + // one-shot ReadableStream, so the second attempt sends 0 bytes + // and triggers UND_ERR_REQ_CONTENT_LENGTH_MISMATCH. Switching + // the snapshot save path to undici.request() preserves the body + // across retries. + server.handle = (_req, res, attempt) => { + if (attempt === 1) { + res.writeHead(503); + res.end('try again'); + } else { + res.writeHead(200); + res.end('ok'); + } + }; + + const storage = createSnapshotsStorage(); + const data = new Uint8Array(64 * 1024).fill(42); + await storage.save('wrun_retry', data, { + eventsCursor: null, + createdAt: new Date('2024-01-02T00:00:00.000Z'), + }); + + expect(server.records).toHaveLength(2); + // BOTH attempts must include the full body. If the body were lost + // on retry, attempt 2 would have bodyBytes === 0 and the request + // would fail with UND_ERR_REQ_CONTENT_LENGTH_MISMATCH before + // reaching the server at all. + for (const r of server.records) { + expect(r.method).toBe('PUT'); + expect(r.bodyBytes).toBeGreaterThan(0); + expect(r.bodyBytes).toBe(Number(r.contentLength)); + } + }); + + it('throws WorkflowWorldError when the server returns 4xx', async () => { + server.handle = (_req, res) => { + res.writeHead(400); + res.end('bad request'); + }; + const storage = createSnapshotsStorage(); + const data = new Uint8Array(16); + await expect( + storage.save('wrun_bad', data, { + eventsCursor: null, + createdAt: new Date(), + }) + ).rejects.toThrow(/HTTP 400/); + }); + }); +}); diff --git a/packages/world-vercel/src/snapshots.ts b/packages/world-vercel/src/snapshots.ts new file mode 100644 index 0000000000..0acaa285f8 --- /dev/null +++ b/packages/world-vercel/src/snapshots.ts @@ -0,0 +1,207 @@ +import { WorkflowWorldError } from '@workflow/errors'; +import type { SnapshotMetadata, Storage } from '@workflow/world'; +import { request as undiciRequest } from 'undici'; +import { getDispatcher } from './http-client.js'; +import { type APIConfig, getHttpConfig } from './utils.js'; + +/** + * Convert a Web `Headers` object into a plain record for undici's + * lower-level `request()` API. Headers in undici-request take + * `Record`, not the Headers object. + */ +function headersToRecord(headers: Headers): Record { + const record: Record = {}; + for (const [key, value] of headers) { + record[key] = value; + } + return record; +} + +/** + * Create snapshot storage backed by the workflow-server API. + * + * Compression and encryption are handled by `@workflow/core`'s + * snapshot entrypoint (`compress(snapshot) → encrypt → save`). This + * world layer transports the bytes opaquely — it does not compress + * (encryption produces ciphertext that doesn't compress) and it does + * not encrypt. + * + * Snapshot endpoints use raw binary transfer: + * - PUT /v2/runs/:runId/snapshot — binary body, metadata in headers + * - GET /v2/runs/:runId/snapshot — binary response, metadata in headers + * - DELETE /v2/runs/:runId/snapshot — no body + */ +export function createSnapshotsStorage( + config?: APIConfig +): Storage['snapshots'] { + return { + async save( + runId: string, + data: Uint8Array, + metadata: SnapshotMetadata + ): Promise { + const t0 = performance.now(); + const { baseUrl, headers } = await getHttpConfig(config); + const url = `${baseUrl}/v2/runs/${encodeURIComponent(runId)}/snapshot`; + + // Bytes arrive opaquely from the core's + // `compress → encrypt` pipeline. Forward verbatim. + headers.set('Content-Type', 'application/octet-stream'); + headers.set('X-Snapshot-Events-Cursor', metadata.eventsCursor ?? ''); + headers.set('X-Snapshot-Created-At', metadata.createdAt.toISOString()); + + // Use undici.request() rather than the global fetch() because + // fetch() + RetryAgent is broken for Buffer/Uint8Array bodies: + // fetch wraps the body in a one-shot ReadableStream (per the + // WHATWG fetch spec), so when the RetryAgent retries (on 5xx or + // network errors), the second attempt sends 0 bytes and undici + // throws `UND_ERR_REQ_CONTENT_LENGTH_MISMATCH`. The lower-level + // `request()` API hands the Buffer to the connection layer + // directly, which can be replayed on retry. + // + // Upstream context: nodejs/undici#3288 (filed May 2024) reported + // this exact failure. The "fix" in nodejs/undici#3294 made + // RetryAgent skip stateful bodies rather than rewind them, and + // the maintainers explicitly recommended switching to + // `undici.request()` for any retried request with a body. Don't + // simplify this back to `fetch()` without first verifying that + // upstream now copies Buffers across retries. + // + // Snapshot bodies are 5-15 MB so the bug fires constantly under + // network turbulence; a single failed save poisons the run + // (handler returns 500 -> queue retries handler -> save fails + // again -> 5xx loop until the run TTL). + const putStart = performance.now(); + const response = await undiciRequest(url, { + method: 'PUT', + body: data, + headers: headersToRecord(headers), + dispatcher: getDispatcher(), + }); + const putDurationMs = Math.round(performance.now() - putStart); + + if (response.statusCode < 200 || response.statusCode >= 300) { + const text = await response.body.text().catch(() => ''); + throw new WorkflowWorldError( + `PUT /v2/runs/${runId}/snapshot -> HTTP ${response.statusCode}: ${text}`, + { url, status: response.statusCode } + ); + } + + // Consume the response body to release the connection + await response.body.text(); + + // CI-visible diagnostic: actual on-the-wire snapshot bytes and + // the HTTP-PUT cost. Mirrors the SNAPSHOT_DIAG checkpoint format + // from `@workflow/core` so a wedged run's entire save/load + // lifecycle is grep-able by runId in Vercel function logs. + // Emitted at warn level (always-on, no DEBUG required). + console.warn('[Workflow] WORLD_SNAPSHOT_DIAG', { + op: 'save', + runId, + // Bytes received from the core — already compressed and + // encrypted upstream. The world transports them opaquely. + wireBytes: data.byteLength, + putDurationMs, + totalDurationMs: Math.round(performance.now() - t0), + }); + }, + + async load( + runId: string + ): Promise<{ data: Uint8Array; metadata: SnapshotMetadata } | null> { + const t0 = performance.now(); + const { baseUrl, headers } = await getHttpConfig(config); + const url = `${baseUrl}/v2/runs/${encodeURIComponent(runId)}/snapshot`; + + headers.set('Accept', 'application/octet-stream'); + + const getStart = performance.now(); + const response = await fetch(url, { + method: 'GET', + headers, + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- undici dispatcher + dispatcher: getDispatcher(), + } as any); + const getDurationMs = Math.round(performance.now() - getStart); + + if (response.status === 404) { + // Consume the response body to release the connection + await response.text().catch(() => {}); + // Diagnostic: emit the not-found case so we can correlate the + // skip-load fast-path in core with whatever the world saw. + console.warn('[Workflow] WORLD_SNAPSHOT_DIAG', { + op: 'load', + runId, + outcome: 'not_found', + getDurationMs, + totalDurationMs: Math.round(performance.now() - t0), + }); + return null; + } + + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new WorkflowWorldError( + `GET /v2/runs/${runId}/snapshot -> HTTP ${response.status}: ${text}`, + { url, status: response.status } + ); + } + + const buffer = await response.arrayBuffer(); + const data = new Uint8Array(buffer); + + const eventsCursor = + response.headers.get('X-Snapshot-Events-Cursor') || null; + const createdAtStr = response.headers.get('X-Snapshot-Created-At'); + const createdAt = createdAtStr ? new Date(createdAtStr) : new Date(); + + // CI-visible diagnostic: actual on-the-wire snapshot bytes and + // HTTP-GET cost. Same format/pairing as the save side above so + // the entire snapshot save/load lifecycle is grep-able from + // Vercel function logs by runId. + console.warn('[Workflow] WORLD_SNAPSHOT_DIAG', { + op: 'load', + runId, + outcome: 'ok', + // Bytes returned by the workflow-server (already + // compressed+encrypted by core; this layer transports them + // opaquely). + wireBytes: data.byteLength, + getDurationMs, + totalDurationMs: Math.round(performance.now() - t0), + }); + + return { + data, + metadata: { + eventsCursor: eventsCursor || null, + createdAt, + }, + }; + }, + + async delete(runId: string): Promise { + const { baseUrl, headers } = await getHttpConfig(config); + const url = `${baseUrl}/v2/runs/${encodeURIComponent(runId)}/snapshot`; + + const response = await fetch(url, { + method: 'DELETE', + headers, + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- undici dispatcher + dispatcher: getDispatcher(), + } as any); + + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new WorkflowWorldError( + `DELETE /v2/runs/${runId}/snapshot -> HTTP ${response.status}: ${text}`, + { url, status: response.status } + ); + } + + // Consume the response body to release the connection + await response.text(); + }, + }; +} diff --git a/packages/world-vercel/src/storage.ts b/packages/world-vercel/src/storage.ts index 38c62b78cb..06dd7a3c25 100644 --- a/packages/world-vercel/src/storage.ts +++ b/packages/world-vercel/src/storage.ts @@ -7,6 +7,7 @@ import { import { getHook, getHookByToken, listHooks } from './hooks.js'; import { instrumentObject } from './instrumentObject.js'; import { getWorkflowRun, listWorkflowRuns } from './runs.js'; +import { createSnapshotsStorage } from './snapshots.js'; import { getStep, listWorkflowRunSteps } from './steps.js'; import type { APIConfig } from './utils.js'; @@ -37,6 +38,7 @@ export function createStorage(config?: APIConfig): Storage { getByToken: (token) => getHookByToken(token, config), list: (params) => listHooks(params, config), }, + snapshots: createSnapshotsStorage(config), }; // Instrument all storage methods with tracing @@ -46,5 +48,6 @@ export function createStorage(config?: APIConfig): Storage { steps: instrumentObject('world.steps', storage.steps), events: instrumentObject('world.events', storage.events), hooks: instrumentObject('world.hooks', storage.hooks), + snapshots: instrumentObject('world.snapshots', storage.snapshots), }; } diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 06ce262df3..6dc7a480f2 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -33,6 +33,8 @@ export { LegacySerializedDataSchemaV1, SerializedDataSchema, } from './serialization.js'; +export type * from './snapshots.js'; +export { SnapshotMetadataSchema } from './snapshots.js'; export type * from './shared.js'; export type { GetChunksOptions, diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index ce8a5d2b94..9137a04330 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -9,6 +9,7 @@ import type { RunCreatedEventRequest, } from './events.js'; import type { GetHookParams, Hook, ListHooksParams } from './hooks.js'; +import type { SnapshotMetadata } from './snapshots.js'; import type { Queue } from './queue.js'; import type { GetWorkflowRunParams, @@ -232,6 +233,51 @@ export interface Storage { getByToken(token: string, params?: GetHookParams): Promise; list(params: ListHooksParams): Promise>; }; + + /** + * VM snapshot storage for the snapshot-based runtime. + * + * Snapshots capture the state of the QuickJS WASM VM at a suspension point, + * allowing workflow execution to resume from the exact point of suspension + * instead of replaying the full event log. + * + * The metadata (including eventsCursor) is stored alongside the snapshot data + * so that on restore, only events created after the snapshot need to be fetched. + */ + snapshots: { + /** + * Save a VM snapshot for a workflow run. + * Each save overwrites the previous snapshot for this run. + * + * @param runId - The workflow run ID + * @param data - The serialized snapshot bytes (from QuickJS.serializeSnapshot()) + * @param metadata - Snapshot metadata including the last processed event ID + */ + save( + runId: string, + data: Uint8Array, + metadata: SnapshotMetadata + ): Promise; + + /** + * Load the most recent VM snapshot for a workflow run. + * Returns null if no snapshot exists (first invocation). + * + * @param runId - The workflow run ID + * @returns The snapshot data and metadata, or null if not found + */ + load( + runId: string + ): Promise<{ data: Uint8Array; metadata: SnapshotMetadata } | null>; + + /** + * Delete the snapshot for a workflow run. + * Called when the workflow reaches a terminal state (completed, failed, cancelled). + * + * @param runId - The workflow run ID + */ + delete(runId: string): Promise; + }; } /** diff --git a/packages/world/src/snapshots.ts b/packages/world/src/snapshots.ts new file mode 100644 index 0000000000..5b3998d5b3 --- /dev/null +++ b/packages/world/src/snapshots.ts @@ -0,0 +1,14 @@ +import { z } from 'zod'; + +export const SnapshotMetadataSchema = z.object({ + /** + * Pagination cursor for events.list() — the snapshot was taken at + * this point in the event log. On restore, only events AFTER this + * cursor need to be fetched. + */ + eventsCursor: z.string().nullable(), + /** Timestamp when the snapshot was created */ + createdAt: z.coerce.date(), +}); + +export type SnapshotMetadata = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 72aeae81cd..31a212be5a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -591,6 +591,9 @@ importers: nanoid: specifier: 5.1.6 version: 5.1.6 + quickjs-wasi: + specifier: 2.0.0 + version: 2.0.0 seedrandom: specifier: 3.0.5 version: 3.0.5 @@ -13314,6 +13317,9 @@ packages: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} engines: {node: '>=10'} + quickjs-wasi@2.0.0: + resolution: {integrity: sha512-9bSUf9KSi4wAWQpgFZNYx/aeL7v0wBd+jgjwNNhL115BD0JsC9ojBfkqiAeh/CynwX39OUYUSU2myisgFteLog==} + radix-ui@1.4.3: resolution: {integrity: sha512-aWizCQiyeAenIdUbqEpXgRA1ya65P13NKn/W8rWkcN0OPkRDxdBVLWnIEDsS2RpwCK2nobI7oMUSmexzTDyAmA==} peerDependencies: @@ -29647,6 +29653,8 @@ snapshots: quick-lru@5.1.1: {} + quickjs-wasi@2.0.0: {} + radix-ui@1.4.3(@types/react-dom@19.1.9(@types/react@19.1.13))(@types/react@19.1.13)(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: '@radix-ui/primitive': 1.1.3 diff --git a/scripts/create-test-matrix.mjs b/scripts/create-test-matrix.mjs index be2573ea9f..3c5275d3cc 100644 --- a/scripts/create-test-matrix.mjs +++ b/scripts/create-test-matrix.mjs @@ -176,4 +176,18 @@ matrix.app.push({ ...DEV_TEST_CONFIGS['tanstack-start'], }); +// Cross-product with the runtime axis: every app is tested against both +// the snapshot runtime (the default) and the event-replay runtime +// (opt-in via WORKFLOW_RUNTIME=replay). Each runtime gets its own +// artifactSuffix and runLabel so CI artifacts and job names are unique. +const RUNTIMES = ['snapshot', 'replay']; +matrix.app = matrix.app.flatMap((app) => + RUNTIMES.map((runtime) => ({ + ...app, + runtime, + runLabel: `${app.runLabel} ${runtime}`, + artifactSuffix: `${app.artifactSuffix}-${runtime}`, + })) +); + console.log(JSON.stringify(matrix));