From 63c7cd38c8b4c51a6c973c1d994aa71dfe92b28a Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Fri, 12 Jun 2026 17:06:04 -0700 Subject: [PATCH 1/5] feat(core,world): gzip-compress serialized payloads behind specVersion 5 Add a composable 'gzip' format prefix layer to the serialization pipeline (compress before encrypt: encr(gzip(devl))), cutting stored payload bytes by ~70-87% on real-world-style workloads. Compression is gated on run specVersion 5 (new SPEC_VERSION_SUPPORTS_COMPRESSION) and on target-deployment capabilities for cross-deployment writes; payloads under 1KB or that don't compress meaningfully are stored unchanged. Reads dispatch on the format prefix so both compressed and uncompressed data are always readable. WORKFLOW_DISABLE_COMPRESSION=1 disables writes. Co-Authored-By: Claude Fable 5 --- .changeset/gzip-ref-compression-core.md | 5 + .changeset/gzip-ref-compression-world.md | 5 + .../core/scripts/benchmark-compression.mjs | 264 ++++++++++++++++ packages/core/src/capabilities.ts | 7 + packages/core/src/runtime.ts | 21 +- packages/core/src/runtime/resume-hook.ts | 11 +- packages/core/src/runtime/start.ts | 27 +- packages/core/src/runtime/step-executor.ts | 46 ++- .../core/src/runtime/step-handler.test.ts | 5 +- packages/core/src/runtime/step-handler.ts | 42 ++- .../core/src/runtime/suspension-handler.ts | 18 +- packages/core/src/serialization-format.ts | 106 ++++++- packages/core/src/serialization.ts | 71 +++-- packages/core/src/serialization/client.ts | 7 +- packages/core/src/serialization/codec.ts | 10 + .../src/serialization/compression.test.ts | 298 ++++++++++++++++++ .../core/src/serialization/compression.ts | 179 +++++++++++ packages/core/src/serialization/index.ts | 8 + packages/core/src/serialization/step.ts | 7 +- packages/core/src/serialization/types.ts | 2 + packages/core/src/workflow.ts | 8 +- packages/world/src/index.ts | 1 + packages/world/src/spec-version.test.ts | 53 ++++ packages/world/src/spec-version.ts | 14 +- 24 files changed, 1150 insertions(+), 65 deletions(-) create mode 100644 .changeset/gzip-ref-compression-core.md create mode 100644 .changeset/gzip-ref-compression-world.md create mode 100644 packages/core/scripts/benchmark-compression.mjs create mode 100644 packages/core/src/serialization/compression.test.ts create mode 100644 packages/core/src/serialization/compression.ts create mode 100644 packages/world/src/spec-version.test.ts diff --git a/.changeset/gzip-ref-compression-core.md b/.changeset/gzip-ref-compression-core.md new file mode 100644 index 0000000000..9393bd3215 --- /dev/null +++ b/.changeset/gzip-ref-compression-core.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': minor +--- + +Gzip-compress serialized payloads (step inputs/outputs, workflow arguments/return values, errors, hook payloads) before storage using a new composable `gzip` format prefix. Compression is applied before encryption, gated on run specVersion 5, and skipped for small or incompressible payloads. Set `WORKFLOW_DISABLE_COMPRESSION=1` to opt out of writes; reads always handle both formats. diff --git a/.changeset/gzip-ref-compression-world.md b/.changeset/gzip-ref-compression-world.md new file mode 100644 index 0000000000..f881f078a4 --- /dev/null +++ b/.changeset/gzip-ref-compression-world.md @@ -0,0 +1,5 @@ +--- +'@workflow/world': minor +--- + +Bump `SPEC_VERSION_CURRENT` to 5 (`SPEC_VERSION_SUPPORTS_COMPRESSION`): runs at spec 5+ may contain gzip-compressed payloads, and older SDKs reject them via `requiresNewerWorld()` instead of failing on individual payloads. diff --git a/packages/core/scripts/benchmark-compression.mjs b/packages/core/scripts/benchmark-compression.mjs new file mode 100644 index 0000000000..5395c0b416 --- /dev/null +++ b/packages/core/scripts/benchmark-compression.mjs @@ -0,0 +1,264 @@ +// Benchmark: serialized payload sizes with and without gzip compression. +// +// Measures the exact bytes the serialization layer hands to the World +// storage backends (S3/DynamoDB refs for world-vercel, bytea columns for +// world-postgres, JSON files for world-local) for a set of real-world-style +// workloads, with compression off vs on. +// +// Usage (from packages/core, after `pnpm build`): +// node scripts/benchmark-compression.mjs +// +// Note: backends that store binary as base64 (DynamoDB inline refs, +// world-local JSON files) amplify every byte by 4/3, so the absolute +// savings there are ~33% larger than the raw numbers below. + +import * as step from '../dist/serialization/step.js'; + +const encoder = new TextEncoder(); + +// Deterministic PRNG (xorshift32) so benchmark runs are reproducible. +function makeRng(seed = 0x9e3779b9) { + let state = seed; + return () => { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + return (state >>> 0) / 0xffffffff; + }; +} + +const VOCAB = + `the a an of to in on for with by from runtime workflow step event log replay deterministic durable execution serverless function queue retry backoff failure deployment cold start state checkpoint persist suspend resume orchestrator sandbox compute storage payload serialization compression encryption stream hook webhook token correlation idempotent schedule timer sleep await promise race parallel batch fetch request response error fatal retryable budget timeout attempt delivery message billing invoice customer subscription usage metric latency throughput region edge node cluster shard partition index query transaction commit rollback migration schema column record entity snapshot version`.split( + /\s+/ + ); + +/** Generate plausible, non-repetitive English-ish text deterministically. */ +function makeText(rng, words) { + const out = []; + let sentenceLen = 0; + for (let i = 0; i < words; i++) { + let word = VOCAB[Math.floor(rng() * VOCAB.length)]; + if (sentenceLen === 0) word = word[0].toUpperCase() + word.slice(1); + out.push(word); + sentenceLen++; + if (sentenceLen > 6 && rng() < 0.18) { + out[out.length - 1] += '.'; + sentenceLen = 0; + } else if (rng() < 0.08) { + out[out.length - 1] += ','; + } + } + return out.join(' '); +} + +// --------------------------------------------------------------------------- +// Real-world-style workloads +// --------------------------------------------------------------------------- + +/** AI agent chat history — the canonical DurableAgent workload. */ +function aiChatHistory(messages = 60) { + const rng = makeRng(0xc0ffee); + const out = []; + for (let i = 0; i < messages; i++) { + if (i % 2 === 0) { + out.push({ + role: 'user', + content: makeText(rng, 30 + Math.floor(rng() * 40)), + }); + } else { + out.push({ + role: 'assistant', + content: makeText(rng, 150 + Math.floor(rng() * 250)), + toolCalls: + i % 6 === 5 + ? [ + { + toolCallId: `call_${i}_${Math.floor(rng() * 1e12).toString(36)}`, + toolName: 'search_documentation', + args: { + query: makeText(rng, 6), + limit: 5, + }, + }, + ] + : undefined, + }); + } + } + return { messages: out, model: 'claude-fable-5', temperature: 0.7 }; +} + +/** Paginated REST API response — list endpoints fetched in steps. */ +function apiUserList(count = 250) { + return { + users: Array.from({ length: count }, (_, i) => ({ + id: `usr_${i.toString(16).padStart(8, '0')}`, + object: 'user', + email: `person.${i}@bigcorp-enterprises.example.com`, + name: `Person Q. Example the ${i}th`, + role: i % 7 === 0 ? 'admin' : i % 3 === 0 ? 'editor' : 'viewer', + teamIds: [`team_${i % 12}`, `team_${i % 5}`], + createdAt: new Date(Date.UTC(2024, i % 12, (i % 27) + 1)).toISOString(), + updatedAt: new Date(Date.UTC(2026, i % 12, (i % 27) + 1)).toISOString(), + settings: { + notifications: { email: true, slack: i % 2 === 0, mobile: false }, + timezone: 'America/Los_Angeles', + locale: 'en-US', + }, + metadata: { source: 'scim-sync', importBatch: `batch_${i % 40}` }, + })), + hasMore: true, + nextCursor: 'usr_000000fa', + }; +} + +/** E-commerce order — checkout/fulfillment workflow state. */ +function ecommerceOrder(lineItems = 30) { + return { + orderId: 'ord_2WqK9mPx7nL4vR8t', + status: 'processing', + customer: { + id: 'cus_9XkL2mNp5qR7sT1v', + email: 'jane.shopper@example.com', + shippingAddress: { + line1: '2001 Workflow Way', + line2: 'Suite 400', + city: 'San Francisco', + state: 'CA', + postalCode: '94107', + country: 'US', + }, + }, + items: Array.from({ length: lineItems }, (_, i) => ({ + sku: `SKU-${1000 + i}`, + name: `Durable Widget ${i} — Professional Edition`, + quantity: (i % 3) + 1, + unitPriceCents: 1999 + i * 250, + taxCents: Math.round((1999 + i * 250) * 0.0875), + fulfillmentStatus: i % 4 === 0 ? 'backordered' : 'allocated', + warehouse: `wh-${i % 6}`, + })), + payments: [ + { + id: 'pay_4YmN8pQr2sT6uV0w', + provider: 'stripe', + amountCents: 84321, + status: 'captured', + capturedAt: '2026-06-11T18:23:11.000Z', + }, + ], + timeline: Array.from({ length: 12 }, (_, i) => ({ + at: `2026-06-11T18:${String(i * 4).padStart(2, '0')}:00.000Z`, + event: ['created', 'paid', 'allocated', 'picked'][i % 4], + actor: 'system', + })), + }; +} + +/** Scraped/generated document text — summarization pipelines. */ +function markdownDocument(paragraphs = 40) { + const rng = makeRng(0xd0c5); + const parts = []; + for (let i = 0; i < paragraphs; i++) { + parts.push(makeText(rng, 60 + Math.floor(rng() * 60))); + } + return { + url: 'https://example.com/blog/durable-execution-deep-dive', + title: 'Durable Execution: A Deep Dive', + fetchedAt: '2026-06-12T09:00:00.000Z', + content: parts.join('\n\n'), + }; +} + +/** Time-series metrics — monitoring/aggregation workloads. */ +function timeSeries(points = 2000) { + const base = 1749700000000; + return { + metric: 'http.server.request.duration', + unit: 'ms', + points: Array.from({ length: points }, (_, i) => [ + base + i * 60_000, + Math.round(40 + 30 * Math.sin(i / 50) + (i % 17)), + ]), + }; +} + +/** Tiny payload — stays below the compression threshold by design. */ +function tinyPayload() { + return { ok: true, id: 'wrun_01J5XYZ', count: 3 }; +} + +/** Incompressible binary — e.g. an image/zip passed through a step. */ +function binaryPayload(bytes = 256 * 1024) { + // Deterministic pseudo-random bytes (xorshift) so runs are reproducible + let state = 0x12345678; + const data = new Uint8Array(bytes); + for (let i = 0; i < bytes; i++) { + state ^= state << 13; + state ^= state >>> 17; + state ^= state << 5; + data[i] = state & 0xff; + } + return data; +} + +// --------------------------------------------------------------------------- +// Measurement +// --------------------------------------------------------------------------- + +function byteLength(data) { + if (data instanceof Uint8Array) return data.byteLength; + return encoder.encode(JSON.stringify(data)).byteLength; +} + +function fmt(n) { + if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(2)} MB`; + if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${n} B`; +} + +const WORKLOADS = [ + ['AI chat history (60 messages)', aiChatHistory()], + ['API response (250 users)', apiUserList()], + ['E-commerce order (30 items)', ecommerceOrder()], + ['Scraped document (~14 KB text)', markdownDocument()], + ['Time series (2000 points)', timeSeries()], + ['Random binary (256 KB)', binaryPayload()], + ['Tiny payload (<1 KB)', tinyPayload()], +]; + +const rows = []; +for (const [name, value] of WORKLOADS) { + const off = await step.serialize(value, undefined, {}); + const on = await step.serialize(value, undefined, { compression: true }); + const offBytes = byteLength(off); + const onBytes = byteLength(on); + const savings = ((1 - onBytes / offBytes) * 100).toFixed(1); + rows.push({ name, offBytes, onBytes, savings }); +} + +console.log('| Workload | Uncompressed | Compressed | Savings |'); +console.log('| --- | ---: | ---: | ---: |'); +for (const { name, offBytes, onBytes, savings } of rows) { + const note = offBytes === onBytes ? ' (passthrough)' : ''; + console.log( + `| ${name} | ${fmt(offBytes)} | ${fmt(onBytes)}${note} | ${offBytes === onBytes ? '—' : `${savings}%`} |` + ); +} + +// Simulated event-log total for a representative run: an AI agent workflow +// with 10 steps that each return a growing chat history (the replay model +// re-serializes the full conversation at every step boundary). +let totalOff = 0; +let totalOn = 0; +for (let i = 1; i <= 10; i++) { + const value = aiChatHistory(6 * i); + totalOff += byteLength(await step.serialize(value, undefined, {})); + totalOn += byteLength( + await step.serialize(value, undefined, { compression: true }) + ); +} +console.log(''); +console.log( + `Simulated 10-step AI agent run (event log total): ${fmt(totalOff)} → ${fmt(totalOn)} (${((1 - totalOn / totalOff) * 100).toFixed(1)}% smaller)` +); diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index 18321c6bb7..05f5ce0f7a 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -28,6 +28,7 @@ * Commit: 7618ac36 "Wire AES-GCM encryption into serialization layer (#1251)" * https://github.com/vercel/workflow/commit/7618ac36 * - `framedByteStreams` (wire-level chunk framing for byte streams): added in `5.0.0-beta.15` + * - `gzip` (gzip payload compression): added in `5.0.0-beta.16` */ import semver from 'semver'; @@ -68,6 +69,12 @@ const FORMAT_VERSION_TABLE: ReadonlyArray<{ minVersion: string; }> = [ { format: SerializationFormat.ENCRYPTED, minVersion: '4.2.0-beta.64' }, + // TODO(release): verify this matches the actual version that ships gzip + // payload compression. If a "Version Packages (beta)" PR merges before this + // change, bump to the next beta. A too-low cutoff makes new producers write + // compressed payloads to consumers that cannot decompress them; too-high + // merely delays the optimization (safe). + { format: SerializationFormat.GZIP, minVersion: '5.0.0-beta.16' }, // Future entries: // { format: SerializationFormat.CBOR, minVersion: '5.x.y' }, // { format: SerializationFormat.ENCRYPTED_V2, minVersion: '5.x.y' }, diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 9d9b6351a3..532caf547d 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -15,6 +15,7 @@ import { getQueueTopicPrefix, resolveQueueNamespace, SPEC_VERSION_CURRENT, + SPEC_VERSION_SUPPORTS_COMPRESSION, WorkflowInvokePayloadSchema, type WorkflowRun, type World, @@ -155,7 +156,13 @@ async function recordFatalRunError({ eventType: 'run_failed', specVersion: SPEC_VERSION_CURRENT, eventData: { - error: await dehydrateRunError(err, runId, await getEncryptionKey()), + error: await dehydrateRunError( + err, + runId, + await getEncryptionKey(), + globalThis, + (workflowRun?.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION + ), errorCode, }, }, @@ -418,6 +425,7 @@ export function workflowEntrypoint( workflowStartedAt: bgStartedAt, stepId: incomingStepId, stepName: incomingStepName, + runSpecVersion: bgRun.specVersion, }); } finally { replayBudget.resume(); @@ -1017,7 +1025,10 @@ export function workflowEntrypoint( error: await dehydrateRunError( suspensionError, runId, - encryptionKey + encryptionKey, + globalThis, + (workflowRun?.specVersion ?? 0) >= + SPEC_VERSION_SUPPORTS_COMPRESSION ), errorCode, }, @@ -1228,6 +1239,7 @@ export function workflowEntrypoint( workflowStartedAt, stepId: inlineStep.correlationId, stepName: inlineStep.stepName, + runSpecVersion: workflowRun.specVersion, }); } finally { replayBudget.resume(); @@ -1403,7 +1415,10 @@ export function workflowEntrypoint( error: await dehydrateRunError( terminalError, runId, - encryptionKey + encryptionKey, + globalThis, + (workflowRun?.specVersion ?? 0) >= + SPEC_VERSION_SUPPORTS_COMPRESSION ), errorCode, }, diff --git a/packages/core/src/runtime/resume-hook.ts b/packages/core/src/runtime/resume-hook.ts index 3c5a653d76..0ffbf4cc16 100644 --- a/packages/core/src/runtime/resume-hook.ts +++ b/packages/core/src/runtime/resume-hook.ts @@ -8,6 +8,7 @@ import { isLegacySpecVersion, SPEC_VERSION_CURRENT, SPEC_VERSION_LEGACY, + SPEC_VERSION_SUPPORTS_COMPRESSION, type WorkflowInvokePayload, type WorkflowRun, } from '@workflow/world'; @@ -139,6 +140,13 @@ export async function resumeHook( encryptionKey = undefined; } + // Compress the payload only when the target run is marked as + // possibly containing compressed payloads (specVersion >= 5) AND + // its deployment can decode the 'gzip' format. + const compression = + (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION && + capabilities.supportedFormats.has(SerializationFormat.GZIP); + // Dehydrate the payload for storage const ops: Promise[] = []; const v1Compat = isLegacySpecVersion(hook.specVersion); @@ -149,7 +157,8 @@ export async function resumeHook( ops, globalThis, v1Compat, - capabilities.framedByteStreams + capabilities.framedByteStreams, + compression ); // These payload-stream ops are flushed in the background; the // promise handed to waitUntil must never reject (an unconsumed diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index c6f6c60ef2..7c3cc99b4e 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -9,6 +9,7 @@ import { isLegacySpecVersion, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, + SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, } from '@workflow/world'; import { monotonicFactory } from 'ulid'; @@ -17,7 +18,10 @@ import { getRunCapabilities } from '../capabilities.js'; import { importKey } from '../encryption.js'; import { runtimeLogger } from '../logger.js'; import type { Serializable } from '../schemas.js'; -import { dehydrateWorkflowArguments } from '../serialization.js'; +import { + dehydrateWorkflowArguments, + SerializationFormat, +} from '../serialization.js'; import * as Attribute from '../telemetry/semantic-conventions.js'; import { serializeTraceCarrier, trace } from '../telemetry.js'; import { version as workflowCoreVersion } from '../version.js'; @@ -211,18 +215,23 @@ export async function start( // Worlds that don't expose the `streams` API (e.g. minimal test // mocks) can't service health checks, so we skip the probe for them. let framedByteStreams: boolean; + let targetSupportsCompression: boolean; if (deploymentId === currentDeploymentId) { framedByteStreams = true; + targetSupportsCompression = true; } else if (typeof world.streams?.get !== 'function') { framedByteStreams = false; + targetSupportsCompression = false; } else { const probe = await healthCheck(world, 'workflow', { deploymentId, timeout: CROSS_DEPLOYMENT_CAPABILITY_PROBE_TIMEOUT_MS, }).catch(() => undefined); - framedByteStreams = getRunCapabilities( - probe?.workflowCoreVersion - ).framedByteStreams; + const capabilities = getRunCapabilities(probe?.workflowCoreVersion); + framedByteStreams = capabilities.framedByteStreams; + targetSupportsCompression = capabilities.supportedFormats.has( + SerializationFormat.GZIP + ); } const ops: Promise[] = []; @@ -296,6 +305,13 @@ export async function start( // Create run via run_created event (event-sourced architecture) // Pass client-generated runId - server will accept and use it + // Compress workflow arguments only when the run itself is marked as + // possibly containing compressed payloads (specVersion >= 5) AND the + // target deployment can decode them (same-deployment, or probed + // capability for cross-deployment starts). + const compression = + targetSupportsCompression && + specVersion >= SPEC_VERSION_SUPPORTS_COMPRESSION; const workflowArguments = await dehydrateWorkflowArguments( args, runId, @@ -303,7 +319,8 @@ export async function start( ops, globalThis, v1Compat, - framedByteStreams + framedByteStreams, + compression ); const executionContext = { diff --git a/packages/core/src/runtime/step-executor.ts b/packages/core/src/runtime/step-executor.ts index 1cfe99a78b..89d43c1426 100644 --- a/packages/core/src/runtime/step-executor.ts +++ b/packages/core/src/runtime/step-executor.ts @@ -10,7 +10,10 @@ import { } from '@workflow/errors'; import { pluralize } from '@workflow/utils'; import type { World } from '@workflow/world'; -import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { + SPEC_VERSION_CURRENT, + SPEC_VERSION_SUPPORTS_COMPRESSION, +} from '@workflow/world'; import type { CryptoKey } from '../encryption.js'; import { runtimeLogger, stepLogger } from '../logger.js'; import { getStepFunction } from '../private.js'; @@ -46,6 +49,12 @@ export interface StepExecutorParams { stepId: string; stepName: string; encryptionKey?: CryptoKey; + /** + * The workflow run's specVersion, used to gate payload compression. + * Step outputs/errors are only gzip-compressed when the run is marked + * as possibly containing compressed payloads (specVersion >= 5). + */ + runSpecVersion?: number; } /** @@ -79,6 +88,10 @@ export async function executeStep( stepName, } = params; const isVercel = process.env.VERCEL_URL !== undefined; + // Gate payload compression on the run's specVersion: only runs marked + // as possibly containing compressed payloads (spec >= 5) get gzip data. + const compression = + (params.runSpecVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; return trace(`STEP ${stepName}`, {}, async (span) => { span?.setAttributes({ @@ -115,7 +128,10 @@ export async function executeStep( error: await dehydrateStepError( new FatalError(errorMessage), workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compression ), }, }); @@ -249,7 +265,10 @@ export async function executeStep( error: await dehydrateStepError( wrappedError, workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compression ), }, }); @@ -354,7 +373,11 @@ export async function executeStep( result, workflowRunId, encryptionKey, - ops + ops, + globalThis, + false, + false, + compression ); const durationMs = Date.now() - startTime; dehydrateSpan?.setAttributes({ @@ -508,7 +531,10 @@ export async function executeStep( error: await dehydrateStepError( effectiveErr, workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compression ), }, }); @@ -573,7 +599,10 @@ export async function executeStep( error: await dehydrateStepError( wrappedError, workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compression ), }, }); @@ -632,7 +661,10 @@ export async function executeStep( error: await dehydrateStepError( err, workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compression ), ...(RetryableError.is(err) && { retryAfter: err.retryAfter }), }, diff --git a/packages/core/src/runtime/step-handler.test.ts b/packages/core/src/runtime/step-handler.test.ts index 278f5a6b0e..29caaad959 100644 --- a/packages/core/src/runtime/step-handler.test.ts +++ b/packages/core/src/runtime/step-handler.test.ts @@ -886,7 +886,10 @@ describe('step-handler fatal vs retryable behavior', () => { expect(dehydrateStepError).toHaveBeenCalledWith( expect.objectContaining({ name: 'FatalError', fatal: true }), 'wrun_test123', - undefined + undefined, + [], + globalThis, + expect.any(Boolean) ); }); diff --git a/packages/core/src/runtime/step-handler.ts b/packages/core/src/runtime/step-handler.ts index 201d93500b..17ff1500fb 100644 --- a/packages/core/src/runtime/step-handler.ts +++ b/packages/core/src/runtime/step-handler.ts @@ -15,6 +15,7 @@ import { getQueueTopicPrefix, resolveQueueNamespace, SPEC_VERSION_CURRENT, + SPEC_VERSION_SUPPORTS_COMPRESSION, type Step, StepInvokePayloadSchema, } from '@workflow/world'; @@ -224,6 +225,14 @@ function createStepHandler(namespace?: string) { // - retryAfter timestamp reached (returns 425 with Retry-After header) // - Workflow still active (returns 410 if completed) let step!: Step; + // Gate payload compression on the step entity's specVersion + // (stamped by the same-deployment orchestrator that created the + // step, so spec >= 5 implies every reader of this run's payloads + // understands the 'gzip' format). Returns false on the early + // failure paths where step_started didn't return an entity. + const compressionForStep = () => + ((step as Step | undefined)?.specVersion ?? 0) >= + SPEC_VERSION_SUPPORTS_COMPRESSION; try { const startResult = await world.events.create( workflowRunId, @@ -360,7 +369,10 @@ function createStepHandler(namespace?: string) { error: await dehydrateStepError( err, workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compressionForStep() ), }, }, @@ -460,7 +472,10 @@ function createStepHandler(namespace?: string) { error: await dehydrateStepError( wrappedError, workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compressionForStep() ), }, }, @@ -527,7 +542,10 @@ function createStepHandler(namespace?: string) { error: await dehydrateStepError( new FatalError(errorMessage), workflowRunId, - await getEncryptionKey() + await getEncryptionKey(), + [], + globalThis, + compressionForStep() ), }, }, @@ -666,7 +684,8 @@ function createStepHandler(namespace?: string) { ops, globalThis, false, - true + true, + compressionForStep() ); const durationMs = Date.now() - startTime; dehydrateSpan?.setAttributes({ @@ -777,7 +796,10 @@ function createStepHandler(namespace?: string) { error: await dehydrateStepError( effectiveErr, workflowRunId, - encryptionKey + encryptionKey, + [], + globalThis, + compressionForStep() ), }, }, @@ -862,7 +884,10 @@ function createStepHandler(namespace?: string) { error: await dehydrateStepError( wrappedError, workflowRunId, - encryptionKey + encryptionKey, + [], + globalThis, + compressionForStep() ), }, }, @@ -931,7 +956,10 @@ function createStepHandler(namespace?: string) { error: await dehydrateStepError( err, workflowRunId, - encryptionKey + encryptionKey, + [], + globalThis, + compressionForStep() ), ...(RetryableError.is(err) && { retryAfter: err.retryAfter, diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index a4f2840376..6a8f42a92d 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -11,6 +11,7 @@ import { type CreateEventRequest, type SerializedData, SPEC_VERSION_CURRENT, + SPEC_VERSION_SUPPORTS_COMPRESSION, type WorkflowRun, type World, } from '@workflow/world'; @@ -170,6 +171,11 @@ export async function handleSuspension({ const rawKey = await world.getEncryptionKeyForRun?.(run); const encryptionKey = rawKey ? await importKey(rawKey) : undefined; + // Gate payload compression on the run's specVersion: only runs marked + // as possibly containing compressed payloads (spec >= 5) get gzip data. + const compression = + (run.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION; + // Build and process hook_created events (same as V1) const hookEvents = await Promise.all( hooksNeedingCreation.map(async (queueItem) => { @@ -180,7 +186,9 @@ export async function handleSuspension({ queueItem.metadata, runId, encryptionKey, - suspension.globalThis + suspension.globalThis, + false, + compression )) as SerializedData); return { queueItem, @@ -287,7 +295,9 @@ export async function handleSuspension({ { aborted: true, reason: queueItem.abortReason }, runId, encryptionKey, - suspension.globalThis + suspension.globalThis, + false, + compression ); // Create hook_received event with abort payload @@ -375,7 +385,9 @@ export async function handleSuspension({ }, runId, encryptionKey, - suspension.globalThis + suspension.globalThis, + false, + compression ); const stepEvent: CreateEventRequest = { eventType: 'step_created' as const, diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index 26d967c984..a49af20e2d 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -17,6 +17,8 @@ export const SerializationFormat = { DEVALUE_V1: 'devl', /** Encrypted payload (inner payload has its own format prefix after decryption) */ ENCRYPTED: 'encr', + /** Gzip-compressed payload (inner payload has its own format prefix after decompression) */ + GZIP: 'gzip', } as const; export type SerializationFormatType = @@ -146,6 +148,76 @@ export function isEncryptedData(data: unknown): boolean { return prefix === SerializationFormat.ENCRYPTED; } +/** + * Check if a binary value has the 'gzip' format prefix indicating compression. + * Browser-safe — does not depend on the full serialization module. + */ +export function isCompressedData(data: unknown): boolean { + if (!(data instanceof Uint8Array) || data.length < FORMAT_PREFIX_LENGTH) { + return false; + } + const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); + return prefix === SerializationFormat.GZIP; +} + +/** + * Synchronously gunzip a payload when running on Node.js. + * + * This module is browser-safe, so `node:zlib` is resolved dynamically via + * `process.getBuiltinModule` (no static Node dependency, invisible to + * browser bundlers). Returns `undefined` when sync decompression isn't + * available in the current runtime — callers fall back to leaving the + * data un-hydrated (the async `hydrateDataWithKey` path handles + * decompression in browsers via `DecompressionStream`). + */ +function gunzipSyncIfAvailable(payload: Uint8Array): Uint8Array | undefined { + try { + const zlib = ( + globalThis as { + process?: { + getBuiltinModule?: (id: string) => { + gunzipSync?: (data: Uint8Array) => Uint8Array; + }; + }; + } + ).process?.getBuiltinModule?.('node:zlib'); + if (zlib?.gunzipSync) { + return new Uint8Array(zlib.gunzipSync(payload)); + } + } catch { + // Fall through — treat as unavailable + } + return undefined; +} + +/** + * Asynchronously gunzip a payload using the web-standard + * DecompressionStream (Node 18+, browsers, edge runtimes). + */ +async function gunzipAsync(payload: Uint8Array): Promise { + const transform = new DecompressionStream('gzip'); + const writer = transform.writable.getWriter(); + const writePromise = writer.write(payload).then(() => writer.close()); + writePromise.catch(() => {}); + const chunks: Uint8Array[] = []; + let total = 0; + const reader = transform.readable.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + total += value.length; + } + await writePromise; + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + // --------------------------------------------------------------------------- // Revivers type (shared across all environments) // --------------------------------------------------------------------------- @@ -189,6 +261,19 @@ export function hydrateData(value: unknown, revivers: Revivers): unknown { const str = new TextDecoder().decode(payload); return parse(str, revivers); } + if (format === SerializationFormat.GZIP) { + // Compressed payload — decompress synchronously when running on + // Node.js (CLI, server o11y). In browsers there is no sync gunzip; + // pass the data through untouched (like encrypted data) so async + // consumers can route it through `hydrateDataWithKey`, which + // decompresses via DecompressionStream. + const inflated = gunzipSyncIfAvailable(payload); + if (inflated === undefined) { + return value; + } + // The inflated bytes carry their own format prefix (e.g. 'devl') + return hydrateData(inflated, revivers); + } throw new Error(`Unsupported serialization format: ${format}`); } @@ -216,16 +301,23 @@ export async function hydrateDataWithKey( revivers: Revivers, key: import('./encryption.js').CryptoKey | undefined ): Promise { - if (value instanceof Uint8Array && isEncryptedData(value) && key) { + let data = value; + if (data instanceof Uint8Array && isEncryptedData(data) && key) { // Decrypt: strip 'encr' prefix, AES-GCM decrypt, then hydrate the result const { decrypt } = await import('./encryption.js'); - const { payload } = decodeFormatPrefix(value); - const decrypted = await decrypt(key, payload); - // The decrypted bytes have their own format prefix (e.g., 'devl') - return hydrateData(decrypted, revivers); + const { payload } = decodeFormatPrefix(data); + data = await decrypt(key, payload); + } + if (data instanceof Uint8Array && isCompressedData(data)) { + // Decompress: strip 'gzip' prefix and inflate via the web-standard + // DecompressionStream (works in browsers, unlike the sync Node path + // inside hydrateData). The inflated bytes carry their own format + // prefix (e.g. 'devl'). + const { payload } = decodeFormatPrefix(data); + data = await gunzipAsync(payload); } - // No key or not encrypted — delegate to sync hydrateData - return hydrateData(value, revivers); + // Delegate the (decrypted/decompressed) result to sync hydrateData + return hydrateData(data, revivers); } // --------------------------------------------------------------------------- diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index f74be5b099..7bb46af84c 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -26,6 +26,7 @@ import { getStepFunction } from './private.js'; // `docs/content/docs/changelog/eager-processing.mdx`. import { getWorldLazy } from './runtime/get-world-lazy.js'; import * as clientModule from './serialization/client.js'; +import { compress, decompress } from './serialization/compression.js'; import { decrypt, type EncryptionKeyParam, @@ -92,6 +93,8 @@ export { isEncrypted, encrypt, decrypt, + compress, + decompress, type EncryptionKeyParam, }; @@ -2444,7 +2447,8 @@ export async function dehydrateWorkflowArguments( ops: Promise[] = [], global: Record = globalThis, v1Compat = false, - framedByteStreams = false + framedByteStreams = false, + compression = false ): Promise { if (v1Compat) { const str = stringify( @@ -2459,6 +2463,7 @@ export async function dehydrateWorkflowArguments( extraReducers: getStreamAndRequestReducers( getExternalReducers(global, ops, runId, key, framedByteStreams) ), + compression, }); } catch (error) { const cause = unwrapSerializationCause(error); @@ -2481,13 +2486,16 @@ export async function hydrateWorkflowArguments( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - return workflowModule.deserialize(await maybeDecrypt(value, key), { - global, - extraRevivers: { - ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), - ...extraRevivers, - }, - }); + return workflowModule.deserialize( + await decompress(await maybeDecrypt(value, key)), + { + global, + extraRevivers: { + ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), + ...extraRevivers, + }, + } + ); } /** @@ -2498,7 +2506,8 @@ export async function dehydrateWorkflowReturnValue( _runId: string, key: CryptoKey | undefined, global: Record = globalThis, - v1Compat = false + v1Compat = false, + compression = false ): Promise { if (v1Compat) { const str = stringify(value, getWorkflowReducers(global)); @@ -2508,6 +2517,7 @@ export async function dehydrateWorkflowReturnValue( return await stepModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers(getWorkflowReducers(global)), + compression, }); } catch (error) { const cause = unwrapSerializationCause(error); @@ -2551,7 +2561,8 @@ export async function dehydrateStepArguments( _runId: string, key: CryptoKey | undefined, global: Record = globalThis, - v1Compat = false + v1Compat = false, + compression = false ): Promise { if (v1Compat) { const str = stringify(value, getWorkflowReducers(global)); @@ -2561,6 +2572,7 @@ export async function dehydrateStepArguments( return await stepModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers(getWorkflowReducers(global)), + compression, }); } catch (error) { const cause = unwrapSerializationCause(error); @@ -2617,7 +2629,8 @@ export async function dehydrateStepReturnValue( ops: Promise[] = [], global: Record = globalThis, v1Compat = false, - framedByteStreams = false + framedByteStreams = false, + compression = false ): Promise { if (v1Compat) { const str = stringify( @@ -2632,6 +2645,7 @@ export async function dehydrateStepReturnValue( extraReducers: getStreamAndRequestReducers( getStepReducers(global, ops, runId, key, framedByteStreams) ), + compression, }); } catch (error) { const cause = unwrapSerializationCause(error); @@ -2664,7 +2678,8 @@ export async function dehydrateStepError( runId: string, key: CryptoKey | undefined, ops: Promise[] = [], - global: Record = globalThis + global: Record = globalThis, + compression = false ): Promise { try { const str = stringify(value, getStepReducers(global, ops, runId, key)); @@ -2673,7 +2688,9 @@ export async function dehydrateStepError( SerializationFormat.DEVALUE_V1, payload ) as Uint8Array; - return (await maybeEncrypt(serialized, key)) as Uint8Array; + // Compress before encrypting — encrypted bytes don't compress. + const compressed = await compress(serialized, compression); + return (await maybeEncrypt(compressed as Uint8Array, key)) as Uint8Array; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError('step error', cause); @@ -2700,7 +2717,7 @@ export async function hydrateStepError( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - const decrypted = await maybeDecrypt(value, key); + const decrypted = await decompress(await maybeDecrypt(value, key)); if (!(decrypted instanceof Uint8Array)) { // Treated as a devalue "flattened" array. In production this branch is @@ -2746,7 +2763,8 @@ export async function dehydrateRunError( value: unknown, _runId: string, key: CryptoKey | undefined, - global: Record = globalThis + global: Record = globalThis, + compression = false ): Promise { try { const str = stringify(value, getWorkflowReducers(global)); @@ -2755,7 +2773,9 @@ export async function dehydrateRunError( SerializationFormat.DEVALUE_V1, payload ) as Uint8Array; - return (await maybeEncrypt(serialized, key)) as Uint8Array; + // Compress before encrypting — encrypted bytes don't compress. + const compressed = await compress(serialized, compression); + return (await maybeEncrypt(compressed as Uint8Array, key)) as Uint8Array; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError('run error', cause); @@ -2783,7 +2803,7 @@ export async function hydrateRunError( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - const decrypted = await maybeDecrypt(value, key); + const decrypted = await decompress(await maybeDecrypt(value, key)); if (!(decrypted instanceof Uint8Array)) { // See the matching note in `hydrateStepError`: this branch is for @@ -2829,13 +2849,16 @@ export async function hydrateStepReturnValue( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - return workflowModule.deserialize(await maybeDecrypt(value, key), { - global, - extraRevivers: { - ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), - ...extraRevivers, - }, - }); + return workflowModule.deserialize( + await decompress(await maybeDecrypt(value, key)), + { + global, + extraRevivers: { + ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), + ...extraRevivers, + }, + } + ); } // ---- Helpers to extract stream/Request/Response reducers and revivers ---- diff --git a/packages/core/src/serialization/client.ts b/packages/core/src/serialization/client.ts index 242fd6ff7b..97366422c9 100644 --- a/packages/core/src/serialization/client.ts +++ b/packages/core/src/serialization/client.ts @@ -8,6 +8,7 @@ import { SerializationError } from '@workflow/errors'; import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; +import { compress, decompress } from './compression.js'; import { type CryptoKey, decrypt as decryptData, @@ -31,7 +32,9 @@ export async function serialize( SerializationFormat.DEVALUE_V1, payload ) as Uint8Array; - return encryptData(prefixed, encryptionKey); + // Compress before encrypting — encrypted bytes don't compress. + const compressed = await compress(prefixed, options?.compression === true); + return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); const { message, hint } = formatSerializationError('client value', error); @@ -47,7 +50,7 @@ export async function deserialize( encryptionKey?: CryptoKey, options?: CodecOptions ): Promise { - const decrypted = await decryptData(data, encryptionKey); + const decrypted = await decompress(await decryptData(data, encryptionKey)); if (!(decrypted instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { diff --git a/packages/core/src/serialization/codec.ts b/packages/core/src/serialization/codec.ts index 6dd9ebe896..62ee88861a 100644 --- a/packages/core/src/serialization/codec.ts +++ b/packages/core/src/serialization/codec.ts @@ -54,6 +54,16 @@ export interface CodecOptions { * or other mode-specific type revivers. */ extraRevivers?: Record any>; + + /** + * Whether to gzip-compress the serialized payload (write side only; + * reads always handle both compressed and uncompressed data). Must + * only be enabled when the target run supports compressed payloads: + * run specVersion >= SPEC_VERSION_SUPPORTS_COMPRESSION, and for + * cross-deployment writes the target deployment's capabilities (see + * `getRunCapabilities` in capabilities.ts). Defaults to `false`. + */ + compression?: boolean; } export interface Codec { diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts new file mode 100644 index 0000000000..2328cecb09 --- /dev/null +++ b/packages/core/src/serialization/compression.test.ts @@ -0,0 +1,298 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { getRunCapabilities } from '../capabilities.js'; +import { importKey } from '../encryption.js'; +import { + dehydrateStepError, + hydrateStepError, + hydrateStepReturnValue, +} from '../serialization.js'; +import { + hydrateData, + hydrateDataWithKey, + isCompressedData, +} from '../serialization-format.js'; +import * as clientModule from './client.js'; +import { + COMPRESSION_MIN_BYTES, + compress, + decompress, + isCompressed, +} from './compression.js'; +import { decrypt } from './encryption.js'; +import { + decodeFormatPrefix, + encodeWithFormatPrefix, + peekFormatPrefix, +} from './format.js'; +import * as stepModule from './step.js'; +import { SerializationFormat } from './types.js'; + +const textEncoder = new TextEncoder(); + +/** A large, highly compressible value (repetitive JSON-ish content). */ +function makeCompressibleValue(items = 200) { + return { + users: Array.from({ length: items }, (_, i) => ({ + id: `user_${i}`, + name: `Test User Number ${i}`, + email: `test.user.${i}@example.com`, + role: i % 3 === 0 ? 'admin' : 'member', + createdAt: '2026-01-15T10:30:00.000Z', + preferences: { theme: 'dark', locale: 'en-US', notifications: true }, + })), + }; +} + +function makeKey() { + return importKey(crypto.getRandomValues(new Uint8Array(32))); +} + +describe('compression layer (compress/decompress)', () => { + it('round-trips a compressible payload through compress/decompress', async () => { + const original = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(JSON.stringify(makeCompressibleValue())) + ) as Uint8Array; + + const compressed = await compress(original, true); + expect(compressed).toBeInstanceOf(Uint8Array); + expect(isCompressed(compressed)).toBe(true); + expect((compressed as Uint8Array).length).toBeLessThan(original.length); + + const decompressed = (await decompress(compressed)) as Uint8Array; + expect(decompressed).toEqual(original); + }); + + it('passes small payloads through unchanged', async () => { + const small = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode('"hello"') + ) as Uint8Array; + expect(small.length).toBeLessThan(COMPRESSION_MIN_BYTES); + + const result = await compress(small, true); + expect(result).toBe(small); + expect(isCompressed(result)).toBe(false); + }); + + it('keeps the original when compression does not help (incompressible data)', async () => { + // Random bytes are incompressible — gzip output would be larger. + const random = crypto.getRandomValues(new Uint8Array(4096)); + const prefixed = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + random + ) as Uint8Array; + + const result = await compress(prefixed, true); + expect(result).toBe(prefixed); + expect(isCompressed(result)).toBe(false); + }); + + it('does not compress when disabled', async () => { + const original = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(JSON.stringify(makeCompressibleValue())) + ) as Uint8Array; + + const result = await compress(original, false); + expect(result).toBe(original); + }); + + it('decompress passes non-compressed and non-binary data through', async () => { + const plain = textEncoder.encode('devl"hello"'); + expect(await decompress(plain)).toBe(plain); + const legacy = [1, 2, 3]; + expect(await decompress(legacy)).toBe(legacy); + }); +}); + +describe('WORKFLOW_DISABLE_COMPRESSION kill switch', () => { + afterEach(() => { + delete process.env.WORKFLOW_DISABLE_COMPRESSION; + }); + + it('disables write-side compression', async () => { + process.env.WORKFLOW_DISABLE_COMPRESSION = '1'; + const original = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(JSON.stringify(makeCompressibleValue())) + ) as Uint8Array; + + const result = await compress(original, true); + expect(result).toBe(original); + }); + + it('does not affect reads of already-compressed data', async () => { + const original = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(JSON.stringify(makeCompressibleValue())) + ) as Uint8Array; + const compressed = await compress(original, true); + expect(isCompressed(compressed)).toBe(true); + + process.env.WORKFLOW_DISABLE_COMPRESSION = '1'; + const decompressed = (await decompress(compressed)) as Uint8Array; + expect(decompressed).toEqual(original); + }); +}); + +describe('mode serializers with compression', () => { + it('step serialize/deserialize round-trips with compression enabled', async () => { + const value = makeCompressibleValue(); + + const compressed = await stepModule.serialize(value, undefined, { + compression: true, + }); + expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.GZIP); + + const uncompressed = await stepModule.serialize(value, undefined, {}); + expect(peekFormatPrefix(uncompressed)).toBe(SerializationFormat.DEVALUE_V1); + expect((compressed as Uint8Array).length).toBeLessThan( + (uncompressed as Uint8Array).length + ); + + const result = await stepModule.deserialize(compressed, undefined, {}); + expect(result).toEqual(value); + }); + + it('client serialize/deserialize round-trips with compression enabled', async () => { + const value = makeCompressibleValue(); + const data = await clientModule.serialize(value, undefined, { + compression: true, + }); + expect(peekFormatPrefix(data)).toBe(SerializationFormat.GZIP); + const result = await clientModule.deserialize(data, undefined, {}); + expect(result).toEqual(value); + }); + + it('nests compression inside encryption: encr(gzip(devl))', async () => { + const key = await makeKey(); + const value = makeCompressibleValue(); + + const data = await stepModule.serialize(value, key, { + compression: true, + }); + // Outer layer must be encryption (encrypted bytes don't compress) + expect(peekFormatPrefix(data)).toBe(SerializationFormat.ENCRYPTED); + + // White-box: the decrypted inner payload carries the gzip prefix + const inner = await decrypt(data, key); + expect(peekFormatPrefix(inner)).toBe(SerializationFormat.GZIP); + const { payload: deflated } = decodeFormatPrefix(inner); + expect(deflated.length).toBeGreaterThan(0); + + // Full round-trip through the public API + const result = await stepModule.deserialize(data, key, {}); + expect(result).toEqual(value); + }); + + it('deserializes uncompressed data written without compression (backwards compat)', async () => { + const value = makeCompressibleValue(); + const data = await stepModule.serialize(value, undefined, {}); + expect(peekFormatPrefix(data)).toBe(SerializationFormat.DEVALUE_V1); + const result = await stepModule.deserialize(data, undefined, {}); + expect(result).toEqual(value); + }); + + it('hydrateStepReturnValue (workflow replay path) decompresses step outputs', async () => { + const value = makeCompressibleValue(); + const data = await stepModule.serialize(value, undefined, { + compression: true, + }); + const result = await hydrateStepReturnValue(data, 'wrun_test', undefined); + expect(result).toEqual(value); + }); + + it('small values stay uncompressed even with compression enabled', async () => { + const data = await stepModule.serialize({ ok: true }, undefined, { + compression: true, + }); + expect(peekFormatPrefix(data)).toBe(SerializationFormat.DEVALUE_V1); + }); +}); + +describe('dehydrateStepError with compression', () => { + it('compresses large errors and round-trips through hydrateStepError', async () => { + const error = new Error('boom'); + // Inflate the stack to push the payload over the compression threshold + error.stack = `Error: boom\n${' at someVeryLongFunctionName (/app/node_modules/some-package/dist/index.js:123:45)\n'.repeat(50)}`; + + const data = await dehydrateStepError( + error, + 'wrun_test', + undefined, + [], + globalThis, + true + ); + expect(peekFormatPrefix(data)).toBe(SerializationFormat.GZIP); + + const hydrated = (await hydrateStepError( + data, + 'wrun_test', + undefined + )) as Error; + expect(hydrated).toBeInstanceOf(Error); + expect(hydrated.message).toBe('boom'); + expect(hydrated.stack).toBe(error.stack); + }); +}); + +describe('o11y hydration of compressed payloads', () => { + it('hydrateData (sync, Node) decompresses gzip payloads', async () => { + const value = makeCompressibleValue(); + const data = await stepModule.serialize(value, undefined, { + compression: true, + }); + expect(isCompressedData(data)).toBe(true); + const hydrated = hydrateData(data, {}); + expect(hydrated).toEqual(value); + }); + + it('hydrateDataWithKey decompresses encrypted + compressed payloads', async () => { + const key = await makeKey(); + const value = makeCompressibleValue(); + const data = await stepModule.serialize(value, key, { + compression: true, + }); + const hydrated = await hydrateDataWithKey(data, {}, key); + expect(hydrated).toEqual(value); + }); + + it('hydrateDataWithKey decompresses unencrypted compressed payloads', async () => { + const value = makeCompressibleValue(); + const data = await stepModule.serialize(value, undefined, { + compression: true, + }); + const hydrated = await hydrateDataWithKey(data, {}, undefined); + expect(hydrated).toEqual(value); + }); +}); + +describe('run capabilities for gzip', () => { + it('supports gzip for core versions >= 5.0.0-beta.16', () => { + expect( + getRunCapabilities('5.0.0-beta.16').supportedFormats.has( + SerializationFormat.GZIP + ) + ).toBe(true); + }); + + it('does not support gzip for older core versions', () => { + for (const version of ['5.0.0-beta.15', '4.2.1', '4.0.0']) { + expect( + getRunCapabilities(version).supportedFormats.has( + SerializationFormat.GZIP + ) + ).toBe(false); + } + }); + + it('assumes no gzip support when the version is unknown', () => { + expect( + getRunCapabilities(undefined).supportedFormats.has( + SerializationFormat.GZIP + ) + ).toBe(false); + }); +}); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts new file mode 100644 index 0000000000..c921d517b4 --- /dev/null +++ b/packages/core/src/serialization/compression.ts @@ -0,0 +1,179 @@ +/** + * Composable compression layer for serialized data. + * + * Wraps/unwraps serialized payloads with gzip compression, using the + * format prefix system to mark compressed data ('gzip' wrapping the + * inner format, e.g. 'gzip' + deflate('devl' + payload)). + * + * Layering order with encryption: compression is applied BEFORE + * encryption (encr(gzip(devl))) — encrypted bytes are high-entropy and + * do not compress, so the reverse order would be a no-op. + * + * Compression is conditional: + * - Payloads smaller than {@link COMPRESSION_MIN_BYTES} are passed + * through unchanged (gzip overhead isn't worth it). + * - If the compressed result isn't meaningfully smaller than the + * original (see {@link COMPRESSION_MIN_SAVINGS_RATIO}), the original + * is kept. This protects already-compressed binary payloads (images, + * archives, etc.) from wasted CPU and size inflation. + * + * Decompression is unconditional: any payload carrying the 'gzip' + * prefix is inflated, so readers transparently handle both compressed + * and uncompressed data regardless of write-side settings. + */ + +import { + decodeFormatPrefix, + encodeWithFormatPrefix, + peekFormatPrefix, +} from './format.js'; +import { SerializationFormat } from './types.js'; + +/** + * Payloads below this size are never compressed. The 4-byte format + * prefix + ~20 bytes of gzip header/trailer overhead means small + * payloads gain nothing, and tiny ones would grow. + */ +export const COMPRESSION_MIN_BYTES = 1024; + +/** + * Compression must shave off at least this fraction of the payload + * size to be kept; otherwise the uncompressed original is stored. + * Guards against incompressible (already-compressed / high-entropy) + * data paying a permanent decompression tax for a negligible win. + */ +export const COMPRESSION_MIN_SAVINGS_RATIO = 0.05; + +/** + * Escape hatch: set WORKFLOW_DISABLE_COMPRESSION=1 to disable + * write-side compression entirely. Reads are unaffected — payloads + * that were already written compressed remain readable. + */ +function isCompressionDisabledByEnv(): boolean { + try { + return ( + typeof process !== 'undefined' && + process.env?.WORKFLOW_DISABLE_COMPRESSION === '1' + ); + } catch { + return false; + } +} + +/** + * Pipe bytes through a (De)CompressionStream and collect the output. + */ +async function pipeThroughTransform( + data: Uint8Array, + transform: { + readable: ReadableStream; + writable: WritableStream; + } +): Promise { + const writer = transform.writable.getWriter(); + // Don't await the write before reading — the transform's internal + // queue can fill up on large payloads, deadlocking writer vs reader. + const writePromise = writer.write(data).then(() => writer.close()); + // If the transform errors, the reader.read() below rejects first and + // propagates; mark the write side as handled so the mirrored rejection + // doesn't surface as an unhandled rejection. + writePromise.catch(() => {}); + const chunks: Uint8Array[] = []; + let total = 0; + const reader = transform.readable.getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + total += value.length; + } + await writePromise; + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + +async function gzipBytes(data: Uint8Array): Promise { + return pipeThroughTransform(data, new CompressionStream('gzip')); +} + +async function gunzipBytes(data: Uint8Array): Promise { + return pipeThroughTransform(data, new DecompressionStream('gzip')); +} + +/** + * Whether the current runtime can compress/decompress. CompressionStream + * is a web standard available in Node.js 18+, browsers, and edge + * runtimes; this guard exists for exotic runtimes only. + */ +function isCompressionAvailable(): boolean { + return ( + typeof CompressionStream === 'function' && + typeof DecompressionStream === 'function' + ); +} + +/** + * Compress a format-prefixed payload if compression is enabled for the + * target run and the payload is worth compressing. + * + * @param data - The format-prefixed serialized data (e.g. 'devl' + bytes) + * @param enabled - Whether the target run supports compressed payloads + * (run specVersion >= SPEC_VERSION_SUPPORTS_COMPRESSION, and for + * cross-deployment writes, the target deployment's capabilities — + * see `getRunCapabilities` in capabilities.ts) + * @returns The compressed data with 'gzip' prefix, or the original data + * when compression is disabled, unavailable, or not worthwhile + */ +export async function compress( + data: Uint8Array | unknown, + enabled: boolean +): Promise { + if (!enabled || !(data instanceof Uint8Array)) return data; + if (data.length < COMPRESSION_MIN_BYTES) return data; + if (isCompressionDisabledByEnv() || !isCompressionAvailable()) return data; + + const compressed = await gzipBytes(data); + const wrappedLength = 4 + compressed.length; // format prefix + payload + if (wrappedLength >= data.length * (1 - COMPRESSION_MIN_SAVINGS_RATIO)) { + return data; + } + return encodeWithFormatPrefix(SerializationFormat.GZIP, compressed); +} + +/** + * Decompress a format-prefixed payload if it's compressed. + * Strips the 'gzip' format prefix and inflates the inner payload + * (which carries its own format prefix, e.g. 'devl'). + * + * Non-compressed data (including non-binary legacy data) is returned + * unchanged, so this is safe to apply unconditionally on read paths. + */ +export async function decompress( + data: Uint8Array | unknown +): Promise { + if (!(data instanceof Uint8Array)) return data; + if (peekFormatPrefix(data) !== SerializationFormat.GZIP) return data; + + if (!isCompressionAvailable()) { + throw new Error( + 'Compressed (gzip) workflow data encountered but DecompressionStream ' + + 'is not available in this runtime. Node.js 18+, browsers, and edge ' + + 'runtimes all support it.' + ); + } + + const { payload } = decodeFormatPrefix(data); + return gunzipBytes(payload); +} + +/** + * Check if data is compressed (has 'gzip' format prefix). + */ +export function isCompressed(data: Uint8Array | unknown): boolean { + return peekFormatPrefix(data) === SerializationFormat.GZIP; +} diff --git a/packages/core/src/serialization/index.ts b/packages/core/src/serialization/index.ts index 531913b410..40e0774f42 100644 --- a/packages/core/src/serialization/index.ts +++ b/packages/core/src/serialization/index.ts @@ -34,6 +34,14 @@ export { type EncryptionKeyParam, } from './encryption.js'; +// Re-export composable compression +export { + compress, + decompress, + isCompressed, + COMPRESSION_MIN_BYTES, +} from './compression.js'; + // Re-export mode-specific modules as namespaces import * as workflow from './workflow.js'; import * as step from './step.js'; diff --git a/packages/core/src/serialization/step.ts b/packages/core/src/serialization/step.ts index f1c29c263a..cb478729e1 100644 --- a/packages/core/src/serialization/step.ts +++ b/packages/core/src/serialization/step.ts @@ -8,6 +8,7 @@ import { SerializationError } from '@workflow/errors'; import type { CodecOptions } from './codec.js'; import { devalueCodec } from './codec-devalue.js'; +import { compress, decompress } from './compression.js'; import { type CryptoKey, decrypt as decryptData, @@ -31,7 +32,9 @@ export async function serialize( SerializationFormat.DEVALUE_V1, payload ) as Uint8Array; - return encryptData(prefixed, encryptionKey); + // Compress before encrypting — encrypted bytes don't compress. + const compressed = await compress(prefixed, options?.compression === true); + return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); const { message, hint } = formatSerializationError('step value', error); @@ -47,7 +50,7 @@ export async function deserialize( encryptionKey?: CryptoKey, options?: CodecOptions ): Promise { - const decrypted = await decryptData(data, encryptionKey); + const decrypted = await decompress(await decryptData(data, encryptionKey)); if (!(decrypted instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index 7b63a53a69..da948bee14 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -32,6 +32,8 @@ export const SerializationFormat = { DEVALUE_V1: 'devl' as FormatPrefix, /** Encrypted payload (inner payload has its own format prefix) */ ENCRYPTED: 'encr' as FormatPrefix, + /** Gzip-compressed payload (inner payload has its own format prefix) */ + GZIP: 'gzip' as FormatPrefix, } as const; // ---- Serializable Types ---- diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index f1d37b468f..7a041015f5 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -8,6 +8,7 @@ import { import { withResolvers } from '@workflow/utils'; import { parseWorkflowName } from '@workflow/utils/parse-name'; import type { Event, WorkflowRun } from '@workflow/world'; +import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; import type { CryptoKey } from './encryption.js'; @@ -810,7 +811,12 @@ export async function runWorkflow( result, workflowRun.runId, encryptionKey, - vmGlobalThis + vmGlobalThis, + false, + // Gate payload compression on the run's specVersion: only runs + // marked as possibly containing compressed payloads (spec >= 5) + // get gzip data. + (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION ); span?.setAttributes({ diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 682cea9d90..85ae00c274 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -70,6 +70,7 @@ export { SPEC_VERSION_LEGACY, SPEC_VERSION_SUPPORTS_ATTRIBUTES, SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT, + SPEC_VERSION_SUPPORTS_COMPRESSION, SPEC_VERSION_SUPPORTS_EVENT_SOURCING, } from './spec-version.js'; export type * from './steps.js'; diff --git a/packages/world/src/spec-version.test.ts b/packages/world/src/spec-version.test.ts new file mode 100644 index 0000000000..75e607611b --- /dev/null +++ b/packages/world/src/spec-version.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { + isLegacySpecVersion, + requiresNewerWorld, + SPEC_VERSION_CURRENT, + SPEC_VERSION_LEGACY, + SPEC_VERSION_SUPPORTS_ATTRIBUTES, + SPEC_VERSION_SUPPORTS_COMPRESSION, +} from './spec-version.js'; + +describe('spec version constants', () => { + it('current spec version is the compression version', () => { + expect(SPEC_VERSION_CURRENT).toBe(SPEC_VERSION_SUPPORTS_COMPRESSION); + expect(SPEC_VERSION_SUPPORTS_COMPRESSION).toBe(5); + }); +}); + +describe('requiresNewerWorld', () => { + it('accepts runs at or below the current spec version', () => { + expect(requiresNewerWorld(SPEC_VERSION_CURRENT)).toBe(false); + expect(requiresNewerWorld(SPEC_VERSION_SUPPORTS_ATTRIBUTES)).toBe(false); + expect(requiresNewerWorld(SPEC_VERSION_LEGACY)).toBe(false); + expect(requiresNewerWorld(undefined)).toBe(false); + expect(requiresNewerWorld(null)).toBe(false); + }); + + it('rejects runs newer than the current spec version', () => { + // This is the contract that protects older SDKs from compressed + // payloads they cannot decode: a spec-5 run read by an SDK whose + // SPEC_VERSION_CURRENT is 4 fails this check up front (with + // RunNotSupportedError at the storage layer) instead of failing on + // individual gzip payloads. + expect(requiresNewerWorld(SPEC_VERSION_CURRENT + 1)).toBe(true); + }); + + it('simulates a v4 reader rejecting a compression-era run', () => { + // A v4 SDK has SPEC_VERSION_CURRENT = 4. Its requiresNewerWorld(v) + // is `v > 4`, so a spec-5 run is rejected. We can't import the old + // constant, so replicate the v4 predicate explicitly. + const v4RequiresNewerWorld = (v: number) => v > 4; + expect(v4RequiresNewerWorld(SPEC_VERSION_SUPPORTS_COMPRESSION)).toBe(true); + }); +}); + +describe('isLegacySpecVersion', () => { + it('is unaffected by the current-version bump', () => { + expect(isLegacySpecVersion(1)).toBe(true); + expect(isLegacySpecVersion(undefined)).toBe(true); + expect(isLegacySpecVersion(2)).toBe(false); + expect(isLegacySpecVersion(4)).toBe(false); + expect(isLegacySpecVersion(5)).toBe(false); + }); +}); diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts index ac2d9a6b65..78b9d531f5 100644 --- a/packages/world/src/spec-version.ts +++ b/packages/world/src/spec-version.ts @@ -24,10 +24,20 @@ export const SPEC_VERSION_LEGACY = 1 as SpecVersion; export const SPEC_VERSION_SUPPORTS_EVENT_SOURCING = 2 as SpecVersion; export const SPEC_VERSION_SUPPORTS_CBOR_QUEUE_TRANSPORT = 3 as SpecVersion; export const SPEC_VERSION_SUPPORTS_ATTRIBUTES = 4 as SpecVersion; +/** + * Runs at this spec version or later may contain gzip-compressed payloads + * ('gzip' format prefix). Readers older than this version cannot decode + * such payloads and reject the run via `requiresNewerWorld()` instead of + * failing on individual payloads. + */ +export const SPEC_VERSION_SUPPORTS_COMPRESSION = 5 as SpecVersion; -/** Current spec version (event-sourced architecture with native attributes). */ +/** + * Current spec version (event-sourced architecture with native attributes + * and compressed payloads). + */ export const SPEC_VERSION_CURRENT = - SPEC_VERSION_SUPPORTS_ATTRIBUTES as SpecVersion; + SPEC_VERSION_SUPPORTS_COMPRESSION as SpecVersion; /** * Check if a spec version is legacy (<= SPEC_VERSION_LEGACY or undefined). From 40c62df5d437766279f00fad4647b2a4482b66b4 Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Fri, 12 Jun 2026 23:33:24 -0700 Subject: [PATCH 2/5] test(core): add CPU/perf compression benchmark + shared workloads Split the compression benchmark into reproducible size and CPU scripts sharing deterministic workloads (lib/workloads.mjs). The CPU benchmark measures serialize/deserialize overhead per payload, total CPU across thousands of events, and compares gzip levels/brotli/deflate. Documents how to run the size, CPU, and end-to-end (bench.bench.ts) benchmarks against local and Vercel in scripts/README.md. Co-Authored-By: Claude Fable 5 --- packages/core/scripts/README.md | 88 ++++++++ .../scripts/benchmark-compression-cpu.mjs | 201 ++++++++++++++++++ .../scripts/benchmark-compression-size.mjs | 65 ++++++ .../workloads.mjs} | 106 ++------- 4 files changed, 377 insertions(+), 83 deletions(-) create mode 100644 packages/core/scripts/README.md create mode 100644 packages/core/scripts/benchmark-compression-cpu.mjs create mode 100644 packages/core/scripts/benchmark-compression-size.mjs rename packages/core/scripts/{benchmark-compression.mjs => lib/workloads.mjs} (64%) diff --git a/packages/core/scripts/README.md b/packages/core/scripts/README.md new file mode 100644 index 0000000000..0a39c41b57 --- /dev/null +++ b/packages/core/scripts/README.md @@ -0,0 +1,88 @@ +# Compression benchmarks + +Reproducible benchmarks for the gzip payload compression feature +(specVersion 5, PR adding the `gzip` serialization format prefix). Two +dimensions are measured: **storage size** (bytes saved) and **CPU cost** +(time added to serialize/deserialize). All workloads are shared and +deterministic — see `lib/workloads.mjs`. + +Build `@workflow/core` first so the scripts can import the compiled +serialization layer: + +```bash +pnpm --filter @workflow/core build +cd packages/core +``` + +## 1. Storage size + +```bash +node scripts/benchmark-compression-size.mjs +``` + +Prints the exact bytes the serialization layer hands to the World storage +backends (S3/DynamoDB refs for vercel, `bytea` columns for postgres, JSON +files for local), compression off vs on, per workload, plus a simulated +10-step AI-agent event-log total. Backends that base64-encode binary +(DynamoDB inline refs, world-local JSON) see ~33% larger absolute savings +than the raw numbers. + +## 2. CPU cost + +```bash +node scripts/benchmark-compression-cpu.mjs +``` + +Three sections: + +1. **Per-payload serialize + deserialize cost** through the real shipping + path (`step.serialize` / `step.deserialize`, which use the Web + `CompressionStream('gzip')`), off vs on, with throughput. +2. **Stress** — total serialization CPU to write + replay-read thousands + of event payloads, modelling a long workflow. +3. **Algorithm comparison** (`node:zlib` sync) — gzip levels 1/6/9, + brotli, deflate-raw — informational, to compare candidate codecs for a + future format prefix (e.g. a `zsd1` zstd codec). Not the shipping path. + +Compression is a **world-independent CPU cost** added to the +serialize/deserialize path. The world only changes the *baseline* you +compare against: local (filesystem) is the fastest baseline so the +relative impact is largest there; Vercel (network + AES encryption + S3) +has the slowest baseline so the relative impact is smallest. The absolute +microbenchmark numbers hold for every backend. + +## 3. End-to-end runtime (local + vercel) + +The end-to-end harness already in the repo drives the stress workflows in +`workbench/example/workflows/97_bench.ts` through a real World and records +per-run `executionTimeMs` (`completedAt − createdAt`) to +`bench-timings--.json`: + +```bash +# Local world (nextjs-turbopack dev server on :3000) +cd workbench/nextjs-turbopack && WORKFLOW_PUBLIC_MANIFEST=1 pnpm dev & +pnpm bench:local # from repo root + +# Full suite incl. 1000-step / 1000-concurrent / 500×10KB cases +BENCHMARK_FULL_SUITE=true pnpm bench:local +``` + +To measure the compression delta, run the harness twice and diff the +output JSON: once normally (compression on, specVersion 5) and once with +`WORKFLOW_DISABLE_COMPRESSION=1` set on **both** the dev server and the +bench runner (compression off, everything else identical): + +```bash +# compression OFF baseline +WORKFLOW_DISABLE_COMPRESSION=1 pnpm dev & # in the workbench +WORKFLOW_DISABLE_COMPRESSION=1 pnpm bench:local # from repo root +mv bench-timings-nextjs-turbopack-local.json bench-timings-...-off.json +``` + +For **Vercel**, the same harness targets a deployment when the Vercel env +vars from `CLAUDE.md` are set (`WORKFLOW_VERCEL_ENV`, `VERCEL_DEPLOYMENT_ID`, +`WORKFLOW_VERCEL_AUTH_TOKEN`, `WORKFLOW_VERCEL_PROJECT`, `VERCEL_OIDC_TOKEN`, +etc.); it then writes `bench-timings--vercel.json`. The +`WORKFLOW_DISABLE_COMPRESSION=1` kill switch must be set on the deployment +(an env var on the Vercel project) for the off baseline, since compression +runs server-side in the step/workflow handlers there. diff --git a/packages/core/scripts/benchmark-compression-cpu.mjs b/packages/core/scripts/benchmark-compression-cpu.mjs new file mode 100644 index 0000000000..32b00448a1 --- /dev/null +++ b/packages/core/scripts/benchmark-compression-cpu.mjs @@ -0,0 +1,201 @@ +// Benchmark: CPU cost of gzip compression in the serialization layer. +// +// Compression is a pure client-side CPU cost added to the serialize +// (write) and deserialize (read) paths — it is WORLD-INDEPENDENT. The +// world (local / vercel / postgres) only changes the *baseline* you +// compare against (filesystem vs network+encryption+S3), so the absolute +// numbers here hold regardless of backend, and the *relative* impact is +// largest on the local world (fast baseline) and smallest on Vercel +// (network + encryption dominate). See scripts/README.md. +// +// Usage (from packages/core, after `pnpm build`): +// node scripts/benchmark-compression-cpu.mjs +// +// Three sections: +// 1. Per-payload serialize + deserialize cost (the real shipping path, +// Web CompressionStream('gzip')), off vs on. +// 2. Stress: total CPU to (de)serialize thousands of event payloads, +// modelling a long workflow + replay. +// 3. Algorithm comparison (node:zlib sync APIs) — informational, to +// compare gzip levels / brotli / deflate for future codecs. + +import zlib from 'node:zlib'; +import * as step from '../dist/serialization/step.js'; +import { ecommerceOrder, WORKLOADS } from './lib/workloads.mjs'; + +const encoder = new TextEncoder(); + +function rawBytes(value) { + if (value instanceof Uint8Array) return value.byteLength; + return encoder.encode(JSON.stringify(value)).byteLength; +} + +/** Time an async fn: warm up, then run until both minIters and minMs met. */ +async function timeAsync( + fn, + { warmup = 30, minIters = 50, minMs = 1500 } = {} +) { + for (let i = 0; i < warmup; i++) await fn(); + let iters = 0; + const start = performance.now(); + let elapsed = 0; + do { + await fn(); + iters++; + elapsed = performance.now() - start; + } while (iters < minIters || elapsed < minMs); + return { usPerOp: (elapsed * 1000) / iters, iters }; +} + +/** Time a sync fn the same way. */ +function timeSync(fn, { warmup = 30, minIters = 50, minMs = 1000 } = {}) { + for (let i = 0; i < warmup; i++) fn(); + let iters = 0; + const start = performance.now(); + let elapsed = 0; + do { + fn(); + iters++; + elapsed = performance.now() - start; + } while (iters < minIters || elapsed < minMs); + return { usPerOp: (elapsed * 1000) / iters, iters }; +} + +const mbPerSec = (bytes, usPerOp) => bytes / (usPerOp / 1e6) / (1024 * 1024); +const pct = (a, b) => `${(((a - b) / b) * 100).toFixed(1)}%`; + +// --------------------------------------------------------------------------- +// 1. Per-payload serialize + deserialize cost (real shipping path) +// --------------------------------------------------------------------------- + +console.log('## Serialize + deserialize CPU cost (Web CompressionStream gzip)'); +console.log(''); +console.log( + '| Workload | ser off | ser on | ser Δ | deser off | deser on | deser Δ | compress MB/s |' +); +console.log('| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'); + +for (const [name, value] of WORKLOADS) { + const bytes = rawBytes(value); + const serOff = await timeAsync(() => step.serialize(value, undefined, {})); + const serOn = await timeAsync(() => + step.serialize(value, undefined, { compression: true }) + ); + + const uncompressed = await step.serialize(value, undefined, {}); + const compressed = await step.serialize(value, undefined, { + compression: true, + }); + const deserOff = await timeAsync(() => + step.deserialize(uncompressed, undefined, {}) + ); + const deserOn = await timeAsync(() => + step.deserialize(compressed, undefined, {}) + ); + + console.log( + `| ${name} | ${serOff.usPerOp.toFixed(1)}µs | ${serOn.usPerOp.toFixed(1)}µs | +${pct(serOn.usPerOp, serOff.usPerOp)} | ${deserOff.usPerOp.toFixed(1)}µs | ${deserOn.usPerOp.toFixed(1)}µs | +${pct(deserOn.usPerOp, deserOff.usPerOp)} | ${mbPerSec(bytes, serOn.usPerOp - serOff.usPerOp).toFixed(0)} |` + ); +} + +// --------------------------------------------------------------------------- +// 2. Stress: thousands of event payloads (long workflow + replay) +// --------------------------------------------------------------------------- +// +// A long workflow writes each step payload once and re-reads (replays) +// every prior payload on each cold start. We model the serialization CPU +// for N events: N serializes + N deserializes, off vs on. The "added" +// number is the total extra CPU compression spends across the whole run. + +console.log(''); +console.log('## Stress: total serialization CPU for N event payloads'); +console.log(''); +console.log( + '(e-commerce order payload, ~6.6 KB each — representative step output)' +); +console.log(''); +console.log( + '| Events | off (ser+deser) | on (ser+deser) | added CPU | per event |' +); +console.log('| ---: | ---: | ---: | ---: | ---: |'); + +const stressValue = ecommerceOrder(); +const stressUncompressed = await step.serialize(stressValue, undefined, {}); +const stressCompressed = await step.serialize(stressValue, undefined, { + compression: true, +}); + +for (const n of [1000, 5000, 10000]) { + // Time one ser+deser cycle for each mode, then multiply by N. Timing + // each cycle (rather than looping N inline) keeps GC pressure realistic + // and the per-op cost stable. + const offCycle = await timeAsync(async () => { + const s = await step.serialize(stressValue, undefined, {}); + await step.deserialize(s, undefined, {}); + }); + const onCycle = await timeAsync(async () => { + const s = await step.serialize(stressValue, undefined, { + compression: true, + }); + await step.deserialize(s, undefined, {}); + }); + const offTotalMs = (offCycle.usPerOp * n) / 1000; + const onTotalMs = (onCycle.usPerOp * n) / 1000; + console.log( + `| ${n.toLocaleString()} | ${offTotalMs.toFixed(0)}ms | ${onTotalMs.toFixed(0)}ms | +${(onTotalMs - offTotalMs).toFixed(0)}ms | +${(onCycle.usPerOp - offCycle.usPerOp).toFixed(1)}µs |` + ); +} +void stressUncompressed; +void stressCompressed; + +// --------------------------------------------------------------------------- +// 3. Algorithm comparison (node:zlib sync) — informational +// --------------------------------------------------------------------------- +// +// Production uses Web CompressionStream('gzip') ≈ zlib gzip level 6. These +// sync numbers let us compare candidate codecs for a future format prefix +// (e.g. a `zsd1` zstd codec) without committing to one. Measured on the +// devalue-serialized payload bytes (what the layer actually compresses). + +console.log(''); +console.log('## Algorithm comparison (node:zlib sync, informational)'); +console.log(''); + +const ALGOS = [ + ['gzip -1', (b) => zlib.gzipSync(b, { level: 1 }), zlib.gunzipSync], + ['gzip -6 (default)', (b) => zlib.gzipSync(b, { level: 6 }), zlib.gunzipSync], + ['gzip -9', (b) => zlib.gzipSync(b, { level: 9 }), zlib.gunzipSync], + [ + 'brotli -q5', + (b) => + zlib.brotliCompressSync(b, { + params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 5 }, + }), + zlib.brotliDecompressSync, + ], + ['deflate-raw', (b) => zlib.deflateRawSync(b), zlib.inflateRawSync], +]; + +// Use the larger text/structured payloads where codec choice matters. +const ALGO_WORKLOADS = WORKLOADS.filter(([name]) => + /chat|API|document|Time series/.test(name) +); + +for (const [name, value] of ALGO_WORKLOADS) { + const input = await step.serialize(value, undefined, {}); // devl + bytes + const inputBytes = input.byteLength; + console.log(`### ${name} (${(inputBytes / 1024).toFixed(1)} KB serialized)`); + console.log(''); + console.log('| Algorithm | ratio | compress | decompress | compress MB/s |'); + console.log('| --- | ---: | ---: | ---: | ---: |'); + for (const [algo, compressFn, decompressFn] of ALGOS) { + const out = compressFn(input); + const ratio = ((1 - out.length / inputBytes) * 100).toFixed(1); + const c = timeSync(() => compressFn(input)); + const d = timeSync(() => decompressFn(out)); + console.log( + `| ${algo} | ${ratio}% | ${c.usPerOp.toFixed(1)}µs | ${d.usPerOp.toFixed(1)}µs | ${mbPerSec(inputBytes, c.usPerOp).toFixed(0)} |` + ); + } + console.log(''); +} diff --git a/packages/core/scripts/benchmark-compression-size.mjs b/packages/core/scripts/benchmark-compression-size.mjs new file mode 100644 index 0000000000..900fbcd409 --- /dev/null +++ b/packages/core/scripts/benchmark-compression-size.mjs @@ -0,0 +1,65 @@ +// Benchmark: serialized payload sizes with and without gzip compression. +// +// Measures the exact bytes the serialization layer hands to the World +// storage backends (S3/DynamoDB refs for world-vercel, bytea columns for +// world-postgres, JSON files for world-local) for a set of real-world-style +// workloads, with compression off vs on. +// +// Usage (from packages/core, after `pnpm build`): +// node scripts/benchmark-compression-size.mjs +// +// Note: backends that store binary as base64 (DynamoDB inline refs, +// world-local JSON files) amplify every byte by 4/3, so the absolute +// savings there are ~33% larger than the raw numbers below. + +import * as step from '../dist/serialization/step.js'; +import { aiChatHistory, WORKLOADS } from './lib/workloads.mjs'; + +const encoder = new TextEncoder(); + +function byteLength(data) { + if (data instanceof Uint8Array) return data.byteLength; + return encoder.encode(JSON.stringify(data)).byteLength; +} + +function fmt(n) { + if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(2)} MB`; + if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${n} B`; +} + +const rows = []; +for (const [name, value] of WORKLOADS) { + const off = await step.serialize(value, undefined, {}); + const on = await step.serialize(value, undefined, { compression: true }); + const offBytes = byteLength(off); + const onBytes = byteLength(on); + const savings = ((1 - onBytes / offBytes) * 100).toFixed(1); + rows.push({ name, offBytes, onBytes, savings }); +} + +console.log('| Workload | Uncompressed | Compressed | Savings |'); +console.log('| --- | ---: | ---: | ---: |'); +for (const { name, offBytes, onBytes, savings } of rows) { + const note = offBytes === onBytes ? ' (passthrough)' : ''; + console.log( + `| ${name} | ${fmt(offBytes)} | ${fmt(onBytes)}${note} | ${offBytes === onBytes ? '—' : `${savings}%`} |` + ); +} + +// Simulated event-log total for a representative run: an AI agent workflow +// with 10 steps that each return a growing chat history (the replay model +// re-serializes the full conversation at every step boundary). +let totalOff = 0; +let totalOn = 0; +for (let i = 1; i <= 10; i++) { + const value = aiChatHistory(6 * i); + totalOff += byteLength(await step.serialize(value, undefined, {})); + totalOn += byteLength( + await step.serialize(value, undefined, { compression: true }) + ); +} +console.log(''); +console.log( + `Simulated 10-step AI agent run (event log total): ${fmt(totalOff)} → ${fmt(totalOn)} (${((1 - totalOn / totalOff) * 100).toFixed(1)}% smaller)` +); diff --git a/packages/core/scripts/benchmark-compression.mjs b/packages/core/scripts/lib/workloads.mjs similarity index 64% rename from packages/core/scripts/benchmark-compression.mjs rename to packages/core/scripts/lib/workloads.mjs index 5395c0b416..91fc58477c 100644 --- a/packages/core/scripts/benchmark-compression.mjs +++ b/packages/core/scripts/lib/workloads.mjs @@ -1,23 +1,14 @@ -// Benchmark: serialized payload sizes with and without gzip compression. +// Shared real-world-style workloads for the compression benchmarks. // -// Measures the exact bytes the serialization layer hands to the World -// storage backends (S3/DynamoDB refs for world-vercel, bytea columns for -// world-postgres, JSON files for world-local) for a set of real-world-style -// workloads, with compression off vs on. +// Used by both `benchmark-compression-size.mjs` (storage bytes) and +// `benchmark-compression-cpu.mjs` (serialize/deserialize CPU cost) so the +// two benchmarks measure the exact same payloads and stay comparable. // -// Usage (from packages/core, after `pnpm build`): -// node scripts/benchmark-compression.mjs -// -// Note: backends that store binary as base64 (DynamoDB inline refs, -// world-local JSON files) amplify every byte by 4/3, so the absolute -// savings there are ~33% larger than the raw numbers below. - -import * as step from '../dist/serialization/step.js'; - -const encoder = new TextEncoder(); +// All generators are deterministic (seeded PRNG) so benchmark runs are +// reproducible and diffable across commits / algorithms. -// Deterministic PRNG (xorshift32) so benchmark runs are reproducible. -function makeRng(seed = 0x9e3779b9) { +// Deterministic PRNG (xorshift32). +export function makeRng(seed = 0x9e3779b9) { let state = seed; return () => { state ^= state << 13; @@ -33,7 +24,7 @@ const VOCAB = ); /** Generate plausible, non-repetitive English-ish text deterministically. */ -function makeText(rng, words) { +export function makeText(rng, words) { const out = []; let sentenceLen = 0; for (let i = 0; i < words; i++) { @@ -51,12 +42,8 @@ function makeText(rng, words) { return out.join(' '); } -// --------------------------------------------------------------------------- -// Real-world-style workloads -// --------------------------------------------------------------------------- - /** AI agent chat history — the canonical DurableAgent workload. */ -function aiChatHistory(messages = 60) { +export function aiChatHistory(messages = 60) { const rng = makeRng(0xc0ffee); const out = []; for (let i = 0; i < messages; i++) { @@ -89,7 +76,7 @@ function aiChatHistory(messages = 60) { } /** Paginated REST API response — list endpoints fetched in steps. */ -function apiUserList(count = 250) { +export function apiUserList(count = 250) { return { users: Array.from({ length: count }, (_, i) => ({ id: `usr_${i.toString(16).padStart(8, '0')}`, @@ -113,7 +100,7 @@ function apiUserList(count = 250) { } /** E-commerce order — checkout/fulfillment workflow state. */ -function ecommerceOrder(lineItems = 30) { +export function ecommerceOrder(lineItems = 30) { return { orderId: 'ord_2WqK9mPx7nL4vR8t', status: 'processing', @@ -156,7 +143,7 @@ function ecommerceOrder(lineItems = 30) { } /** Scraped/generated document text — summarization pipelines. */ -function markdownDocument(paragraphs = 40) { +export function markdownDocument(paragraphs = 40) { const rng = makeRng(0xd0c5); const parts = []; for (let i = 0; i < paragraphs; i++) { @@ -171,7 +158,7 @@ function markdownDocument(paragraphs = 40) { } /** Time-series metrics — monitoring/aggregation workloads. */ -function timeSeries(points = 2000) { +export function timeSeries(points = 2000) { const base = 1749700000000; return { metric: 'http.server.request.duration', @@ -184,13 +171,13 @@ function timeSeries(points = 2000) { } /** Tiny payload — stays below the compression threshold by design. */ -function tinyPayload() { +export function tinyPayload() { return { ok: true, id: 'wrun_01J5XYZ', count: 3 }; } /** Incompressible binary — e.g. an image/zip passed through a step. */ -function binaryPayload(bytes = 256 * 1024) { - // Deterministic pseudo-random bytes (xorshift) so runs are reproducible +export function binaryPayload(bytes = 256 * 1024) { + // Deterministic pseudo-random bytes (xorshift) so runs are reproducible. let state = 0x12345678; const data = new Uint8Array(bytes); for (let i = 0; i < bytes; i++) { @@ -202,63 +189,16 @@ function binaryPayload(bytes = 256 * 1024) { return data; } -// --------------------------------------------------------------------------- -// Measurement -// --------------------------------------------------------------------------- - -function byteLength(data) { - if (data instanceof Uint8Array) return data.byteLength; - return encoder.encode(JSON.stringify(data)).byteLength; -} - -function fmt(n) { - if (n >= 1024 * 1024) return `${(n / (1024 * 1024)).toFixed(2)} MB`; - if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`; - return `${n} B`; -} - -const WORKLOADS = [ +/** + * The standard workload set, shared by both benchmarks. Each entry is + * `[label, value]`. Order matters only for display. + */ +export const WORKLOADS = [ ['AI chat history (60 messages)', aiChatHistory()], ['API response (250 users)', apiUserList()], ['E-commerce order (30 items)', ecommerceOrder()], - ['Scraped document (~14 KB text)', markdownDocument()], + ['Scraped document (~27 KB text)', markdownDocument()], ['Time series (2000 points)', timeSeries()], ['Random binary (256 KB)', binaryPayload()], ['Tiny payload (<1 KB)', tinyPayload()], ]; - -const rows = []; -for (const [name, value] of WORKLOADS) { - const off = await step.serialize(value, undefined, {}); - const on = await step.serialize(value, undefined, { compression: true }); - const offBytes = byteLength(off); - const onBytes = byteLength(on); - const savings = ((1 - onBytes / offBytes) * 100).toFixed(1); - rows.push({ name, offBytes, onBytes, savings }); -} - -console.log('| Workload | Uncompressed | Compressed | Savings |'); -console.log('| --- | ---: | ---: | ---: |'); -for (const { name, offBytes, onBytes, savings } of rows) { - const note = offBytes === onBytes ? ' (passthrough)' : ''; - console.log( - `| ${name} | ${fmt(offBytes)} | ${fmt(onBytes)}${note} | ${offBytes === onBytes ? '—' : `${savings}%`} |` - ); -} - -// Simulated event-log total for a representative run: an AI agent workflow -// with 10 steps that each return a growing chat history (the replay model -// re-serializes the full conversation at every step boundary). -let totalOff = 0; -let totalOn = 0; -for (let i = 1; i <= 10; i++) { - const value = aiChatHistory(6 * i); - totalOff += byteLength(await step.serialize(value, undefined, {})); - totalOn += byteLength( - await step.serialize(value, undefined, { compression: true }) - ); -} -console.log(''); -console.log( - `Simulated 10-step AI agent run (event log total): ${fmt(totalOff)} → ${fmt(totalOn)} (${((1 - totalOn / totalOff) * 100).toFixed(1)}% smaller)` -); From 5f1eb1e06c5af4ec8fbb5b3af146a997b42db71e Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Sat, 13 Jun 2026 14:53:13 -0700 Subject: [PATCH 3/5] feat(world-vercel): advertise specVersion 5 to enable compression on Vercel Now that workflow-server declares spec-5 support (vercel/workflow-server#520), bump the Vercel world's advertised specVersion from 4 to 5 so new Vercel runs are stamped spec 5 and become eligible for gzip payload compression. Payloads stay opaque to the server (compression is client-side); spec 5 is a superset of spec 4, so initial run attributes still work. Co-Authored-By: Claude Fable 5 --- .changeset/gzip-ref-compression-world-vercel.md | 5 +++++ packages/world-vercel/src/index.ts | 15 +++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 .changeset/gzip-ref-compression-world-vercel.md diff --git a/.changeset/gzip-ref-compression-world-vercel.md b/.changeset/gzip-ref-compression-world-vercel.md new file mode 100644 index 0000000000..d974488940 --- /dev/null +++ b/.changeset/gzip-ref-compression-world-vercel.md @@ -0,0 +1,5 @@ +--- +'@workflow/world-vercel': minor +--- + +Advertise specVersion 5 so new Vercel runs are eligible for gzip payload compression. The workflow-server declared spec-5 support in vercel/workflow-server#520; payloads remain opaque to the server (compression is client-side). Spec 5 is a superset of spec 4, so initial run attributes still work. diff --git a/packages/world-vercel/src/index.ts b/packages/world-vercel/src/index.ts index d93a7ed574..0bdaa91ed6 100644 --- a/packages/world-vercel/src/index.ts +++ b/packages/world-vercel/src/index.ts @@ -1,5 +1,5 @@ import type { World } from '@workflow/world'; -import { SPEC_VERSION_SUPPORTS_ATTRIBUTES } from '@workflow/world'; +import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; import { createGetEncryptionKeyForRun } from './encryption.js'; import { instrumentObject } from './instrumentObject.js'; import { createQueue } from './queue.js'; @@ -25,11 +25,14 @@ export function createVercelWorld(config?: APIConfig): World { config?.projectConfig?.projectId || process.env.VERCEL_PROJECT_ID; return { - // Spec v4: the workflow-server materializes native `attr_set` events - // and accepts initial run attributes on creation. New runs are stamped - // with this version; the server must be at least this version (it - // rejects runs newer than its own SPEC_VERSION_CURRENT). - specVersion: SPEC_VERSION_SUPPORTS_ATTRIBUTES, + // Spec v5: new runs may carry gzip-compressed payloads (compression is + // entirely client-side — the workflow-server stores payloads opaquely + // via RemoteRef and never deserializes them). Spec 5 is a superset of + // spec 4, so native `attr_set` events and initial run attributes still + // work. New runs are stamped with this version; the server must support + // at least it — workflow-server declared spec-5 support in + // vercel/workflow-server#520. + specVersion: SPEC_VERSION_SUPPORTS_COMPRESSION, // On Vercel the platform fails the function invocation when the // process exits non-zero, and VQS redelivers the queue message via a // fresh invocation. The core runtime uses this to decide whether From 78a9a3e38768be52800d906a616071a2f261e972 Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Sun, 14 Jun 2026 01:02:33 -0700 Subject: [PATCH 4/5] feat(core): emit OTel span attributes for compression impact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track gzip payload compression on both the serialize (write) and deserialize (read) paths via span attributes: workflow.serialization.{operation,compressed,uncompressed_bytes, stored_bytes,compression_ratio}. Sizes are measured at the compression boundary (pre-encryption), so they reflect compression's effect rather than the at-rest size. The compression codec stays pure — compress/decompress optionally populate a CompressionStats sink, threaded through CodecOptions to the mode serializers and read by the dehydrate/hydrate wrappers, which set attributes on the active span. Telemetry failures are swallowed so they can never break the serialize/deserialize data path. Co-Authored-By: Claude Fable 5 --- .../compression-telemetry-attributes.md | 5 + packages/core/src/runtime/start.test.ts | 1 + .../core/src/runtime/step-handler.test.ts | 1 + packages/core/src/serialization.ts | 165 ++++++++++++++---- packages/core/src/serialization/client.ts | 11 +- packages/core/src/serialization/codec.ts | 8 + .../compression-telemetry.test.ts | 102 +++++++++++ .../src/serialization/compression.test.ts | 110 ++++++++++++ .../core/src/serialization/compression.ts | 70 +++++++- packages/core/src/serialization/index.ts | 52 +++--- packages/core/src/serialization/step.ts | 11 +- .../src/telemetry/semantic-conventions.ts | 32 ++++ 12 files changed, 499 insertions(+), 69 deletions(-) create mode 100644 .changeset/compression-telemetry-attributes.md create mode 100644 packages/core/src/serialization/compression-telemetry.test.ts diff --git a/.changeset/compression-telemetry-attributes.md b/.changeset/compression-telemetry-attributes.md new file mode 100644 index 0000000000..1296c77b32 --- /dev/null +++ b/.changeset/compression-telemetry-attributes.md @@ -0,0 +1,5 @@ +--- +'@workflow/core': patch +--- + +Emit OpenTelemetry span attributes for payload compression on the serialize (write) and deserialize (read) paths: `workflow.serialization.{operation,compressed,uncompressed_bytes,stored_bytes,compression_ratio}`. Sizes are measured at the compression boundary (pre-encryption). Telemetry failures never affect serialization. diff --git a/packages/core/src/runtime/start.test.ts b/packages/core/src/runtime/start.test.ts index 14b8e7675a..7c11fefae2 100644 --- a/packages/core/src/runtime/start.test.ts +++ b/packages/core/src/runtime/start.test.ts @@ -29,6 +29,7 @@ vi.mock('@vercel/functions', () => ({ vi.mock('../telemetry.js', () => ({ serializeTraceCarrier: vi.fn().mockResolvedValue({}), trace: vi.fn((_name, fn) => fn(undefined)), + getActiveSpan: vi.fn().mockResolvedValue(undefined), })); describe('start', () => { diff --git a/packages/core/src/runtime/step-handler.test.ts b/packages/core/src/runtime/step-handler.test.ts index 29caaad959..778e5fed90 100644 --- a/packages/core/src/runtime/step-handler.test.ts +++ b/packages/core/src/runtime/step-handler.test.ts @@ -97,6 +97,7 @@ vi.mock('../telemetry.js', () => ({ getSpanKind: vi.fn().mockResolvedValue(undefined), linkToCurrentContext: vi.fn().mockResolvedValue([]), withWorkflowBaggage: vi.fn((_attrs: unknown, fn: () => unknown) => fn()), + getActiveSpan: vi.fn().mockResolvedValue(undefined), })); // Mock logger diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index 7bb46af84c..a7a932b4d7 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -26,7 +26,11 @@ import { getStepFunction } from './private.js'; // `docs/content/docs/changelog/eager-processing.mdx`. import { getWorldLazy } from './runtime/get-world-lazy.js'; import * as clientModule from './serialization/client.js'; -import { compress, decompress } from './serialization/compression.js'; +import { + type CompressionStats, + compress, + decompress, +} from './serialization/compression.js'; import { decrypt, type EncryptionKeyParam, @@ -78,6 +82,8 @@ import { STREAM_TYPE_SYMBOL, WEBHOOK_RESPONSE_WRITABLE, } from './symbols.js'; +import * as Attr from './telemetry/semantic-conventions.js'; +import { getActiveSpan } from './telemetry.js'; import { getAbortStreamId } from './util.js'; import { WorkflowAbortSignal } from './workflow/abort-controller.js'; @@ -162,6 +168,43 @@ function unwrapSerializationCause(error: unknown): unknown { return error; } +/** + * Emit compression telemetry onto the active span after a (de)serialize. + * + * The compression layer populates `stats` only when it actually ran (binary + * data on a spec >= 5 path); legacy / v1Compat paths leave it unrecorded, so + * this no-ops for them and avoids the `getActiveSpan` lookup. Attributes land + * on whatever span is active — typically the dedicated `step.dehydrate` / + * `step.hydrate` span, otherwise the enclosing run/start span. + */ +async function recordCompression( + stats: CompressionStats, + operation: 'serialize' | 'deserialize' +): Promise { + if (!stats.recorded) return; + // Telemetry must never break the serialize/deserialize data path — a + // missing/failing tracer is purely an observability loss. + try { + const span = await getActiveSpan(); + if (!span) return; + const uncompressedBytes = stats.uncompressedBytes ?? 0; + const storedBytes = stats.storedBytes ?? 0; + span.setAttributes({ + ...Attr.SerializationOperation(operation), + ...Attr.SerializationCompressed(stats.compressed ?? false), + ...Attr.SerializationUncompressedBytes(uncompressedBytes), + ...Attr.SerializationStoredBytes(storedBytes), + ...(stats.compressed && uncompressedBytes > 0 + ? Attr.SerializationCompressionRatio( + 1 - storedBytes / uncompressedBytes + ) + : {}), + }); + } catch { + // ignore telemetry failures + } +} + export function getSerializeStream( reducers: Partial, cryptoKey: EncryptionKeyParam @@ -2458,13 +2501,17 @@ export async function dehydrateWorkflowArguments( return revive(str); } try { - return await clientModule.serialize(value, key, { + const compressionStats: CompressionStats = {}; + const result = await clientModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers( getExternalReducers(global, ops, runId, key, framedByteStreams) ), compression, + compressionStats, }); + await recordCompression(compressionStats, 'serialize'); + return result; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError( @@ -2486,16 +2533,19 @@ export async function hydrateWorkflowArguments( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - return workflowModule.deserialize( - await decompress(await maybeDecrypt(value, key)), - { - global, - extraRevivers: { - ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), - ...extraRevivers, - }, - } + const compressionStats: CompressionStats = {}; + const inflated = await decompress( + await maybeDecrypt(value, key), + compressionStats ); + await recordCompression(compressionStats, 'deserialize'); + return workflowModule.deserialize(inflated, { + global, + extraRevivers: { + ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), + ...extraRevivers, + }, + }); } /** @@ -2514,11 +2564,15 @@ export async function dehydrateWorkflowReturnValue( return revive(str); } try { - return await stepModule.serialize(value, key, { + const compressionStats: CompressionStats = {}; + const result = await stepModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers(getWorkflowReducers(global)), compression, + compressionStats, }); + await recordCompression(compressionStats, 'serialize'); + return result; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError( @@ -2541,7 +2595,8 @@ export async function hydrateWorkflowReturnValue( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - return clientModule.deserialize(value, key, { + const compressionStats: CompressionStats = {}; + const result = await clientModule.deserialize(value, key, { global, extraRevivers: { ...getStreamAndRequestRevivers( @@ -2549,7 +2604,10 @@ export async function hydrateWorkflowReturnValue( ), ...extraRevivers, }, + compressionStats, }); + await recordCompression(compressionStats, 'deserialize'); + return result; } /** @@ -2569,11 +2627,15 @@ export async function dehydrateStepArguments( return revive(str); } try { - return await stepModule.serialize(value, key, { + const compressionStats: CompressionStats = {}; + const result = await stepModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers(getWorkflowReducers(global)), compression, + compressionStats, }); + await recordCompression(compressionStats, 'serialize'); + return result; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError('step arguments', cause); @@ -2594,7 +2656,8 @@ export async function hydrateStepArguments( extraRevivers: Record any> = {}, deploymentId?: string ): Promise { - return stepModule.deserialize(value, key, { + const compressionStats: CompressionStats = {}; + const result = await stepModule.deserialize(value, key, { global, extraRevivers: { ...getStreamAndRequestRevivers( @@ -2602,7 +2665,10 @@ export async function hydrateStepArguments( ), ...extraRevivers, }, + compressionStats, }); + await recordCompression(compressionStats, 'deserialize'); + return result; } /** @@ -2640,13 +2706,17 @@ export async function dehydrateStepReturnValue( return revive(str); } try { - return await stepModule.serialize(value, key, { + const compressionStats: CompressionStats = {}; + const result = await stepModule.serialize(value, key, { global, extraReducers: getStreamAndRequestReducers( getStepReducers(global, ops, runId, key, framedByteStreams) ), compression, + compressionStats, }); + await recordCompression(compressionStats, 'serialize'); + return result; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError( @@ -2689,8 +2759,18 @@ export async function dehydrateStepError( payload ) as Uint8Array; // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress(serialized, compression); - return (await maybeEncrypt(compressed as Uint8Array, key)) as Uint8Array; + const compressionStats: CompressionStats = {}; + const compressed = await compress( + serialized, + compression, + compressionStats + ); + const encrypted = (await maybeEncrypt( + compressed as Uint8Array, + key + )) as Uint8Array; + await recordCompression(compressionStats, 'serialize'); + return encrypted; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError('step error', cause); @@ -2717,7 +2797,12 @@ export async function hydrateStepError( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - const decrypted = await decompress(await maybeDecrypt(value, key)); + const compressionStats: CompressionStats = {}; + const decrypted = await decompress( + await maybeDecrypt(value, key), + compressionStats + ); + await recordCompression(compressionStats, 'deserialize'); if (!(decrypted instanceof Uint8Array)) { // Treated as a devalue "flattened" array. In production this branch is @@ -2774,8 +2859,18 @@ export async function dehydrateRunError( payload ) as Uint8Array; // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress(serialized, compression); - return (await maybeEncrypt(compressed as Uint8Array, key)) as Uint8Array; + const compressionStats: CompressionStats = {}; + const compressed = await compress( + serialized, + compression, + compressionStats + ); + const encrypted = (await maybeEncrypt( + compressed as Uint8Array, + key + )) as Uint8Array; + await recordCompression(compressionStats, 'serialize'); + return encrypted; } catch (error) { const cause = unwrapSerializationCause(error); const { message, hint } = formatSerializationError('run error', cause); @@ -2803,7 +2898,12 @@ export async function hydrateRunError( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - const decrypted = await decompress(await maybeDecrypt(value, key)); + const compressionStats: CompressionStats = {}; + const decrypted = await decompress( + await maybeDecrypt(value, key), + compressionStats + ); + await recordCompression(compressionStats, 'deserialize'); if (!(decrypted instanceof Uint8Array)) { // See the matching note in `hydrateStepError`: this branch is for @@ -2849,16 +2949,19 @@ export async function hydrateStepReturnValue( global: Record = globalThis, extraRevivers: Record any> = {} ): Promise { - return workflowModule.deserialize( - await decompress(await maybeDecrypt(value, key)), - { - global, - extraRevivers: { - ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), - ...extraRevivers, - }, - } + const compressionStats: CompressionStats = {}; + const inflated = await decompress( + await maybeDecrypt(value, key), + compressionStats ); + await recordCompression(compressionStats, 'deserialize'); + return workflowModule.deserialize(inflated, { + global, + extraRevivers: { + ...getStreamAndRequestRevivers(getWorkflowRevivers(global)), + ...extraRevivers, + }, + }); } // ---- Helpers to extract stream/Request/Response reducers and revivers ---- diff --git a/packages/core/src/serialization/client.ts b/packages/core/src/serialization/client.ts index 97366422c9..cbdc7fc721 100644 --- a/packages/core/src/serialization/client.ts +++ b/packages/core/src/serialization/client.ts @@ -33,7 +33,11 @@ export async function serialize( payload ) as Uint8Array; // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress(prefixed, options?.compression === true); + const compressed = await compress( + prefixed, + options?.compression === true, + options?.compressionStats + ); return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); @@ -50,7 +54,10 @@ export async function deserialize( encryptionKey?: CryptoKey, options?: CodecOptions ): Promise { - const decrypted = await decompress(await decryptData(data, encryptionKey)); + const decrypted = await decompress( + await decryptData(data, encryptionKey), + options?.compressionStats + ); if (!(decrypted instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { diff --git a/packages/core/src/serialization/codec.ts b/packages/core/src/serialization/codec.ts index 62ee88861a..1bba221770 100644 --- a/packages/core/src/serialization/codec.ts +++ b/packages/core/src/serialization/codec.ts @@ -14,6 +14,7 @@ * plain objects). No Date, Map, Set, typed arrays, etc. */ +import type { CompressionStats } from './compression.js'; import type { FormatPrefix } from './types.js'; /** @@ -64,6 +65,13 @@ export interface CodecOptions { * `getRunCapabilities` in capabilities.ts). Defaults to `false`. */ compression?: boolean; + + /** + * Optional telemetry sink populated by the compression layer with what + * it did to the payload (whether it compressed, logical vs stored size). + * Used by the dehydrate/hydrate wrappers to emit OTel span attributes. + */ + compressionStats?: CompressionStats; } export interface Codec { diff --git a/packages/core/src/serialization/compression-telemetry.test.ts b/packages/core/src/serialization/compression-telemetry.test.ts new file mode 100644 index 0000000000..607fab8fb9 --- /dev/null +++ b/packages/core/src/serialization/compression-telemetry.test.ts @@ -0,0 +1,102 @@ +/** + * Verifies that the dehydrate/hydrate wrappers emit compression telemetry + * onto the active span. The telemetry module is mocked so `getActiveSpan` + * returns a fake span whose `setAttributes` calls are captured. + */ +import { describe, expect, it, vi } from 'vitest'; + +const { recordedAttributes } = vi.hoisted(() => ({ + recordedAttributes: [] as Array>, +})); + +vi.mock('../telemetry.js', () => ({ + getActiveSpan: vi.fn(async () => ({ + setAttributes: (attrs: Record) => { + recordedAttributes.push(attrs); + }, + })), +})); + +import { + dehydrateStepReturnValue, + hydrateStepReturnValue, +} from '../serialization.js'; + +function makeCompressibleValue(items = 200) { + return { + users: Array.from({ length: items }, (_, i) => ({ + id: `user_${i}`, + name: `Test User Number ${i}`, + email: `test.user.${i}@example.com`, + role: i % 3 === 0 ? 'admin' : 'member', + })), + }; +} + +/** The most recently recorded span attributes; throws if none were set. */ +function lastAttrs(): Record { + const attrs = recordedAttributes[recordedAttributes.length - 1]; + if (!attrs) throw new Error('expected compression attributes to be recorded'); + return attrs; +} + +describe('compression telemetry attributes', () => { + it('records write-path attributes with savings when compressed', async () => { + recordedAttributes.length = 0; + const value = makeCompressibleValue(); + + const data = await dehydrateStepReturnValue( + value, + 'wrun_test', + undefined, + [], + globalThis, + false, + false, + true // compression enabled + ); + + const attrs = lastAttrs(); + expect(attrs['workflow.serialization.operation']).toBe('serialize'); + expect(attrs['workflow.serialization.compressed']).toBe(true); + expect(attrs['workflow.serialization.uncompressed_bytes']).toBeGreaterThan( + attrs['workflow.serialization.stored_bytes'] as number + ); + const ratio = attrs['workflow.serialization.compression_ratio'] as number; + expect(ratio).toBeGreaterThan(0); + expect(ratio).toBeLessThan(1); + + // Read path records the inflate with the same logical size. + recordedAttributes.length = 0; + const result = await hydrateStepReturnValue(data, 'wrun_test', undefined); + expect(result).toEqual(value); + const readAttrs = lastAttrs(); + expect(readAttrs['workflow.serialization.operation']).toBe('deserialize'); + expect(readAttrs['workflow.serialization.compressed']).toBe(true); + expect(readAttrs['workflow.serialization.uncompressed_bytes']).toBe( + attrs['workflow.serialization.uncompressed_bytes'] + ); + }); + + it('records compressed=false (no ratio) when compression is disabled', async () => { + recordedAttributes.length = 0; + await dehydrateStepReturnValue( + makeCompressibleValue(), + 'wrun_test', + undefined, + [], + globalThis, + false, + false, + false // compression disabled + ); + + const attrs = lastAttrs(); + expect(attrs['workflow.serialization.operation']).toBe('serialize'); + expect(attrs['workflow.serialization.compressed']).toBe(false); + expect(attrs['workflow.serialization.compression_ratio']).toBeUndefined(); + expect(attrs['workflow.serialization.stored_bytes']).toBe( + attrs['workflow.serialization.uncompressed_bytes'] + ); + }); +}); diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts index 2328cecb09..83bef6aa45 100644 --- a/packages/core/src/serialization/compression.test.ts +++ b/packages/core/src/serialization/compression.test.ts @@ -14,6 +14,7 @@ import { import * as clientModule from './client.js'; import { COMPRESSION_MIN_BYTES, + type CompressionStats, compress, decompress, isCompressed, @@ -106,6 +107,115 @@ describe('compression layer (compress/decompress)', () => { }); }); +describe('CompressionStats telemetry sink', () => { + function devlBytes(json: string): Uint8Array { + return encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(json) + ) as Uint8Array; + } + + it('records a kept compression on the write path', async () => { + const original = devlBytes(JSON.stringify(makeCompressibleValue())); + const stats: CompressionStats = {}; + const compressed = await compress(original, true, stats); + + expect(stats.recorded).toBe(true); + expect(stats.compressed).toBe(true); + expect(stats.uncompressedBytes).toBe(original.length); + expect(stats.storedBytes).toBe((compressed as Uint8Array).length); + expect(stats.storedBytes!).toBeLessThan(stats.uncompressedBytes!); + }); + + it('records compressed=false for incompressible data (kept original)', async () => { + const random = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + crypto.getRandomValues(new Uint8Array(4096)) + ) as Uint8Array; + const stats: CompressionStats = {}; + const result = await compress(random, true, stats); + + expect(result).toBe(random); + expect(stats.recorded).toBe(true); + expect(stats.compressed).toBe(false); + expect(stats.uncompressedBytes).toBe(random.length); + expect(stats.storedBytes).toBe(random.length); + }); + + it('records compressed=false for below-threshold payloads', async () => { + const small = devlBytes('"hi"'); + expect(small.length).toBeLessThan(COMPRESSION_MIN_BYTES); + const stats: CompressionStats = {}; + await compress(small, true, stats); + + expect(stats.recorded).toBe(true); + expect(stats.compressed).toBe(false); + expect(stats.uncompressedBytes).toBe(small.length); + expect(stats.storedBytes).toBe(small.length); + }); + + it('records the uncompressed baseline even when compression is disabled', async () => { + const original = devlBytes(JSON.stringify(makeCompressibleValue())); + const stats: CompressionStats = {}; + await compress(original, false, stats); + + expect(stats.recorded).toBe(true); + expect(stats.compressed).toBe(false); + expect(stats.uncompressedBytes).toBe(original.length); + expect(stats.storedBytes).toBe(original.length); + }); + + it('does not record for non-binary (legacy) data', async () => { + const stats: CompressionStats = {}; + await compress({ not: 'binary' }, true, stats); + expect(stats.recorded).toBeFalsy(); + }); + + it('records the inflate on the read path', async () => { + const original = devlBytes(JSON.stringify(makeCompressibleValue())); + const compressed = (await compress(original, true)) as Uint8Array; + + const stats: CompressionStats = {}; + const inflated = (await decompress(compressed, stats)) as Uint8Array; + + expect(inflated).toEqual(original); + expect(stats.recorded).toBe(true); + expect(stats.compressed).toBe(true); + expect(stats.storedBytes).toBe(compressed.length); + expect(stats.uncompressedBytes).toBe(original.length); + expect(stats.storedBytes!).toBeLessThan(stats.uncompressedBytes!); + }); + + it('records compressed=false when reading uncompressed data', async () => { + const plain = devlBytes('"hello"'); + const stats: CompressionStats = {}; + await decompress(plain, stats); + + expect(stats.recorded).toBe(true); + expect(stats.compressed).toBe(false); + expect(stats.uncompressedBytes).toBe(plain.length); + expect(stats.storedBytes).toBe(plain.length); + }); + + it('round-trips stats through the step mode serializer (write + read)', async () => { + const value = makeCompressibleValue(); + const writeStats: CompressionStats = {}; + const data = await stepModule.serialize(value, undefined, { + compression: true, + compressionStats: writeStats, + }); + expect(writeStats.compressed).toBe(true); + + const readStats: CompressionStats = {}; + const result = await stepModule.deserialize(data, undefined, { + compressionStats: readStats, + }); + expect(result).toEqual(value); + expect(readStats.compressed).toBe(true); + expect(readStats.uncompressedBytes).toBe(writeStats.uncompressedBytes); + }); +}); + describe('WORKFLOW_DISABLE_COMPRESSION kill switch', () => { afterEach(() => { delete process.env.WORKFLOW_DISABLE_COMPRESSION; diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index c921d517b4..1258337f83 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -117,6 +117,44 @@ function isCompressionAvailable(): boolean { ); } +/** + * Telemetry sink describing what the compression layer did to a payload. + * Populated by {@link compress} (write) and {@link decompress} (read) when + * a `stats` object is passed. Sizes are measured at the compression + * boundary — i.e. before encryption is layered on the write side and after + * decryption on the read side — so they reflect compression's effect, not + * the at-rest size (which also includes the `encr` envelope and, on some + * backends, base64 expansion). + * + * Field meanings are identical for both directions: + * - `uncompressedBytes`: the logical (devalue-prefixed) payload size. + * - `storedBytes`: the size handed to / read from storage (compressed when + * the gzip codec applied, otherwise equal to `uncompressedBytes`). + */ +export interface CompressionStats { + /** True once the compression layer ran (i.e. saw binary data). */ + recorded?: boolean; + /** Whether the gzip codec was applied (write) or present (read). */ + compressed?: boolean; + /** Logical, uncompressed payload size in bytes. */ + uncompressedBytes?: number; + /** Stored (post-compression) payload size in bytes. */ + storedBytes?: number; +} + +function recordStats( + stats: CompressionStats | undefined, + compressed: boolean, + uncompressedBytes: number, + storedBytes: number +): void { + if (!stats) return; + stats.recorded = true; + stats.compressed = compressed; + stats.uncompressedBytes = uncompressedBytes; + stats.storedBytes = storedBytes; +} + /** * Compress a format-prefixed payload if compression is enabled for the * target run and the payload is worth compressing. @@ -126,22 +164,34 @@ function isCompressionAvailable(): boolean { * (run specVersion >= SPEC_VERSION_SUPPORTS_COMPRESSION, and for * cross-deployment writes, the target deployment's capabilities — * see `getRunCapabilities` in capabilities.ts) + * @param stats - Optional telemetry sink; populated when `data` is binary. * @returns The compressed data with 'gzip' prefix, or the original data * when compression is disabled, unavailable, or not worthwhile */ export async function compress( data: Uint8Array | unknown, - enabled: boolean + enabled: boolean, + stats?: CompressionStats ): Promise { - if (!enabled || !(data instanceof Uint8Array)) return data; - if (data.length < COMPRESSION_MIN_BYTES) return data; - if (isCompressionDisabledByEnv() || !isCompressionAvailable()) return data; + if (!(data instanceof Uint8Array)) return data; + // From here `data` is binary, so every return path records stats. + if ( + !enabled || + data.length < COMPRESSION_MIN_BYTES || + isCompressionDisabledByEnv() || + !isCompressionAvailable() + ) { + recordStats(stats, false, data.length, data.length); + return data; + } const compressed = await gzipBytes(data); const wrappedLength = 4 + compressed.length; // format prefix + payload if (wrappedLength >= data.length * (1 - COMPRESSION_MIN_SAVINGS_RATIO)) { + recordStats(stats, false, data.length, data.length); return data; } + recordStats(stats, true, data.length, wrappedLength); return encodeWithFormatPrefix(SerializationFormat.GZIP, compressed); } @@ -154,10 +204,14 @@ export async function compress( * unchanged, so this is safe to apply unconditionally on read paths. */ export async function decompress( - data: Uint8Array | unknown + data: Uint8Array | unknown, + stats?: CompressionStats ): Promise { if (!(data instanceof Uint8Array)) return data; - if (peekFormatPrefix(data) !== SerializationFormat.GZIP) return data; + if (peekFormatPrefix(data) !== SerializationFormat.GZIP) { + recordStats(stats, false, data.length, data.length); + return data; + } if (!isCompressionAvailable()) { throw new Error( @@ -168,7 +222,9 @@ export async function decompress( } const { payload } = decodeFormatPrefix(data); - return gunzipBytes(payload); + const inflated = await gunzipBytes(payload); + recordStats(stats, true, inflated.length, data.length); + return inflated; } /** diff --git a/packages/core/src/serialization/index.ts b/packages/core/src/serialization/index.ts index 40e0774f42..55acf8c014 100644 --- a/packages/core/src/serialization/index.ts +++ b/packages/core/src/serialization/index.ts @@ -5,47 +5,45 @@ * the codec/format/encryption abstractions. */ -// Re-export types -export type { - FormatPrefix, - SerializableSpecial, - Reducers, - Revivers, -} from './types.js'; -export { SerializationFormat, isFormatPrefix } from './types.js'; - // Re-export codec interface and mode type export type { Codec, SerializationMode } from './codec.js'; export { devalueCodec } from './codec-devalue.js'; - -// Re-export format prefix utilities +// Re-export composable compression export { - encodeWithFormatPrefix, - decodeFormatPrefix, - peekFormatPrefix, - isEncrypted, -} from './format.js'; - + COMPRESSION_MIN_BYTES, + type CompressionStats, + compress, + decompress, + isCompressed, +} from './compression.js'; // Re-export composable encryption export { - encrypt, - decrypt, type CryptoKey, + decrypt, type EncryptionKeyParam, + encrypt, } from './encryption.js'; -// Re-export composable compression +// Re-export format prefix utilities export { - compress, - decompress, - isCompressed, - COMPRESSION_MIN_BYTES, -} from './compression.js'; + decodeFormatPrefix, + encodeWithFormatPrefix, + isEncrypted, + peekFormatPrefix, +} from './format.js'; +// Re-export types +export type { + FormatPrefix, + Reducers, + Revivers, + SerializableSpecial, +} from './types.js'; +export { isFormatPrefix, SerializationFormat } from './types.js'; +import * as client from './client.js'; +import * as step from './step.js'; // Re-export mode-specific modules as namespaces import * as workflow from './workflow.js'; -import * as step from './step.js'; -import * as client from './client.js'; export { workflow, step, client }; // Re-export revive helper (used by legacy compat in serialization.ts) diff --git a/packages/core/src/serialization/step.ts b/packages/core/src/serialization/step.ts index cb478729e1..da34369980 100644 --- a/packages/core/src/serialization/step.ts +++ b/packages/core/src/serialization/step.ts @@ -33,7 +33,11 @@ export async function serialize( payload ) as Uint8Array; // Compress before encrypting — encrypted bytes don't compress. - const compressed = await compress(prefixed, options?.compression === true); + const compressed = await compress( + prefixed, + options?.compression === true, + options?.compressionStats + ); return encryptData(compressed, encryptionKey); } catch (error) { rethrowIfRuntimeError(error); @@ -50,7 +54,10 @@ export async function deserialize( encryptionKey?: CryptoKey, options?: CodecOptions ): Promise { - const decrypted = await decompress(await decryptData(data, encryptionKey)); + const decrypted = await decompress( + await decryptData(data, encryptionKey), + options?.compressionStats + ); if (!(decrypted instanceof Uint8Array)) { if (devalueCodec.deserializeLegacy) { diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index 58cfd9cda0..1f0d8bb5a7 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -304,6 +304,38 @@ export const QueueSerializeTimeMs = SemanticConvention( 'workflow.queue.serialize_time_ms' ); +// Payload compression attributes (gzip codec, specVersion >= 5) +// +// Sizes are measured at the compression boundary: before encryption on the +// write path and after decryption on the read path. They therefore reflect +// compression's effect, not the at-rest size (which also includes the +// ~28-byte `encr` envelope and, on some backends, base64 expansion). + +/** Whether this serialize/deserialize was a write or read. */ +export const SerializationOperation = SemanticConvention< + 'serialize' | 'deserialize' +>('workflow.serialization.operation'); + +/** Whether the gzip codec was applied (write) / present (read). */ +export const SerializationCompressed = SemanticConvention( + 'workflow.serialization.compressed' +); + +/** Logical (uncompressed, devalue-prefixed) payload size in bytes. */ +export const SerializationUncompressedBytes = SemanticConvention( + 'workflow.serialization.uncompressed_bytes' +); + +/** Stored (post-compression, pre-encryption) payload size in bytes. */ +export const SerializationStoredBytes = SemanticConvention( + 'workflow.serialization.stored_bytes' +); + +/** Fraction of bytes saved by compression (0..1); set only when compressed. */ +export const SerializationCompressionRatio = SemanticConvention( + 'workflow.serialization.compression_ratio' +); + // RPC/Peer Service attributes - For service maps and dependency tracking // See: https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ From bf6383d558f215497b6472f304d5ca34f07b65a0 Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Tue, 16 Jun 2026 11:33:06 -0700 Subject: [PATCH 5/5] feat(core,web-shared): prefer zstd compression codec (gzip fallback) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch the payload compression codec to zstd, which benchmarks 3–7× faster than gzip at an equal-or-better ratio on representative workloads (compression runs at every step boundary, so the write CPU is a per-step tax). zstd uses node:zlib (>= 22.15); gzip via the portable CompressionStream remains the fallback when zstd is unavailable, and WORKFLOW_COMPRESSION_CODEC=gzip forces it. Reads dispatch on the format prefix, so 'zstd' and 'gzip' payloads are both always decodable. zstd is Node-only (Web CompressionStream has no zstd), so the browser o11y read path registers a WASM-backed decoder (@tootallnate/zstd-wasm) via a new registerZstdDecoder hook; node:zlib handles Node-side reads (runtime replay, CLI, server o11y). A new workflow.serialization.codec span attribute reports which codec applied. gzip and zstd read support co-ship, so the existing specVersion-5 capability gate is unchanged. Verified end-to-end: spec-5 runs store zstd-prefixed payloads on disk and replay/complete correctly; the WASM decoder round-trips node:zlib zstd output. Benchmarks updated to compare zstd vs gzip. Co-Authored-By: Claude Fable 5 --- .changeset/gzip-ref-compression-core.md | 2 +- .changeset/zstd-web-decoder.md | 5 + .../scripts/benchmark-compression-cpu.mjs | 31 ++- packages/core/src/capabilities.ts | 12 +- packages/core/src/serialization-format.ts | 121 +++++++--- packages/core/src/serialization.ts | 1 + .../compression-telemetry.test.ts | 3 + .../src/serialization/compression.test.ts | 116 ++++++--- .../core/src/serialization/compression.ts | 221 +++++++++++++----- packages/core/src/serialization/index.ts | 1 + packages/core/src/serialization/types.ts | 2 + .../src/telemetry/semantic-conventions.ts | 7 +- packages/web-shared/package.json | 1 + packages/web-shared/src/lib/hydration.ts | 7 + .../src/lib/zstd-browser-decoder.ts | 42 ++++ packages/web-shared/test/zstd-decoder.test.ts | 56 +++++ pnpm-lock.yaml | 32 ++- 17 files changed, 524 insertions(+), 136 deletions(-) create mode 100644 .changeset/zstd-web-decoder.md create mode 100644 packages/web-shared/src/lib/zstd-browser-decoder.ts create mode 100644 packages/web-shared/test/zstd-decoder.test.ts diff --git a/.changeset/gzip-ref-compression-core.md b/.changeset/gzip-ref-compression-core.md index 9393bd3215..aae40c8c51 100644 --- a/.changeset/gzip-ref-compression-core.md +++ b/.changeset/gzip-ref-compression-core.md @@ -2,4 +2,4 @@ '@workflow/core': minor --- -Gzip-compress serialized payloads (step inputs/outputs, workflow arguments/return values, errors, hook payloads) before storage using a new composable `gzip` format prefix. Compression is applied before encryption, gated on run specVersion 5, and skipped for small or incompressible payloads. Set `WORKFLOW_DISABLE_COMPRESSION=1` to opt out of writes; reads always handle both formats. +Compress serialized payloads (step inputs/outputs, workflow arguments/return values, errors, hook payloads) before storage using composable codec format prefixes. zstd is the preferred codec (markedly faster than gzip at an equal-or-better ratio, via `node:zlib`); gzip (`CompressionStream`) is the portable fallback when zstd is unavailable. Reads dispatch on the prefix, so both codecs are always decodable. Compression is applied before encryption, gated on run specVersion 5, and skipped for small or incompressible payloads. `WORKFLOW_DISABLE_COMPRESSION=1` disables writes; `WORKFLOW_COMPRESSION_CODEC=gzip` forces the portable codec. diff --git a/.changeset/zstd-web-decoder.md b/.changeset/zstd-web-decoder.md new file mode 100644 index 0000000000..56a04c1493 --- /dev/null +++ b/.changeset/zstd-web-decoder.md @@ -0,0 +1,5 @@ +--- +'@workflow/web-shared': minor +--- + +Decode zstd-compressed workflow payloads in the observability UI. Since the Web `DecompressionStream` has no zstd support, the web o11y registers a WASM-backed zstd decoder (`@tootallnate/zstd-wasm`) with `@workflow/core` before hydrating payloads; the WASM is compiled lazily on first use. diff --git a/packages/core/scripts/benchmark-compression-cpu.mjs b/packages/core/scripts/benchmark-compression-cpu.mjs index 32b00448a1..ac733d8cd4 100644 --- a/packages/core/scripts/benchmark-compression-cpu.mjs +++ b/packages/core/scripts/benchmark-compression-cpu.mjs @@ -1,4 +1,4 @@ -// Benchmark: CPU cost of gzip compression in the serialization layer. +// Benchmark: CPU cost of payload compression in the serialization layer. // // Compression is a pure client-side CPU cost added to the serialize // (write) and deserialize (read) paths — it is WORLD-INDEPENDENT. The @@ -12,12 +12,13 @@ // node scripts/benchmark-compression-cpu.mjs // // Three sections: -// 1. Per-payload serialize + deserialize cost (the real shipping path, -// Web CompressionStream('gzip')), off vs on. +// 1. Per-payload serialize + deserialize cost via the real shipping +// path (the SDK's preferred codec — zstd when node:zlib has it, +// else gzip), off vs on. Force gzip with WORKFLOW_COMPRESSION_CODEC=gzip. // 2. Stress: total CPU to (de)serialize thousands of event payloads, // modelling a long workflow + replay. // 3. Algorithm comparison (node:zlib sync APIs) — informational, to -// compare gzip levels / brotli / deflate for future codecs. +// compare gzip levels / zstd levels / brotli / deflate. import zlib from 'node:zlib'; import * as step from '../dist/serialization/step.js'; @@ -68,7 +69,10 @@ const pct = (a, b) => `${(((a - b) / b) * 100).toFixed(1)}%`; // 1. Per-payload serialize + deserialize cost (real shipping path) // --------------------------------------------------------------------------- -console.log('## Serialize + deserialize CPU cost (Web CompressionStream gzip)'); +const writeCodec = process.env.WORKFLOW_COMPRESSION_CODEC || 'zstd (default)'; +console.log( + `## Serialize + deserialize CPU cost (real shipping path, codec: ${writeCodec})` +); console.log(''); console.log( '| Workload | ser off | ser on | ser Δ | deser off | deser on | deser Δ | compress MB/s |' @@ -161,10 +165,27 @@ console.log(''); console.log('## Algorithm comparison (node:zlib sync, informational)'); console.log(''); +// zstd entries are gated on availability (node:zlib >= 22.15). The +// production gzip path ships via the Web CompressionStream (≈ gzip -6); the +// node:zlib gzip rows here isolate pure codec speed from that stream +// overhead, and are the apples-to-apples comparison against zstd. +const hasZstd = typeof zlib.zstdCompressSync === 'function'; +const zstdAt = (level) => (b) => + zlib.zstdCompressSync(b, { + params: { [zlib.constants.ZSTD_c_compressionLevel]: level }, + }); + const ALGOS = [ ['gzip -1', (b) => zlib.gzipSync(b, { level: 1 }), zlib.gunzipSync], ['gzip -6 (default)', (b) => zlib.gzipSync(b, { level: 6 }), zlib.gunzipSync], ['gzip -9', (b) => zlib.gzipSync(b, { level: 9 }), zlib.gunzipSync], + ...(hasZstd + ? [ + ['zstd -3 (default)', zstdAt(3), zlib.zstdDecompressSync], + ['zstd -9', zstdAt(9), zlib.zstdDecompressSync], + ['zstd -19', zstdAt(19), zlib.zstdDecompressSync], + ] + : []), [ 'brotli -q5', (b) => diff --git a/packages/core/src/capabilities.ts b/packages/core/src/capabilities.ts index 05f5ce0f7a..a1d7844bea 100644 --- a/packages/core/src/capabilities.ts +++ b/packages/core/src/capabilities.ts @@ -29,6 +29,8 @@ * https://github.com/vercel/workflow/commit/7618ac36 * - `framedByteStreams` (wire-level chunk framing for byte streams): added in `5.0.0-beta.15` * - `gzip` (gzip payload compression): added in `5.0.0-beta.16` + * - `zstd` (zstd payload compression, preferred codec): added in `5.0.0-beta.16` + * alongside gzip — they co-ship, so any run that can read one can read both. */ import semver from 'semver'; @@ -69,12 +71,14 @@ const FORMAT_VERSION_TABLE: ReadonlyArray<{ minVersion: string; }> = [ { format: SerializationFormat.ENCRYPTED, minVersion: '4.2.0-beta.64' }, - // TODO(release): verify this matches the actual version that ships gzip - // payload compression. If a "Version Packages (beta)" PR merges before this - // change, bump to the next beta. A too-low cutoff makes new producers write + // TODO(release): verify this matches the actual version that ships payload + // compression. If a "Version Packages (beta)" PR merges before this change, + // bump to the next beta. A too-low cutoff makes new producers write // compressed payloads to consumers that cannot decompress them; too-high - // merely delays the optimization (safe). + // merely delays the optimization (safe). gzip and zstd ship together, so + // they share a min version — a run that can read one can read both. { format: SerializationFormat.GZIP, minVersion: '5.0.0-beta.16' }, + { format: SerializationFormat.ZSTD, minVersion: '5.0.0-beta.16' }, // Future entries: // { format: SerializationFormat.CBOR, minVersion: '5.x.y' }, // { format: SerializationFormat.ENCRYPTED_V2, minVersion: '5.x.y' }, diff --git a/packages/core/src/serialization-format.ts b/packages/core/src/serialization-format.ts index a49af20e2d..1fbbd6bfdb 100644 --- a/packages/core/src/serialization-format.ts +++ b/packages/core/src/serialization-format.ts @@ -19,6 +19,8 @@ export const SerializationFormat = { ENCRYPTED: 'encr', /** Gzip-compressed payload (inner payload has its own format prefix after decompression) */ GZIP: 'gzip', + /** Zstandard-compressed payload (inner payload has its own format prefix after decompression) */ + ZSTD: 'zstd', } as const; export type SerializationFormatType = @@ -149,7 +151,7 @@ export function isEncryptedData(data: unknown): boolean { } /** - * Check if a binary value has the 'gzip' format prefix indicating compression. + * Check if a binary value has a compression format prefix ('gzip' or 'zstd'). * Browser-safe — does not depend on the full serialization module. */ export function isCompressedData(data: unknown): boolean { @@ -157,33 +159,52 @@ export function isCompressedData(data: unknown): boolean { return false; } const prefix = formatDecoder.decode(data.subarray(0, FORMAT_PREFIX_LENGTH)); - return prefix === SerializationFormat.GZIP; + return ( + prefix === SerializationFormat.GZIP || prefix === SerializationFormat.ZSTD + ); +} + +interface NodeZlibDecode { + gunzipSync?: (data: Uint8Array) => Uint8Array; + zstdDecompressSync?: (data: Uint8Array) => Uint8Array; } /** - * Synchronously gunzip a payload when running on Node.js. - * - * This module is browser-safe, so `node:zlib` is resolved dynamically via - * `process.getBuiltinModule` (no static Node dependency, invisible to - * browser bundlers). Returns `undefined` when sync decompression isn't - * available in the current runtime — callers fall back to leaving the - * data un-hydrated (the async `hydrateDataWithKey` path handles - * decompression in browsers via `DecompressionStream`). + * Resolve `node:zlib` via `process.getBuiltinModule` — no static Node + * dependency, invisible to browser bundlers. Returns undefined off Node. */ -function gunzipSyncIfAvailable(payload: Uint8Array): Uint8Array | undefined { +function getNodeZlib(): NodeZlibDecode | undefined { try { - const zlib = ( + return ( globalThis as { - process?: { - getBuiltinModule?: (id: string) => { - gunzipSync?: (data: Uint8Array) => Uint8Array; - }; - }; + process?: { getBuiltinModule?: (id: string) => NodeZlibDecode }; } ).process?.getBuiltinModule?.('node:zlib'); - if (zlib?.gunzipSync) { + } catch { + return undefined; + } +} + +/** + * Synchronously decompress a `gzip`/`zstd` payload when running on Node.js. + * + * Returns `undefined` when sync decompression isn't available (e.g. in the + * browser, or zstd on Node < 22.15) — callers fall back to leaving the data + * un-hydrated (the async `hydrateDataWithKey` path handles decompression in + * browsers via `DecompressionStream` / a registered zstd decoder). + */ +function decompressSyncIfAvailable( + format: string, + payload: Uint8Array +): Uint8Array | undefined { + try { + const zlib = getNodeZlib(); + if (format === SerializationFormat.GZIP && zlib?.gunzipSync) { return new Uint8Array(zlib.gunzipSync(payload)); } + if (format === SerializationFormat.ZSTD && zlib?.zstdDecompressSync) { + return new Uint8Array(zlib.zstdDecompressSync(payload)); + } } catch { // Fall through — treat as unavailable } @@ -191,10 +212,45 @@ function gunzipSyncIfAvailable(payload: Uint8Array): Uint8Array | undefined { } /** - * Asynchronously gunzip a payload using the web-standard - * DecompressionStream (Node 18+, browsers, edge runtimes). + * Browser zstd decoder, registered by the o11y host (web-shared) since the + * Web `DecompressionStream` has no zstd support. Node decodes via `node:zlib` + * and never needs this. See `registerZstdDecoder`. + */ +let zstdBrowserDecoder: + | ((payload: Uint8Array) => Promise) + | undefined; + +/** + * Register a browser zstd decoder (e.g. a WASM-backed one). The web o11y UI + * calls this at init so `hydrateDataWithKey` can inflate zstd payloads after + * client-side decryption. Node readers use `node:zlib` and ignore this. */ -async function gunzipAsync(payload: Uint8Array): Promise { +export function registerZstdDecoder( + decoder: (payload: Uint8Array) => Promise +): void { + zstdBrowserDecoder = decoder; +} + +/** + * Asynchronously decompress a `gzip`/`zstd` payload. + * - gzip: web-standard `DecompressionStream` (Node 18+, browsers, edge). + * - zstd: `node:zlib` when on Node, else the registered browser decoder. + */ +async function decompressAsync( + format: string, + payload: Uint8Array +): Promise { + if (format === SerializationFormat.ZSTD) { + const sync = decompressSyncIfAvailable(format, payload); + if (sync) return sync; + if (zstdBrowserDecoder) return zstdBrowserDecoder(payload); + throw new Error( + 'zstd-compressed workflow data encountered but no zstd decoder is ' + + 'available. Node.js 22.15+ decodes natively; in the browser register ' + + 'one via registerZstdDecoder (the web o11y package does this).' + ); + } + const transform = new DecompressionStream('gzip'); const writer = transform.writable.getWriter(); const writePromise = writer.write(payload).then(() => writer.close()); @@ -261,13 +317,16 @@ export function hydrateData(value: unknown, revivers: Revivers): unknown { const str = new TextDecoder().decode(payload); return parse(str, revivers); } - if (format === SerializationFormat.GZIP) { + if ( + format === SerializationFormat.GZIP || + format === SerializationFormat.ZSTD + ) { // Compressed payload — decompress synchronously when running on - // Node.js (CLI, server o11y). In browsers there is no sync gunzip; + // Node.js (CLI, server o11y). In browsers there is no sync codec; // pass the data through untouched (like encrypted data) so async // consumers can route it through `hydrateDataWithKey`, which - // decompresses via DecompressionStream. - const inflated = gunzipSyncIfAvailable(payload); + // decompresses via DecompressionStream / a registered zstd decoder. + const inflated = decompressSyncIfAvailable(format, payload); if (inflated === undefined) { return value; } @@ -309,12 +368,12 @@ export async function hydrateDataWithKey( data = await decrypt(key, payload); } if (data instanceof Uint8Array && isCompressedData(data)) { - // Decompress: strip 'gzip' prefix and inflate via the web-standard - // DecompressionStream (works in browsers, unlike the sync Node path - // inside hydrateData). The inflated bytes carry their own format - // prefix (e.g. 'devl'). - const { payload } = decodeFormatPrefix(data); - data = await gunzipAsync(payload); + // Decompress: strip the codec prefix and inflate. gzip uses the + // web-standard DecompressionStream (works in browsers); zstd uses + // node:zlib on Node or the registered WASM decoder in the browser. + // The inflated bytes carry their own format prefix (e.g. 'devl'). + const { format, payload } = decodeFormatPrefix(data); + data = await decompressAsync(format, payload); } // Delegate the (decrypted/decompressed) result to sync hydrateData return hydrateData(data, revivers); diff --git a/packages/core/src/serialization.ts b/packages/core/src/serialization.ts index a7a932b4d7..399a2a2ba9 100644 --- a/packages/core/src/serialization.ts +++ b/packages/core/src/serialization.ts @@ -192,6 +192,7 @@ async function recordCompression( span.setAttributes({ ...Attr.SerializationOperation(operation), ...Attr.SerializationCompressed(stats.compressed ?? false), + ...Attr.SerializationCodec(stats.codec ?? 'none'), ...Attr.SerializationUncompressedBytes(uncompressedBytes), ...Attr.SerializationStoredBytes(storedBytes), ...(stats.compressed && uncompressedBytes > 0 diff --git a/packages/core/src/serialization/compression-telemetry.test.ts b/packages/core/src/serialization/compression-telemetry.test.ts index 607fab8fb9..cadf68a53a 100644 --- a/packages/core/src/serialization/compression-telemetry.test.ts +++ b/packages/core/src/serialization/compression-telemetry.test.ts @@ -59,6 +59,8 @@ describe('compression telemetry attributes', () => { const attrs = lastAttrs(); expect(attrs['workflow.serialization.operation']).toBe('serialize'); expect(attrs['workflow.serialization.compressed']).toBe(true); + // zstd is the preferred codec when node:zlib has it (Node >= 22.15). + expect(attrs['workflow.serialization.codec']).toBe('zstd'); expect(attrs['workflow.serialization.uncompressed_bytes']).toBeGreaterThan( attrs['workflow.serialization.stored_bytes'] as number ); @@ -94,6 +96,7 @@ describe('compression telemetry attributes', () => { const attrs = lastAttrs(); expect(attrs['workflow.serialization.operation']).toBe('serialize'); expect(attrs['workflow.serialization.compressed']).toBe(false); + expect(attrs['workflow.serialization.codec']).toBe('none'); expect(attrs['workflow.serialization.compression_ratio']).toBeUndefined(); expect(attrs['workflow.serialization.stored_bytes']).toBe( attrs['workflow.serialization.uncompressed_bytes'] diff --git a/packages/core/src/serialization/compression.test.ts b/packages/core/src/serialization/compression.test.ts index 83bef6aa45..2d42ffa6c7 100644 --- a/packages/core/src/serialization/compression.test.ts +++ b/packages/core/src/serialization/compression.test.ts @@ -253,7 +253,8 @@ describe('mode serializers with compression', () => { const compressed = await stepModule.serialize(value, undefined, { compression: true, }); - expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.GZIP); + // zstd is the preferred codec when available (node:zlib >= 22.15). + expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.ZSTD); const uncompressed = await stepModule.serialize(value, undefined, {}); expect(peekFormatPrefix(uncompressed)).toBe(SerializationFormat.DEVALUE_V1); @@ -270,12 +271,12 @@ describe('mode serializers with compression', () => { const data = await clientModule.serialize(value, undefined, { compression: true, }); - expect(peekFormatPrefix(data)).toBe(SerializationFormat.GZIP); + expect(peekFormatPrefix(data)).toBe(SerializationFormat.ZSTD); const result = await clientModule.deserialize(data, undefined, {}); expect(result).toEqual(value); }); - it('nests compression inside encryption: encr(gzip(devl))', async () => { + it('nests compression inside encryption: encr(zstd(devl))', async () => { const key = await makeKey(); const value = makeCompressibleValue(); @@ -285,9 +286,9 @@ describe('mode serializers with compression', () => { // Outer layer must be encryption (encrypted bytes don't compress) expect(peekFormatPrefix(data)).toBe(SerializationFormat.ENCRYPTED); - // White-box: the decrypted inner payload carries the gzip prefix + // White-box: the decrypted inner payload carries the codec prefix const inner = await decrypt(data, key); - expect(peekFormatPrefix(inner)).toBe(SerializationFormat.GZIP); + expect(peekFormatPrefix(inner)).toBe(SerializationFormat.ZSTD); const { payload: deflated } = decodeFormatPrefix(inner); expect(deflated.length).toBeGreaterThan(0); @@ -321,6 +322,61 @@ describe('mode serializers with compression', () => { }); }); +describe('codec selection (zstd preferred, gzip fallback)', () => { + afterEach(() => { + delete process.env.WORKFLOW_COMPRESSION_CODEC; + }); + + it('prefers zstd by default and reports it in stats', async () => { + const original = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(JSON.stringify(makeCompressibleValue())) + ) as Uint8Array; + const stats: CompressionStats = {}; + const compressed = await compress(original, true, stats); + expect(isCompressed(compressed)).toBe(true); + expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.ZSTD); + expect(stats.codec).toBe('zstd'); + }); + + it('WORKFLOW_COMPRESSION_CODEC=gzip forces the portable codec', async () => { + process.env.WORKFLOW_COMPRESSION_CODEC = 'gzip'; + const original = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(JSON.stringify(makeCompressibleValue())) + ) as Uint8Array; + const stats: CompressionStats = {}; + const compressed = await compress(original, true, stats); + expect(peekFormatPrefix(compressed)).toBe(SerializationFormat.GZIP); + expect(stats.codec).toBe('gzip'); + + // Read path still inflates gzip and reports the codec. + const readStats: CompressionStats = {}; + const inflated = (await decompress(compressed, readStats)) as Uint8Array; + expect(inflated).toEqual(original); + expect(readStats.codec).toBe('gzip'); + }); + + it('decompress handles both zstd and gzip prefixes (mixed log)', async () => { + const original = encodeWithFormatPrefix( + SerializationFormat.DEVALUE_V1, + textEncoder.encode(JSON.stringify(makeCompressibleValue())) + ) as Uint8Array; + + const zstd = (await compress(original, true)) as Uint8Array; + expect(peekFormatPrefix(zstd)).toBe(SerializationFormat.ZSTD); + + process.env.WORKFLOW_COMPRESSION_CODEC = 'gzip'; + const gzip = (await compress(original, true)) as Uint8Array; + expect(peekFormatPrefix(gzip)).toBe(SerializationFormat.GZIP); + delete process.env.WORKFLOW_COMPRESSION_CODEC; + + // Both decode regardless of the current write-side codec setting. + expect(await decompress(zstd)).toEqual(original); + expect(await decompress(gzip)).toEqual(original); + }); +}); + describe('dehydrateStepError with compression', () => { it('compresses large errors and round-trips through hydrateStepError', async () => { const error = new Error('boom'); @@ -335,7 +391,7 @@ describe('dehydrateStepError with compression', () => { globalThis, true ); - expect(peekFormatPrefix(data)).toBe(SerializationFormat.GZIP); + expect(peekFormatPrefix(data)).toBe(SerializationFormat.ZSTD); const hydrated = (await hydrateStepError( data, @@ -379,30 +435,30 @@ describe('o11y hydration of compressed payloads', () => { }); }); -describe('run capabilities for gzip', () => { - it('supports gzip for core versions >= 5.0.0-beta.16', () => { - expect( - getRunCapabilities('5.0.0-beta.16').supportedFormats.has( - SerializationFormat.GZIP - ) - ).toBe(true); - }); - - it('does not support gzip for older core versions', () => { - for (const version of ['5.0.0-beta.15', '4.2.1', '4.0.0']) { +describe('run capabilities for compression codecs', () => { + // gzip and zstd co-ship, so both are gated on the same min version. + for (const fmt of [ + SerializationFormat.GZIP, + SerializationFormat.ZSTD, + ] as const) { + it(`supports ${fmt} for core versions >= 5.0.0-beta.16`, () => { expect( - getRunCapabilities(version).supportedFormats.has( - SerializationFormat.GZIP - ) - ).toBe(false); - } - }); + getRunCapabilities('5.0.0-beta.16').supportedFormats.has(fmt) + ).toBe(true); + }); - it('assumes no gzip support when the version is unknown', () => { - expect( - getRunCapabilities(undefined).supportedFormats.has( - SerializationFormat.GZIP - ) - ).toBe(false); - }); + it(`does not support ${fmt} for older core versions`, () => { + for (const version of ['5.0.0-beta.15', '4.2.1', '4.0.0']) { + expect(getRunCapabilities(version).supportedFormats.has(fmt)).toBe( + false + ); + } + }); + + it(`assumes no ${fmt} support when the version is unknown`, () => { + expect(getRunCapabilities(undefined).supportedFormats.has(fmt)).toBe( + false + ); + }); + } }); diff --git a/packages/core/src/serialization/compression.ts b/packages/core/src/serialization/compression.ts index 1258337f83..2a99303be4 100644 --- a/packages/core/src/serialization/compression.ts +++ b/packages/core/src/serialization/compression.ts @@ -1,25 +1,35 @@ /** * Composable compression layer for serialized data. * - * Wraps/unwraps serialized payloads with gzip compression, using the - * format prefix system to mark compressed data ('gzip' wrapping the - * inner format, e.g. 'gzip' + deflate('devl' + payload)). + * Wraps/unwraps serialized payloads with a compression codec, using the + * format prefix system to mark compressed data (e.g. 'zstd' or 'gzip' + * wrapping the inner format: 'zstd' + zstd('devl' + payload)). + * + * Codec selection (write side): zstd is preferred — it is markedly faster + * than gzip at a comparable-or-better ratio (see scripts/README.md), and + * compression runs at every step boundary so the write CPU is a per-step + * tax. zstd requires `node:zlib` >= 22.15 (Web `CompressionStream` has no + * zstd), so on a runtime without it we fall back to gzip via the portable + * `CompressionStream`. `WORKFLOW_COMPRESSION_CODEC=gzip` forces the + * portable codec. + * + * Read side: dispatch on the format prefix, so both 'zstd' and 'gzip' + * payloads are always decodable regardless of which codec wrote them. + * (The browser o11y read path decodes zstd via a registered WASM decoder — + * see `serialization-format.ts`; this module's `decompress` is the Node + * runtime/replay path and uses `node:zlib`.) * * Layering order with encryption: compression is applied BEFORE - * encryption (encr(gzip(devl))) — encrypted bytes are high-entropy and + * encryption (encr(zstd(devl))) — encrypted bytes are high-entropy and * do not compress, so the reverse order would be a no-op. * * Compression is conditional: * - Payloads smaller than {@link COMPRESSION_MIN_BYTES} are passed - * through unchanged (gzip overhead isn't worth it). + * through unchanged (codec overhead isn't worth it). * - If the compressed result isn't meaningfully smaller than the * original (see {@link COMPRESSION_MIN_SAVINGS_RATIO}), the original * is kept. This protects already-compressed binary payloads (images, * archives, etc.) from wasted CPU and size inflation. - * - * Decompression is unconditional: any payload carrying the 'gzip' - * prefix is inflated, so readers transparently handle both compressed - * and uncompressed data regardless of write-side settings. */ import { @@ -31,8 +41,8 @@ import { SerializationFormat } from './types.js'; /** * Payloads below this size are never compressed. The 4-byte format - * prefix + ~20 bytes of gzip header/trailer overhead means small - * payloads gain nothing, and tiny ones would grow. + * prefix + codec header/trailer overhead means small payloads gain + * nothing, and tiny ones would grow. */ export const COMPRESSION_MIN_BYTES = 1024; @@ -44,6 +54,12 @@ export const COMPRESSION_MIN_BYTES = 1024; */ export const COMPRESSION_MIN_SAVINGS_RATIO = 0.05; +/** Default zstd compression level — the sweet spot of speed vs ratio. */ +const ZSTD_LEVEL = 3; + +/** Which codec compressed a payload (or `none` when stored uncompressed). */ +export type CompressionCodec = 'zstd' | 'gzip' | 'none'; + /** * Escape hatch: set WORKFLOW_DISABLE_COMPRESSION=1 to disable * write-side compression entirely. Reads are unaffected — payloads @@ -60,6 +76,61 @@ function isCompressionDisabledByEnv(): boolean { } } +/** + * Optional codec override (`WORKFLOW_COMPRESSION_CODEC=gzip|zstd`). Lets an + * operator pin the portable codec (gzip) — useful for A/B comparisons or + * runtimes where zstd read support isn't yet everywhere. + */ +function codecOverrideFromEnv(): 'gzip' | 'zstd' | undefined { + try { + const v = process.env?.WORKFLOW_COMPRESSION_CODEC; + return v === 'gzip' || v === 'zstd' ? v : undefined; + } catch { + return undefined; + } +} + +interface NodeZlib { + zstdCompressSync?: (data: Uint8Array, opts?: unknown) => Uint8Array; + zstdDecompressSync?: (data: Uint8Array) => Uint8Array; + constants?: Record; +} + +/** + * Resolve `node:zlib` via `process.getBuiltinModule` — no static import, so + * this module stays bundler-safe for browser/edge targets (where it returns + * undefined and we fall back to gzip). + */ +function getNodeZlib(): NodeZlib | undefined { + try { + return ( + globalThis as { + process?: { getBuiltinModule?: (id: string) => NodeZlib }; + } + ).process?.getBuiltinModule?.('node:zlib'); + } catch { + return undefined; + } +} + +function isZstdAvailable(): boolean { + const z = getNodeZlib(); + return ( + typeof z?.zstdCompressSync === 'function' && + typeof z?.zstdDecompressSync === 'function' + ); +} + +/** + * gzip via the web-standard `CompressionStream` (Node 18+, browsers, edge). + */ +function isGzipAvailable(): boolean { + return ( + typeof CompressionStream === 'function' && + typeof DecompressionStream === 'function' + ); +} + /** * Pipe bytes through a (De)CompressionStream and collect the output. */ @@ -105,16 +176,26 @@ async function gunzipBytes(data: Uint8Array): Promise { return pipeThroughTransform(data, new DecompressionStream('gzip')); } -/** - * Whether the current runtime can compress/decompress. CompressionStream - * is a web standard available in Node.js 18+, browsers, and edge - * runtimes; this guard exists for exotic runtimes only. - */ -function isCompressionAvailable(): boolean { - return ( - typeof CompressionStream === 'function' && - typeof DecompressionStream === 'function' - ); +function zstdBytes(data: Uint8Array): Uint8Array { + const z = getNodeZlib(); + const level = z?.constants?.ZSTD_c_compressionLevel; + const opts = + level !== undefined ? { params: { [level]: ZSTD_LEVEL } } : undefined; + // biome-ignore lint/style/noNonNullAssertion: guarded by isZstdAvailable() + return new Uint8Array(z!.zstdCompressSync!(data, opts)); +} + +function unzstdBytes(data: Uint8Array): Uint8Array { + const z = getNodeZlib(); + if (!z?.zstdDecompressSync) { + throw new Error( + 'Compressed (zstd) workflow data encountered but node:zlib zstd ' + + 'support is not available in this runtime (requires Node.js 22.15+). ' + + 'In the browser, register a zstd decoder via registerZstdDecoder ' + + '(serialization-format.ts).' + ); + } + return new Uint8Array(z.zstdDecompressSync(data)); } /** @@ -129,13 +210,16 @@ function isCompressionAvailable(): boolean { * Field meanings are identical for both directions: * - `uncompressedBytes`: the logical (devalue-prefixed) payload size. * - `storedBytes`: the size handed to / read from storage (compressed when - * the gzip codec applied, otherwise equal to `uncompressedBytes`). + * a codec applied, otherwise equal to `uncompressedBytes`). + * - `codec`: which codec applied (`none` when stored uncompressed). */ export interface CompressionStats { /** True once the compression layer ran (i.e. saw binary data). */ recorded?: boolean; - /** Whether the gzip codec was applied (write) or present (read). */ + /** Whether a codec was applied (write) or present (read). */ compressed?: boolean; + /** Which codec applied / was present. */ + codec?: CompressionCodec; /** Logical, uncompressed payload size in bytes. */ uncompressedBytes?: number; /** Stored (post-compression) payload size in bytes. */ @@ -144,17 +228,31 @@ export interface CompressionStats { function recordStats( stats: CompressionStats | undefined, - compressed: boolean, + codec: CompressionCodec, uncompressedBytes: number, storedBytes: number ): void { if (!stats) return; stats.recorded = true; - stats.compressed = compressed; + stats.compressed = codec !== 'none'; + stats.codec = codec; stats.uncompressedBytes = uncompressedBytes; stats.storedBytes = storedBytes; } +/** + * Choose the write-side codec given runtime availability and the optional + * env override. zstd is preferred; gzip is the portable fallback. + */ +function selectWriteCodec(): 'zstd' | 'gzip' | 'none' { + const override = codecOverrideFromEnv(); + if (override === 'gzip') return isGzipAvailable() ? 'gzip' : 'none'; + // Default and explicit 'zstd' both prefer zstd, then fall back to gzip. + if (isZstdAvailable()) return 'zstd'; + if (isGzipAvailable()) return 'gzip'; + return 'none'; +} + /** * Compress a format-prefixed payload if compression is enabled for the * target run and the payload is worth compressing. @@ -163,10 +261,11 @@ function recordStats( * @param enabled - Whether the target run supports compressed payloads * (run specVersion >= SPEC_VERSION_SUPPORTS_COMPRESSION, and for * cross-deployment writes, the target deployment's capabilities — - * see `getRunCapabilities` in capabilities.ts) + * see `getRunCapabilities` in capabilities.ts). zstd and gzip read + * support co-ship, so a single boolean is sufficient. * @param stats - Optional telemetry sink; populated when `data` is binary. - * @returns The compressed data with 'gzip' prefix, or the original data - * when compression is disabled, unavailable, or not worthwhile + * @returns The compressed data with a codec prefix, or the original data + * when compression is disabled, unavailable, or not worthwhile. */ export async function compress( data: Uint8Array | unknown, @@ -178,27 +277,34 @@ export async function compress( if ( !enabled || data.length < COMPRESSION_MIN_BYTES || - isCompressionDisabledByEnv() || - !isCompressionAvailable() + isCompressionDisabledByEnv() ) { - recordStats(stats, false, data.length, data.length); + recordStats(stats, 'none', data.length, data.length); + return data; + } + + const codec = selectWriteCodec(); + if (codec === 'none') { + recordStats(stats, 'none', data.length, data.length); return data; } - const compressed = await gzipBytes(data); + const compressed = codec === 'zstd' ? zstdBytes(data) : await gzipBytes(data); + const format = + codec === 'zstd' ? SerializationFormat.ZSTD : SerializationFormat.GZIP; const wrappedLength = 4 + compressed.length; // format prefix + payload if (wrappedLength >= data.length * (1 - COMPRESSION_MIN_SAVINGS_RATIO)) { - recordStats(stats, false, data.length, data.length); + recordStats(stats, 'none', data.length, data.length); return data; } - recordStats(stats, true, data.length, wrappedLength); - return encodeWithFormatPrefix(SerializationFormat.GZIP, compressed); + recordStats(stats, codec, data.length, wrappedLength); + return encodeWithFormatPrefix(format, compressed); } /** * Decompress a format-prefixed payload if it's compressed. - * Strips the 'gzip' format prefix and inflates the inner payload - * (which carries its own format prefix, e.g. 'devl'). + * Dispatches on the prefix ('zstd' or 'gzip') and inflates the inner + * payload (which carries its own format prefix, e.g. 'devl'). * * Non-compressed data (including non-binary legacy data) is returned * unchanged, so this is safe to apply unconditionally on read paths. @@ -208,28 +314,39 @@ export async function decompress( stats?: CompressionStats ): Promise { if (!(data instanceof Uint8Array)) return data; - if (peekFormatPrefix(data) !== SerializationFormat.GZIP) { - recordStats(stats, false, data.length, data.length); - return data; + const prefix = peekFormatPrefix(data); + + if (prefix === SerializationFormat.ZSTD) { + const { payload } = decodeFormatPrefix(data); + const inflated = unzstdBytes(payload); + recordStats(stats, 'zstd', inflated.length, data.length); + return inflated; } - if (!isCompressionAvailable()) { - throw new Error( - 'Compressed (gzip) workflow data encountered but DecompressionStream ' + - 'is not available in this runtime. Node.js 18+, browsers, and edge ' + - 'runtimes all support it.' - ); + if (prefix === SerializationFormat.GZIP) { + if (!isGzipAvailable()) { + throw new Error( + 'Compressed (gzip) workflow data encountered but DecompressionStream ' + + 'is not available in this runtime. Node.js 18+, browsers, and edge ' + + 'runtimes all support it.' + ); + } + const { payload } = decodeFormatPrefix(data); + const inflated = await gunzipBytes(payload); + recordStats(stats, 'gzip', inflated.length, data.length); + return inflated; } - const { payload } = decodeFormatPrefix(data); - const inflated = await gunzipBytes(payload); - recordStats(stats, true, inflated.length, data.length); - return inflated; + recordStats(stats, 'none', data.length, data.length); + return data; } /** - * Check if data is compressed (has 'gzip' format prefix). + * Check if data is compressed (has a 'zstd' or 'gzip' format prefix). */ export function isCompressed(data: Uint8Array | unknown): boolean { - return peekFormatPrefix(data) === SerializationFormat.GZIP; + const prefix = peekFormatPrefix(data); + return ( + prefix === SerializationFormat.ZSTD || prefix === SerializationFormat.GZIP + ); } diff --git a/packages/core/src/serialization/index.ts b/packages/core/src/serialization/index.ts index 55acf8c014..5416f3bc44 100644 --- a/packages/core/src/serialization/index.ts +++ b/packages/core/src/serialization/index.ts @@ -11,6 +11,7 @@ export { devalueCodec } from './codec-devalue.js'; // Re-export composable compression export { COMPRESSION_MIN_BYTES, + type CompressionCodec, type CompressionStats, compress, decompress, diff --git a/packages/core/src/serialization/types.ts b/packages/core/src/serialization/types.ts index da948bee14..3668c05148 100644 --- a/packages/core/src/serialization/types.ts +++ b/packages/core/src/serialization/types.ts @@ -34,6 +34,8 @@ export const SerializationFormat = { ENCRYPTED: 'encr' as FormatPrefix, /** Gzip-compressed payload (inner payload has its own format prefix) */ GZIP: 'gzip' as FormatPrefix, + /** Zstandard-compressed payload (inner payload has its own format prefix) */ + ZSTD: 'zstd' as FormatPrefix, } as const; // ---- Serializable Types ---- diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index 1f0d8bb5a7..53a5e494a4 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -316,11 +316,16 @@ export const SerializationOperation = SemanticConvention< 'serialize' | 'deserialize' >('workflow.serialization.operation'); -/** Whether the gzip codec was applied (write) / present (read). */ +/** Whether a compression codec was applied (write) / present (read). */ export const SerializationCompressed = SemanticConvention( 'workflow.serialization.compressed' ); +/** Which compression codec applied / was present (`zstd`, `gzip`, or `none`). */ +export const SerializationCodec = SemanticConvention<'zstd' | 'gzip' | 'none'>( + 'workflow.serialization.codec' +); + /** Logical (uncompressed, devalue-prefixed) payload size in bytes. */ export const SerializationUncompressedBytes = SemanticConvention( 'workflow.serialization.uncompressed_bytes' diff --git a/packages/web-shared/package.json b/packages/web-shared/package.json index fb4540aba7..0573204f6c 100644 --- a/packages/web-shared/package.json +++ b/packages/web-shared/package.json @@ -50,6 +50,7 @@ "@radix-ui/react-slot": "1.1.1", "@radix-ui/react-tooltip": "1.2.8", "@tailwindcss/postcss": "4", + "@tootallnate/zstd-wasm": "0.0.2", "@workflow/core": "workspace:*", "@workflow/utils": "workspace:*", "@workflow/world": "workspace:*", diff --git a/packages/web-shared/src/lib/hydration.ts b/packages/web-shared/src/lib/hydration.ts index 936c35c1fb..dbf1d34d6f 100644 --- a/packages/web-shared/src/lib/hydration.ts +++ b/packages/web-shared/src/lib/hydration.ts @@ -476,6 +476,13 @@ export async function hydrateResourceIOWithKey( '@workflow/core/serialization-format' ); const { importKey } = await import('@workflow/core/encryption'); + // Payloads may be zstd-compressed (the Web DecompressionStream has no zstd); + // register the WASM-backed browser decoder before hydrating. Idempotent and + // lazy — the WASM is only compiled when a zstd payload is actually decoded. + const { ensureZstdDecoderRegistered } = await import( + './zstd-browser-decoder.js' + ); + ensureZstdDecoderRegistered(); const cryptoKey = await importKey(key); const revivers = getRevivers(); diff --git a/packages/web-shared/src/lib/zstd-browser-decoder.ts b/packages/web-shared/src/lib/zstd-browser-decoder.ts new file mode 100644 index 0000000000..05cf339dfd --- /dev/null +++ b/packages/web-shared/src/lib/zstd-browser-decoder.ts @@ -0,0 +1,42 @@ +/** + * Browser zstd decoder for the o11y read path. + * + * The Web `DecompressionStream` has no zstd support, so `@workflow/core`'s + * `hydrateDataWithKey` delegates zstd inflation to a decoder registered via + * `registerZstdDecoder`. This module supplies that decoder, backed by the + * `@tootallnate/zstd-wasm` single-file WASM decoder. + * + * The package leaves WASM sourcing to the caller; we resolve the shipped + * `zstd.wasm` as a bundler asset (`new URL(..., import.meta.url)`, the same + * pattern the trace-viewer Worker uses) and compile it once, lazily — the + * WASM is fetched only the first time a zstd payload is actually decoded. + */ +import { registerZstdDecoder } from '@workflow/core/serialization-format'; + +let registered = false; +let modulePromise: Promise | undefined; + +function loadWasmModule(): Promise { + if (!modulePromise) { + const url = new URL('@tootallnate/zstd-wasm/zstd.wasm', import.meta.url); + modulePromise = fetch(url) + .then((res) => res.arrayBuffer()) + .then((bytes) => WebAssembly.compile(bytes)); + } + return modulePromise; +} + +/** + * Register the browser zstd decoder with `@workflow/core` (idempotent). + * Call this before hydrating payloads that may be zstd-compressed; the + * actual WASM compile + decode happens lazily on first use. + */ +export function ensureZstdDecoderRegistered(): void { + if (registered) return; + registered = true; + registerZstdDecoder(async (payload) => { + const { decompressBytes } = await import('@tootallnate/zstd-wasm'); + const wasmModule = await loadWasmModule(); + return decompressBytes(wasmModule, payload); + }); +} diff --git a/packages/web-shared/test/zstd-decoder.test.ts b/packages/web-shared/test/zstd-decoder.test.ts new file mode 100644 index 0000000000..ac3dd2bab8 --- /dev/null +++ b/packages/web-shared/test/zstd-decoder.test.ts @@ -0,0 +1,56 @@ +/** + * Compatibility test for the browser zstd decode path: payloads written by + * the SDK's `node:zlib` zstd codec must decode via the `@tootallnate/zstd-wasm` + * decoder the web o11y uses. If these ever disagree, the dashboard can't read + * compressed runs — so this locks the cross-codec contract in. + */ +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import zlib from 'node:zlib'; +import { decompressBytes } from '@tootallnate/zstd-wasm'; +import { beforeAll, describe, expect, it } from 'vitest'; + +const require = createRequire(import.meta.url); + +let wasmModule: WebAssembly.Module; + +beforeAll(async () => { + const wasmPath = require.resolve('@tootallnate/zstd-wasm/zstd.wasm'); + wasmModule = await WebAssembly.compile(readFileSync(wasmPath)); +}); + +function zstd(bytes: Uint8Array): Uint8Array { + return new Uint8Array( + zlib.zstdCompressSync(bytes, { + params: { [zlib.constants.ZSTD_c_compressionLevel]: 3 }, + }) + ); +} + +describe('zstd WASM decoder ↔ node:zlib zstd compatibility', () => { + it('decodes a payload compressed by the SDK codec', async () => { + const original = new TextEncoder().encode( + JSON.stringify({ + // Repetitive + varied content, like a real serialized payload. + users: Array.from({ length: 300 }, (_, i) => ({ + id: `user_${i}`, + email: `user.${i}@example.com`, + role: i % 3 === 0 ? 'admin' : 'member', + })), + }) + ); + const compressed = zstd(original); + expect(compressed.length).toBeLessThan(original.length); + + const decoded = await decompressBytes(wasmModule, compressed); + expect(new Uint8Array(decoded)).toEqual(original); + }); + + it('round-trips an empty and a tiny payload', async () => { + for (const s of ['', '{}', 'x']) { + const original = new TextEncoder().encode(s); + const decoded = await decompressBytes(wasmModule, zstd(original)); + expect(new Uint8Array(decoded)).toEqual(original); + } + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 81a2a2dba5..ca0f1df7d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -834,7 +834,7 @@ importers: devDependencies: '@nuxt/module-builder': specifier: 1.0.2 - version: 1.0.2(@nuxt/cli@3.35.2(@nuxt/schema@4.4.7)(cac@6.7.14)(magicast@0.5.2))(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(typescript@5.9.3)(vue@3.5.35(typescript@5.9.3)) + version: 1.0.2(@nuxt/cli@3.35.2(@nuxt/schema@4.4.7)(cac@6.7.14)(magicast@0.5.2))(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(typescript@5.9.3)(vue@3.5.35(typescript@5.9.3)) '@nuxt/schema': specifier: 4.4.7 version: 4.4.7 @@ -1179,6 +1179,9 @@ importers: '@tailwindcss/postcss': specifier: '4' version: 4.1.13 + '@tootallnate/zstd-wasm': + specifier: 0.0.2 + version: 0.0.2 '@workflow/core': specifier: workspace:* version: link:../core @@ -9273,6 +9276,9 @@ packages: '@tokenizer/token@0.3.0': resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} + '@tootallnate/zstd-wasm@0.0.2': + resolution: {integrity: sha512-g2vM+SRF90TEuEUZ0lNGFGoUSgVP0wl1NumrpPP64oxyMIkpHNmVeojkjfV8HGEyLxpUMHlylNsZPQcKhaIoxg==} + '@ts-morph/common@0.11.1': resolution: {integrity: sha512-7hWZS0NRpEsNV8vWJzg7FEz6V8MaLNeJOmwmghqUXTpzk16V1LLZhdo+4QvE/+zv4cVci0OviuJFnqhEfoV3+g==} @@ -20681,7 +20687,7 @@ snapshots: transitivePeerDependencies: - magicast - '@nuxt/module-builder@1.0.2(@nuxt/cli@3.35.2(@nuxt/schema@4.4.7)(cac@6.7.14)(magicast@0.5.2))(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(typescript@5.9.3)(vue@3.5.35(typescript@5.9.3))': + '@nuxt/module-builder@1.0.2(@nuxt/cli@3.35.2(@nuxt/schema@4.4.7)(cac@6.7.14)(magicast@0.5.2))(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(typescript@5.9.3)(vue@3.5.35(typescript@5.9.3))': dependencies: '@nuxt/cli': 3.35.2(@nuxt/schema@4.4.7)(cac@6.7.14)(magicast@0.5.2) citty: 0.1.6 @@ -20689,14 +20695,14 @@ snapshots: defu: 6.1.4 jiti: 2.6.1 magic-regexp: 0.10.0 - mkdist: 2.4.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)) + mkdist: 2.4.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)) mlly: 1.8.0 pathe: 2.0.3 pkg-types: 2.3.0 tsconfck: 3.1.6(typescript@5.9.3) typescript: 5.9.3 - unbuild: 3.6.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)) - vue-sfc-transformer: 0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)) + unbuild: 3.6.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)) + vue-sfc-transformer: 0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)) transitivePeerDependencies: - '@vue/compiler-core' - esbuild @@ -25796,6 +25802,8 @@ snapshots: '@tokenizer/token@0.3.0': {} + '@tootallnate/zstd-wasm@0.0.2': {} + '@ts-morph/common@0.11.1': dependencies: fast-glob: 3.3.3 @@ -31722,7 +31730,7 @@ snapshots: mkdirp@3.0.1: {} - mkdist@2.4.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)): + mkdist@2.4.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)): dependencies: autoprefixer: 10.4.21(postcss@8.5.6) citty: 0.1.6 @@ -31740,7 +31748,7 @@ snapshots: optionalDependencies: typescript: 5.9.3 vue: 3.5.35(typescript@5.9.3) - vue-sfc-transformer: 0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)) + vue-sfc-transformer: 0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)) mlly@1.8.0: dependencies: @@ -36067,7 +36075,7 @@ snapshots: ultrahtml@1.6.0: {} - unbuild@3.6.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)): + unbuild@3.6.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)): dependencies: '@rollup/plugin-alias': 5.1.1(rollup@4.60.0) '@rollup/plugin-commonjs': 28.0.9(rollup@4.60.0) @@ -36083,7 +36091,7 @@ snapshots: hookable: 5.5.3 jiti: 2.6.1 magic-string: 0.30.21 - mkdist: 2.4.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)) + mkdist: 2.4.1(typescript@5.9.3)(vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)))(vue@3.5.35(typescript@5.9.3)) mlly: 1.8.0 pathe: 2.0.3 pkg-types: 2.3.0 @@ -36948,7 +36956,7 @@ snapshots: vitest@4.0.18(@opentelemetry/api@1.9.0)(@types/node@22.19.0)(jiti@2.7.0)(jsdom@26.1.0)(lightningcss@1.32.0)(terser@5.44.0)(tsx@4.20.6)(yaml@2.9.0): dependencies: '@vitest/expect': 4.0.18 - '@vitest/mocker': 4.0.18(vite@7.3.2(@types/node@22.19.0)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.44.0)(tsx@4.20.6)(yaml@2.9.0)) + '@vitest/mocker': 4.0.18(vite@7.3.2(@types/node@24.6.2)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.44.0)(tsx@4.20.6)(yaml@2.9.0)) '@vitest/pretty-format': 4.0.18 '@vitest/runner': 4.0.18 '@vitest/snapshot': 4.0.18 @@ -37140,11 +37148,11 @@ snapshots: '@vue/compiler-sfc': 3.5.35 vite: 7.3.5(@types/node@22.19.0)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.44.0)(tsx@4.20.6)(yaml@2.9.0) - vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.27.7)(vue@3.5.35(typescript@5.9.3)): + vue-sfc-transformer@0.1.17(@vue/compiler-core@3.5.35)(esbuild@0.28.0)(vue@3.5.35(typescript@5.9.3)): dependencies: '@babel/parser': 7.28.5 '@vue/compiler-core': 3.5.35 - esbuild: 0.27.7 + esbuild: 0.28.0 vue: 3.5.35(typescript@5.9.3) vue@3.5.30(typescript@5.9.3):