From 6f581131dea68d9d58e5857e710761dca0ccc680 Mon Sep 17 00:00:00 2001 From: yao Date: Fri, 15 May 2026 16:16:30 +0800 Subject: [PATCH 1/2] [nest] Use AST-based CommonJS import rewriting Signed-off-by: yao --- .changeset/nest-cjs-ast-rewrite.md | 5 + packages/nest/src/cjs-rewrite.test.ts | 145 ++++++++++--- packages/nest/src/cjs-rewrite.ts | 295 ++++++++++++++++++++++---- 3 files changed, 379 insertions(+), 66 deletions(-) create mode 100644 .changeset/nest-cjs-ast-rewrite.md diff --git a/.changeset/nest-cjs-ast-rewrite.md b/.changeset/nest-cjs-ast-rewrite.md new file mode 100644 index 0000000000..82c2799fab --- /dev/null +++ b/.changeset/nest-cjs-ast-rewrite.md @@ -0,0 +1,5 @@ +--- +"@workflow/nest": patch +--- + +Use AST-based import rewriting for NestJS CommonJS workflow bundles. diff --git a/packages/nest/src/cjs-rewrite.test.ts b/packages/nest/src/cjs-rewrite.test.ts index c5289acd83..bb347c39f3 100644 --- a/packages/nest/src/cjs-rewrite.test.ts +++ b/packages/nest/src/cjs-rewrite.test.ts @@ -2,33 +2,8 @@ import { describe, expect, it } from 'vitest'; import { mapSourceToDistPath, rewriteTsImportsInContent, - TS_IMPORT_REGEX, } from './cjs-rewrite.js'; -describe('TS_IMPORT_REGEX', () => { - const testRegex = () => - new RegExp(TS_IMPORT_REGEX.source, TS_IMPORT_REGEX.flags); - - it('matches named imports from .ts files', () => { - const s = 'import { foo, bar } from "../src/services/helper.ts";'; - expect(testRegex().test(s)).toBe(true); - }); - - it('matches named imports from .tsx files', () => { - const s = 'import { foo } from "./components/Widget.tsx";'; - expect(testRegex().test(s)).toBe(true); - }); - - it('matches imports with "as" alias', () => { - const s = 'import { hasValue as hv } from "../utils.ts";'; - expect(testRegex().test(s)).toBe(true); - }); - - it('does not match imports from node_modules', () => { - expect(testRegex().test('import { x } from "@workflow/core";')).toBe(false); - }); -}); - describe('rewriteTsImportsInContent', () => { const opts = { outDir: '/proj/.nestjs/workflow', @@ -53,6 +28,27 @@ describe('rewriteTsImportsInContent', () => { expect(result).toMatch(/\bfoo\b.*\bbar\b/); }); + it('rewrites multiline named imports', () => { + const content = [ + 'import {', + ' foo,', + ' bar as renamedBar,', + '} from "../../src/services/helper.ts";', + 'const x = foo;', + ].join('\n'); + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toContain( + 'const { foo, bar: renamedBar } = require("../../dist/services/helper.js");' + ); + expect(result).toContain('const x = foo;'); + }); + it('rewrites imports with "as" alias', () => { const content = 'import { hasValue as hv } from "../../src/utils.ts";'; @@ -66,6 +62,93 @@ describe('rewriteTsImportsInContent', () => { expect(result).toContain('require("../../dist/utils.js")'); }); + it('rewrites default imports through a module binding', () => { + const content = 'import helper from "../../src/services/helper.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toContain( + 'const __workflow_cjs_import_0 = require("../../dist/services/helper.js");' + ); + expect(result).toContain( + 'const helper = __workflow_cjs_import_0 != null && Object.prototype.hasOwnProperty.call(__workflow_cjs_import_0, "default") ? __workflow_cjs_import_0.default : __workflow_cjs_import_0;' + ); + }); + + it('rewrites mixed default and named imports', () => { + const content = + 'import helper, { foo, bar as renamedBar } from "../../src/services/helper.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toContain( + 'const __workflow_cjs_import_0 = require("../../dist/services/helper.js");' + ); + expect(result).toContain( + 'const helper = __workflow_cjs_import_0 != null && Object.prototype.hasOwnProperty.call(__workflow_cjs_import_0, "default") ? __workflow_cjs_import_0.default : __workflow_cjs_import_0;' + ); + expect(result).toContain( + 'const { foo, bar: renamedBar } = __workflow_cjs_import_0;' + ); + }); + + it('rewrites namespace imports', () => { + const content = 'import * as helper from "../../src/services/helper.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toBe( + 'const __workflow_cjs_import_0 = require("../../dist/services/helper.js");\n' + + 'const helper = __workflow_cjs_import_0;' + ); + }); + + it('rewrites side-effect imports', () => { + const content = 'import "../../src/setup.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toBe('require("../../dist/setup.js");'); + }); + + it('rewrites imports after non-ascii content', () => { + const content = [ + '// 你好', + 'import { foo } from "../../src/services/helper.ts";', + 'const x = foo;', + ].join('\n'); + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toBe( + [ + '// 你好', + 'const { foo } = require("../../dist/services/helper.js");', + 'const x = foo;', + ].join('\n') + ); + }); + it('handles .tsx files', () => { const content = 'import { Widget } from "../../src/components/Widget.tsx";'; @@ -89,6 +172,18 @@ describe('rewriteTsImportsInContent', () => { expect(result).toBe(content); }); + it('does not rewrite non-relative imports', () => { + const content = 'import { x } from "@workflow/core";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(0); + expect(result).toBe(content); + }); + it('rewrites multiple imports', () => { const content = [ 'import { a } from "../../src/a.ts";', diff --git a/packages/nest/src/cjs-rewrite.ts b/packages/nest/src/cjs-rewrite.ts index a53d891991..ee1588de36 100644 --- a/packages/nest/src/cjs-rewrite.ts +++ b/packages/nest/src/cjs-rewrite.ts @@ -1,21 +1,12 @@ -import { join, resolve, relative } from 'pathe'; - -/** - * Regex to match named imports from relative .ts/.tsx files (externalized by esbuild). - * esbuild's externalized output emits named imports (`import { ... } from "..."`). - * Namespace imports (import * as) and default imports are not emitted for externalized - * source files in this context. - */ -export const TS_IMPORT_REGEX = - /import\s*\{([^}]+)\}\s*from\s*["']((?:\.\.?\/)+[^"']+\.tsx?)["']\s*;?/g; +import { type ModuleItem, parseSync } from '@swc/core'; +import { join, relative, resolve } from 'pathe'; /** * Rewrite externalized .ts/.tsx imports in steps bundle content to use require() * for CommonJS compatibility. * * @returns Object with rewritten content and the number of imports rewritten. - * matchCount is 0 when no .ts/.tsx imports were found (valid when no externalized - * imports exist, or could indicate esbuild output format change). + * matchCount is 0 when no relative .ts/.tsx imports were found. */ export function rewriteTsImportsInContent( stepsContent: string, @@ -27,39 +18,44 @@ export function rewriteTsImportsInContent( } ): { content: string; matchCount: number } { const { outDir, workingDir, distDir, dirs } = options; - const countRegex = new RegExp(TS_IMPORT_REGEX.source, TS_IMPORT_REGEX.flags); - const matches: Array<{ fullMatch: string; imports: string; path: string }> = - []; - let m; - while ((m = countRegex.exec(stepsContent)) !== null) { - matches.push({ fullMatch: m[0], imports: m[1], path: m[2] }); - } + const module = parseSync(stepsContent, { + syntax: 'ecmascript', + target: 'es2022', + comments: false, + }); - if (matches.length === 0) { - return { content: stepsContent, matchCount: 0 }; - } + if (module.body.length === 0) return { content: stepsContent, matchCount: 0 }; - const replaceRegex = new RegExp( - TS_IMPORT_REGEX.source, - TS_IMPORT_REGEX.flags - ); - const rewritten = stepsContent.replace( - replaceRegex, - (_match, imports: string, tsRelativePath: string) => { - const absSourcePath = resolve(outDir, tsRelativePath); - const relToWorkingDir = relative(workingDir, absSourcePath); - const distRelPath = mapSourceToDistPath(relToWorkingDir, dirs, distDir); - const distAbsPath = join(workingDir, distRelPath); - let newRelPath = relative(outDir, distAbsPath).replace(/\\/g, '/'); - if (!newRelPath.startsWith('.')) { - newRelPath = `./${newRelPath}`; - } - const cjsImports = imports.replace(/\s+as\s+/g, ': '); - return `const {${cjsImports}} = require("${newRelPath}");`; - } + // SWC spans are global across parse calls and skip leading comments. + const toStringIndex = createBytePositionMapper( + stepsContent, + module.span.start - getLeadingSyntaxByteOffset(stepsContent) - 1 ); - return { content: rewritten, matchCount: matches.length }; + const replacements = module.body.flatMap((item, index) => { + const rewriteOptions = { + importIndex: index, + toStringIndex, + outDir, + workingDir, + distDir, + dirs, + }; + return getImportRewrite(item, rewriteOptions); + }); + + if (replacements.length === 0) + return { content: stepsContent, matchCount: 0 }; + + let rewritten = stepsContent; + for (const replacement of [...replacements].reverse()) { + rewritten = + rewritten.slice(0, replacement.start) + + replacement.code + + rewritten.slice(replacement.end); + } + + return { content: rewritten, matchCount: replacements.length }; } /** @@ -88,3 +84,220 @@ export function mapSourceToDistPath( return join(distDir, normalized).replace(/\.tsx?$/, '.js'); } + +type ImportRewriteOptions = { + importIndex: number; + toStringIndex: (swcBytePosition: number) => number; + outDir: string; + workingDir: string; + distDir: string; + dirs: string[]; +}; + +type Replacement = { + start: number; + end: number; + code: string; +}; + +function getImportRewrite( + item: ModuleItem, + options: ImportRewriteOptions +): Replacement[] { + if (item.type !== 'ImportDeclaration') return []; + + const source = item.source.value; + if (!isRelativeTypeScriptImport(source)) return []; + + const requirePath = getRequirePath(source, options); + const code = importDeclarationToRequire( + item, + requirePath, + options.importIndex + ); + + return [ + { + start: options.toStringIndex(item.span.start), + end: options.toStringIndex(item.span.end), + code, + }, + ]; +} + +function isRelativeTypeScriptImport(source: string): boolean { + return ( + (source.startsWith('./') || source.startsWith('../')) && + /\.tsx?$/.test(source) + ); +} + +function getRequirePath( + tsRelativePath: string, + { outDir, workingDir, distDir, dirs }: ImportRewriteOptions +): string { + const absSourcePath = resolve(outDir, tsRelativePath); + const relToWorkingDir = relative(workingDir, absSourcePath); + const distRelPath = mapSourceToDistPath(relToWorkingDir, dirs, distDir); + const distAbsPath = join(workingDir, distRelPath); + let newRelPath = relative(outDir, distAbsPath).replace(/\\/g, '/'); + if (!newRelPath.startsWith('.')) { + newRelPath = `./${newRelPath}`; + } + return newRelPath; +} + +function importDeclarationToRequire( + declaration: Extract, + requirePath: string, + importIndex: number +): string { + const requireCall = `require(${JSON.stringify(requirePath)})`; + const bindings = getImportBindings(declaration); + + if (declaration.specifiers.length === 0) { + return `${requireCall};`; + } + + if (bindings.defaultLocal || bindings.namespaceLocal) { + return moduleBindingToRequire(bindings, requireCall, importIndex); + } + + return `const { ${bindings.namedProperties.join(', ')} } = ${requireCall};`; +} + +type ImportBindings = { + defaultLocal?: string; + namespaceLocal?: string; + namedProperties: string[]; +}; + +function getImportBindings( + declaration: Extract +): ImportBindings { + const bindings: ImportBindings = { namedProperties: [] }; + + for (const specifier of declaration.specifiers) { + if (specifier.type === 'ImportDefaultSpecifier') + bindings.defaultLocal = specifier.local.value; + if (specifier.type === 'ImportNamespaceSpecifier') + bindings.namespaceLocal = specifier.local.value; + if (specifier.type === 'ImportSpecifier') { + bindings.namedProperties.push(getNamedProperty(specifier)); + } + } + + return bindings; +} + +function getNamedProperty( + specifier: Extract< + Extract['specifiers'][number], + { type: 'ImportSpecifier' } + > +): string { + const imported = + specifier.imported?.type === 'StringLiteral' + ? JSON.stringify(specifier.imported.value) + : (specifier.imported?.value ?? specifier.local.value); + const local = specifier.local.value; + return imported === local ? local : `${imported}: ${local}`; +} + +function moduleBindingToRequire( + bindings: ImportBindings, + requireCall: string, + importIndex: number +): string { + const moduleBinding = `__workflow_cjs_import_${importIndex}`; + const statements = [`const ${moduleBinding} = ${requireCall};`]; + + if (bindings.defaultLocal) { + statements.push( + defaultBindingToRequire(bindings.defaultLocal, moduleBinding) + ); + } + if (bindings.namespaceLocal) { + statements.push(`const ${bindings.namespaceLocal} = ${moduleBinding};`); + } + if (bindings.namedProperties.length > 0) { + statements.push( + `const { ${bindings.namedProperties.join(', ')} } = ${moduleBinding};` + ); + } + + return statements.join('\n'); +} + +function defaultBindingToRequire( + localName: string, + moduleBinding: string +): string { + return ( + `const ${localName} = ${moduleBinding} != null && ` + + `Object.prototype.hasOwnProperty.call(${moduleBinding}, "default") ` + + `? ${moduleBinding}.default : ${moduleBinding};` + ); +} + +function createBytePositionMapper( + source: string, + swcBasePosition: number +): (swcBytePosition: number) => number { + const byteOffsetToStringIndex = new Map(); + const encoder = new TextEncoder(); + let byteOffset = 0; + + for (let index = 0; index < source.length; ) { + byteOffsetToStringIndex.set(byteOffset, index); + const codePoint = source.codePointAt(index); + if (codePoint === undefined) break; + const character = String.fromCodePoint(codePoint); + byteOffset += encoder.encode(character).byteLength; + index += character.length; + } + byteOffsetToStringIndex.set(byteOffset, source.length); + + return (swcBytePosition: number) => { + // SWC positions are 1-based UTF-8 byte offsets. + const byteOffset = swcBytePosition - swcBasePosition - 1; + const stringIndex = byteOffsetToStringIndex.get(byteOffset); + if (stringIndex === undefined) { + throw new Error( + `Unable to map SWC byte position ${swcBytePosition} to a string index` + ); + } + return stringIndex; + }; +} + +function getLeadingSyntaxByteOffset(source: string): number { + let index = source.startsWith('#!') ? skipLineComment(source, 0) : 0; + + while (index < source.length) { + if (/\s/.test(source[index])) { + index += 1; + continue; + } + + if (source.startsWith('//', index)) { + index = skipLineComment(source, index); + continue; + } + + if (source.startsWith('/*', index)) { + const commentEnd = source.indexOf('*/', index + 2); + index = commentEnd === -1 ? source.length : commentEnd + 2; + continue; + } + + break; + } + + return new TextEncoder().encode(source.slice(0, index)).byteLength; +} + +function skipLineComment(source: string, index: number): number { + const newlineIndex = source.indexOf('\n', index); + return newlineIndex === -1 ? source.length : newlineIndex + 1; +} From de28f1b5ceef1ada93cc5c1fdd7d5c5aaa985c92 Mon Sep 17 00:00:00 2001 From: Peter Wielander Date: Fri, 22 May 2026 11:45:23 +0200 Subject: [PATCH 2/2] [nest] Fix shebang handling and document byte-position math MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SWC's `module.span.start` keeps the leading shebang line inside the module span but excludes leading line/block comments and a BOM. The previous `getLeadingSyntaxByteOffset` skipped the shebang, which made `module.span.start - leadingOffset - 1` shift every subsequent byte→string mapping. A steps bundle that starts with `#!/usr/bin/env node` would either throw `Unable to map SWC byte position X to a string index` or silently corrupt the rewrite. - Rename `getLeadingSyntaxByteOffset` → `getLeadingCommentByteOffset` and drop the shebang branch. - Document the invariant tying SWC's 1-based global byte counter to source-relative offsets so a future change to SWC's span behavior doesn't silently break the rewriter. - Add test coverage for shebang, shebang+comment, BOM, CRLF, dynamic imports, empty named imports, `default as`, mixed default+namespace, duplicate imports of the same module, and template-literal substrings. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/nest/src/cjs-rewrite.test.ts | 159 ++++++++++++++++++++++++++ packages/nest/src/cjs-rewrite.ts | 30 ++++- 2 files changed, 185 insertions(+), 4 deletions(-) diff --git a/packages/nest/src/cjs-rewrite.test.ts b/packages/nest/src/cjs-rewrite.test.ts index bb347c39f3..80019ae51c 100644 --- a/packages/nest/src/cjs-rewrite.test.ts +++ b/packages/nest/src/cjs-rewrite.test.ts @@ -199,6 +199,165 @@ describe('rewriteTsImportsInContent', () => { expect(result).toContain('require("../../dist/a.js")'); expect(result).toContain('require("../../dist/b.js")'); }); + + it('handles a leading shebang line', () => { + const content = [ + '#!/usr/bin/env node', + 'import { foo } from "../../src/services/helper.ts";', + 'const x = foo;', + ].join('\n'); + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toBe( + [ + '#!/usr/bin/env node', + 'const { foo } = require("../../dist/services/helper.js");', + 'const x = foo;', + ].join('\n') + ); + }); + + it('handles a shebang followed by a leading comment', () => { + const content = [ + '#!/usr/bin/env node', + '// banner', + 'import { foo } from "../../src/services/helper.ts";', + ].join('\n'); + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toBe( + [ + '#!/usr/bin/env node', + '// banner', + 'const { foo } = require("../../dist/services/helper.js");', + ].join('\n') + ); + }); + + it('handles a UTF-8 BOM at the start of the file', () => { + const content = 'import { foo } from "../../src/services/helper.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toBe( + 'const { foo } = require("../../dist/services/helper.js");' + ); + }); + + it('handles CRLF line endings', () => { + const content = + '// banner\r\nimport { foo } from "../../src/services/helper.ts";\r\nconst x = foo;'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toBe( + '// banner\r\nconst { foo } = require("../../dist/services/helper.js");\r\nconst x = foo;' + ); + }); + + it('does not rewrite dynamic imports', () => { + const content = 'const m = import("../../src/dynamic.ts");'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(0); + expect(result).toBe(content); + }); + + it('rewrites empty named imports as a side-effect require', () => { + const content = 'import {} from "../../src/setup.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toContain('require("../../dist/setup.js")'); + }); + + it('rewrites a `default as` named import', () => { + const content = + 'import { default as helper } from "../../src/services/helper.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toContain( + 'const { default: helper } = require("../../dist/services/helper.js");' + ); + }); + + it('rewrites combined default + namespace imports', () => { + const content = + 'import helper, * as helperNs from "../../src/services/helper.ts";'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(1); + expect(result).toContain( + 'const __workflow_cjs_import_0 = require("../../dist/services/helper.js");' + ); + expect(result).toContain( + 'const helper = __workflow_cjs_import_0 != null && Object.prototype.hasOwnProperty.call(__workflow_cjs_import_0, "default") ? __workflow_cjs_import_0.default : __workflow_cjs_import_0;' + ); + expect(result).toContain('const helperNs = __workflow_cjs_import_0;'); + }); + + it('rewrites two imports of the same module without name collisions', () => { + const content = [ + 'import a from "../../src/x.ts";', + 'import b from "../../src/x.ts";', + ].join('\n'); + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(2); + expect(result).toContain('__workflow_cjs_import_0'); + expect(result).toContain('__workflow_cjs_import_1'); + }); + + it('does not rewrite an import-shaped substring inside a template literal', () => { + const content = 'const s = `import { foo } from "../../src/x.ts";`;'; + + const { content: result, matchCount } = rewriteTsImportsInContent( + content, + opts + ); + + expect(matchCount).toBe(0); + expect(result).toBe(content); + }); }); describe('mapSourceToDistPath', () => { diff --git a/packages/nest/src/cjs-rewrite.ts b/packages/nest/src/cjs-rewrite.ts index ee1588de36..6355083fa8 100644 --- a/packages/nest/src/cjs-rewrite.ts +++ b/packages/nest/src/cjs-rewrite.ts @@ -26,10 +26,24 @@ export function rewriteTsImportsInContent( if (module.body.length === 0) return { content: stepsContent, matchCount: 0 }; - // SWC spans are global across parse calls and skip leading comments. + // SWC's `span.start` is a 1-based UTF-8 byte offset into a global counter + // shared across all parseSync calls in the process (each parse leaves the + // cursor at the end of the previous source). For a freshly-parsed source, + // `module.span.start` points to the first byte SWC considers part of the + // module's text — which empirically: + // • starts BEFORE leading line/block comments and a leading BOM + // (SWC skips those out of `module.span`), but + // • starts AT a leading shebang line (SWC keeps the shebang inside the + // module span and exposes its text via `module.interpreter`). + // We want a base such that for any token, `token.span.start - base - 1` + // equals the local UTF-8 byte offset of that token within `stepsContent`. + // Subtracting the byte-length of leading comments/BOM (but NOT the + // shebang) from `module.span.start` gives us the SWC position + // corresponding to source byte 0, and the trailing `- 1` converts the + // 1-based offset into a 0-based base. const toStringIndex = createBytePositionMapper( stepsContent, - module.span.start - getLeadingSyntaxByteOffset(stepsContent) - 1 + module.span.start - getLeadingCommentByteOffset(stepsContent) - 1 ); const replacements = module.body.flatMap((item, index) => { @@ -271,8 +285,16 @@ function createBytePositionMapper( }; } -function getLeadingSyntaxByteOffset(source: string): number { - let index = source.startsWith('#!') ? skipLineComment(source, 0) : 0; +/** + * Compute the UTF-8 byte length of any leading content that SWC excludes from + * `module.span` — leading whitespace (including a BOM), `//` line comments, + * and `/* … *\/` block comments. + * + * A leading shebang line is intentionally NOT skipped: SWC keeps the shebang + * inside `module.span`, so this helper must return 0 for it. + */ +function getLeadingCommentByteOffset(source: string): number { + let index = 0; while (index < source.length) { if (/\s/.test(source[index])) {