From 76cf10a9d135be39923fb1380af15958ffba1578 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 15 Jul 2026 20:00:45 +0200 Subject: [PATCH 01/18] feat: add portable bundler wrapper API Bundler plugins need to own virtual module IDs and resolution. Returning import metadata separately keeps canonical module URLs out of generated import specifiers without duplicating IITM's export-binding logic. Refs: https://github.com/DataDog/dd-trace-js/issues/9383 --- README.md | 30 +++ bundler.d.mts | 60 ++++++ bundler.mjs | 175 ++++++++++++++++++ create-hook.mjs | 289 +---------------------------- lib/bundler-runtime.js | 9 + lib/get-exports.mjs | 13 +- lib/wrapper.mjs | 304 +++++++++++++++++++++++++++++++ test/low-level/bundler.mjs | 241 ++++++++++++++++++++++++ test/typescript/bundler.test.mts | 22 +++ 9 files changed, 857 insertions(+), 286 deletions(-) create mode 100644 bundler.d.mts create mode 100644 bundler.mjs create mode 100644 lib/bundler-runtime.js create mode 100644 lib/wrapper.mjs create mode 100644 test/low-level/bundler.mjs create mode 100644 test/typescript/bundler.test.mts diff --git a/README.md b/README.md index eae9f47e..55d5a334 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,36 @@ fs.readFileSync('file.txt') node --import=./instrument.mjs ./my-app.mjs ``` +## Bundler integrations + +Bundlers can generate the same live-binding wrapper as the Node.js loader with +`createWrapperModule`: + +```js +import { createWrapperModule } from 'import-in-the-middle/bundler.mjs' + +const wrapper = await createWrapperModule({ + module: { url, format, source, specifier }, + resolve, + load +}) +``` + +`url` is the canonical `file:` or `node:` URL reported to hooks. `resolve` and +`load` adapt the bundler's resolver and source loader to the same URL-based +module graph. + +The result contains generated `code`, an `imports` manifest, `watchFiles`, and +`sideEffects: true`. The code imports only relative placeholder specifiers. The +bundler adapter provides it as a virtual module and maps each placeholder using +the manifest, so filesystem paths, virtual IDs, external modules, and cache +invalidation remain owned by the bundler. `watchFiles` are file URLs that the +adapter converts to its native watch-dependency format. + +The runtime import in the manifest must be bundled with the wrapper. Keeping it +external can create a second hook registry at runtime. It is CommonJS and must +go through the bundler's normal CommonJS transform. + ## Synchronous loader hooks On Node.js versions that support diff --git a/bundler.d.mts b/bundler.d.mts new file mode 100644 index 00000000..a589bbf6 --- /dev/null +++ b/bundler.d.mts @@ -0,0 +1,60 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. + +export type WrapperSource = string | ArrayBuffer | ArrayBufferView + +export type BundlerModule = { + url: string + format: string + specifier: string + source?: WrapperSource +} + +export type ModuleContext = { + format?: string + parentURL?: string +} + +export type ModuleTarget = { + url: string + format?: string +} + +export type ResolveResult = ModuleTarget & { + watchFiles?: Iterable +} + +export type LoadResult = { + source?: WrapperSource + format?: string + watchFiles?: Iterable +} + +export type WrapperImport = { + specifier: string + kind: 'module' | 'runtime' + target: ModuleTarget + external: boolean +} + +export type WrapperModule = { + code: string + imports: WrapperImport[] + watchFiles: string[] + sideEffects: true +} + +export type CreateWrapperModuleOptions = { + module: BundlerModule + resolve: ( + specifier: string, + context: ModuleContext + ) => ResolveResult | Promise + load: ( + url: string, + context: ModuleContext + ) => LoadResult | Promise +} + +export declare function createWrapperModule(options: CreateWrapperModuleOptions): Promise diff --git a/bundler.mjs b/bundler.mjs new file mode 100644 index 00000000..681446a3 --- /dev/null +++ b/bundler.mjs @@ -0,0 +1,175 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. + +'use strict' + +import { builtinModules } from 'module' + +import { driveAsync } from './lib/io.mjs' +import { buildWrapperSource, processModule } from './lib/wrapper.mjs' + +const RUNTIME_SPECIFIER = './__iitm_runtime__.js' +const MODULE_SPECIFIER_PREFIX = './__iitm_module_' +const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href + +/** + * @typedef {object} BundlerModule + * @property {string} url + * @property {string} format + * @property {string} specifier + * @property {string | ArrayBuffer | ArrayBufferView} [source] + */ + +/** + * @typedef {object} ModuleContext + * @property {string} [format] + * @property {string} [parentURL] + */ + +/** + * @typedef {object} ResolveResult + * @property {string} url + * @property {string} [format] + * @property {Iterable} [watchFiles] + */ + +/** + * @typedef {object} WrapperImport + * @property {string} specifier + * @property {'module' | 'runtime'} kind + * @property {{ url: string, format?: string }} target + * @property {boolean} external + */ + +/** + * @typedef {object} LoadResult + * @property {string | ArrayBuffer | ArrayBufferView} [source] + * @property {string} [format] + * @property {Iterable} [watchFiles] + */ + +/** + * Creates an ESM wrapper without embedding bundler-specific module identifiers. + * + * @param {object} options + * @param {BundlerModule} options.module + * @param {(specifier: string, context: ModuleContext) => + * (ResolveResult | Promise)} options.resolve + * @param {(url: string, context: ModuleContext) => (LoadResult | Promise)} options.load + * @returns {Promise<{ + * code: string, + * imports: WrapperImport[], + * watchFiles: string[], + * sideEffects: true + * }>} + */ +export async function createWrapperModule ({ module: moduleData, resolve, load }) { + const context = { format: moduleData.format, cache: false } + const watchFiles = new Set() + const formats = new Map([[moduleData.url, moduleData.format]]) + + if (moduleData.url.startsWith('file:')) { + watchFiles.add(moduleData.url) + } + + /** + * @param {string} url + * @param {ModuleContext} loadContext + * @returns {Promise} + */ + const loadModule = async (url, loadContext) => { + if (url === moduleData.url && moduleData.source !== undefined) { + return { + source: moduleData.source, + format: moduleData.format + } + } + + const result = await load(url, loadContext) + if (result.format !== undefined) { + formats.set(url, result.format) + } + if (url.startsWith('file:')) { + watchFiles.add(url) + } + if (result.watchFiles !== undefined) { + for (const watchFile of result.watchFiles) { + watchFiles.add(watchFile) + } + } + return result + } + + /** + * @param {string} specifier + * @param {ModuleContext} resolveContext + * @returns {Promise} + */ + const resolveModule = async (specifier, resolveContext) => { + const result = await resolve(specifier, resolveContext) + if (result.format !== undefined) { + formats.set(result.url, result.format) + } + if (result.watchFiles !== undefined) { + for (const watchFile of result.watchFiles) { + watchFiles.add(watchFile) + } + } + return result + } + + const { bindings } = await driveAsync( + processModule({ srcUrl: moduleData.url, context }), + { resolve: resolveModule, load: loadModule } + ) + + /** @type {WrapperImport[]} */ + const imports = [{ + specifier: RUNTIME_SPECIFIER, + kind: 'runtime', + target: { + url: runtimeUrl, + format: 'commonjs' + }, + external: false + }] + const moduleSpecifiers = new Map() + + /** + * @param {string} url + * @returns {string} + */ + const mapImport = (url) => { + let specifier = moduleSpecifiers.get(url) + if (specifier === undefined) { + specifier = `${MODULE_SPECIFIER_PREFIX}${moduleSpecifiers.size}__.js` + moduleSpecifiers.set(url, specifier) + imports.push({ + specifier, + kind: 'module', + target: { + url, + format: formats.get(url) + }, + external: url.startsWith('node:') || builtinModules.includes(url) + }) + } + return specifier + } + + const code = buildWrapperSource({ + realUrl: moduleData.url, + bindings, + originalSpecifier: moduleData.specifier, + runtimeSpecifier: RUNTIME_SPECIFIER, + mapImport + }) + + return { + code, + imports, + watchFiles: Array.from(watchFiles), + sideEffects: true + } +} diff --git a/create-hook.mjs b/create-hook.mjs index 1ca53840..b4b8badb 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -5,8 +5,8 @@ import { URL, fileURLToPath } from 'url' import { inspect } from 'util' import { builtinModules } from 'module' -import { getExports } from './lib/get-exports.mjs' -import { RESOLVE, driveSync, driveAsync } from './lib/io.mjs' +import { driveSync, driveAsync } from './lib/io.mjs' +import { buildWrapperSource, processModule } from './lib/wrapper.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' // Re-exported for backwards compatibility: `supportsSyncHooks` now lives in its @@ -16,12 +16,6 @@ export { supportsSyncHooks } const isWin = process.platform === 'win32' -// Depth at which `processModule` starts tracking visited URLs to break an -// `export *` cycle. Real re-export chains are only a few levels deep, so this -// is far beyond any legitimate graph yet well below the call-stack limit a -// cycle would otherwise hit. Below it the recursion pays only an integer -// compare per level and allocates no set. -const STAR_CYCLE_DEPTH = 100 // FIXME: Typescript extensions are added temporarily until we find a better // way of supporting arbitrary extensions @@ -36,12 +30,6 @@ const TRACE_WARNINGS = process.execArgv.includes('--trace-warnings') /** @typedef {import('node:module').LoadHookContext} LoadContext */ /** @typedef {import('node:module').LoadFnOutput} LoadResult */ /** @typedef {string | { specifier: string, format: 'module-typescript' | 'commonjs-typescript' }} SpecifierData */ -/** @typedef {{ name: string, origin: string }} StarBinding */ -/** - * @typedef {object} ProcessResult - * @property {string[] | Map} bindings - * @property {Map | undefined} origins - */ function hasIitm (url) { // Fast path: avoid URL parsing on the hot path when there's clearly no iitm. @@ -85,33 +73,6 @@ function deleteIitm (url) { return resultUrl } -function isBareSpecifier (specifier) { - // Relative and absolute paths are not bare specifiers. - if ( - specifier.startsWith('.') || - specifier.startsWith('/')) { - return false - } - - // Valid URLs are not bare specifiers. (file:, http:, node:, etc.) - - // eslint-disable-next-line no-prototype-builtins - if (URL.hasOwnProperty('canParse')) { - return !URL.canParse(specifier) - } - - const stackTraceLimit = Error.stackTraceLimit - try { - Error.stackTraceLimit = 0 - // eslint-disable-next-line no-new - new URL(specifier) - return false - } catch (err) { - return true - } finally { - Error.stackTraceLimit = stackTraceLimit - } -} /** * Determines whether the input is a bare specifier, file URL or a regular expression. @@ -182,167 +143,6 @@ function emitWarning (err) { process.emitWarning(warnMessage) } -/** - * @param {string} name The exported name. - * @param {string} sourceUrl The URL of the module that defines the export. - */ -function shouldReexport (name, sourceUrl) { - return name !== 'module.exports' || - (!sourceUrl.startsWith('node:') && !builtinModules.includes(sourceUrl)) -} - -/** - * @param {string} name The exported name. - * @param {string} sourceUrl The URL of the module that defines the export. - */ -function shouldExcludeExport (name, sourceUrl) { - return name === 'default' || !shouldReexport(name, sourceUrl) -} - -/** - * Processes a module's exports and builds its wrapper bindings. - * - * Written as a "sans-io" generator (see `lib/io.mjs`): instead of calling the - * loader's resolve/load hooks directly it `yield`s `[RESOLVE, ...]` to resolve - * star re-exports and `[LOAD, ...]` (via {@link getExports}) to read source, - * and is driven by either {@link driveSync} (for - * `module.registerHooks`) or {@link driveAsync} (for `module.register`). The - * body is identical for both, so there is a single implementation to maintain. - * - * @param {object} params - * @param {string} params.srcUrl The full URL to the module to process. - * @param {LoadContext} params.context Provided by the loaders API. - * @param {boolean} [params.excludeDefault = false] Exclude the default export. - * @param {number} [params.depth = 0] Star-re-export recursion depth. Used to - * detect `export *` cycles (`a` re-exports `b`, `b` re-exports `a`) cheaply: - * the acyclic common case pays only an integer compare per level, and the - * cycle-tracking set is allocated only once recursion is implausibly deep. - * @param {Set} [params.seen] URLs currently on the recursion stack, - * created lazily once `depth` crosses {@link STAR_CYCLE_DEPTH}. A URL is added - * before descending into its subtree and removed once that subtree finishes, so - * it tracks the active path rather than every URL ever visited. - * @returns {Generator} - * A generator that yields I/O operations and ultimately returns the shimmed - * bindings for all the exports from the module and any transitive export all - * modules. `origins` (the defining module per `*`-sourced name) is `undefined` - * for a module with no `export *`. - */ -function * processModule ({ srcUrl, context, excludeDefault = false, depth = 0, seen }) { - const { exportNames, starReexports } = yield * getExports(srcUrl, context) - - // Most modules have no export star. Keep that path array-backed so it pays - // neither merge bookkeeping nor a Map lookup for each direct export. - if (starReexports === undefined) { - if (!excludeDefault) { - return { bindings: exportNames, origins: undefined } - } - - const bindings = [] - for (const name of exportNames) { - if (shouldExcludeExport(name, srcUrl)) continue - bindings.push(name) - } - return { bindings, origins: undefined } - } - - const bindings = new Map() - - // Maps each live `*`-sourced name to the module that defined it. Its keys - // double as "this name came from a `*` re-export" (so an explicit export can - // override it), and its values let two `*` re-exports of the same name be told - // apart. Allocated on the first `export *`, never for a module without one; a - // single Map carries both facts so a star with no collision pays one structure - // and one write per name, not two. - let starOrigins - let ambiguousStars - let firstStarUrl - let processedStarUrls - - for (const name of exportNames) { - if (excludeDefault && shouldExcludeExport(name, srcUrl)) continue - bindings.set(name, name) - } - - for (const { specifier, parentURL } of starReexports) { - // Relative paths need to be resolved relative to the module declaring the star. - const newSpecifier = isBareSpecifier(specifier) ? specifier : new URL(specifier, parentURL).href - // We need to resolve bare specifiers to a full URL. We also need to - // resolve all sub-modules to get the `format`. We can't rely on the - // parent's `format` to know if this sub-module is ESM or CJS! - const result = yield [RESOLVE, newSpecifier, { parentURL }] - - // Most star modules have one target. Defer the collection until a second - // distinct target while still ignoring repeated declarations. - if (firstStarUrl === undefined) { - firstStarUrl = result.url - } else if (processedStarUrls === undefined) { - if (result.url === firstStarUrl) continue - processedStarUrls = [firstStarUrl, result.url] - } else { - if (processedStarUrls.includes(result.url)) continue - processedStarUrls.push(result.url) - } - - // First `*` re-export: allocate the origin bookkeeping lazily. - starOrigins ??= new Map() - - // `export *` graphs are normally only a handful of levels deep. A cycle - // (`a` re-exports `b`, `b` re-exports `a`) instead recurses without bound - // and exhausts memory. Rather than track every URL on the common shallow - // path, only start recording once the depth is implausibly large for a - // real graph; from there a re-export pointing back at a module already on - // the recursion stack is the cycle, and is skipped (its exports are - // collected by the in-progress ancestor frame). `seen` mirrors the stack, - // not every URL visited: a module reached and fully processed through one - // sibling branch must stay reachable through a later, more direct branch, - // so it is removed again once its subtree finishes. - if (depth >= STAR_CYCLE_DEPTH) { - seen ??= new Set() - if (seen.has(result.url)) continue - seen.add(result.url) - } - - try { - const sub = yield * processModule({ - srcUrl: result.url, - context: { ...context, format: result.format }, - excludeDefault: true, - depth: depth + 1, - seen - }) - - for (const binding of sub.bindings.values()) { - const directName = typeof binding === 'string' ? binding : undefined - const name = directName ?? binding.name - if (ambiguousStars?.has(name)) continue - - const origin = directName === undefined ? binding.origin : sub.origins?.get(name) ?? result.url - if (bindings.has(name)) { - // An explicit export shadows every star re-export. - if (!starOrigins.has(name)) continue - - if (starOrigins.get(name) === origin) { - // IITM's aggregate namespace sees the wrapped paths as ambiguous. - // Retain the defining URL so source generation can import it once. - bindings.set(name, { name, origin }) - } else { - bindings.delete(name) - starOrigins.delete(name) - ambiguousStars ??= new Set() - ambiguousStars.add(name) - } - } else { - starOrigins.set(name, origin) - bindings.set(name, binding) - } - } - } finally { - seen?.delete(result.url) - } - } - - return { bindings, origins: starOrigins } -} function addIitm (url) { const urlObj = new URL(url) @@ -594,84 +394,6 @@ export function createHook (meta) { return finishResolve(result, specifier, context, parentURL) } - /** - * Builds the wrapper module source shared by the asynchronous and synchronous hooks. - * - * @param {string} realUrl The URL of the wrapped module. - * @param {string[] | Map} bindings Its exported bindings. - * @param {string} originalSpecifier The specifier used to import the module. - */ - function buildWrapperSource (realUrl, bindings, originalSpecifier) { - // The wrapped module imports its namespace as `namespace`, which serves - // every export but the ones a same-origin `export *` collision forced onto - // their defining module (#171): the aggregate namespace drops those as - // ambiguous under iitm, so each such defining module gets its own alias the - // wrapper imports. Without such a collision nothing is added. - let originImports = '' - let originNamespaces - let declarationNames = '' - let bindingNames = '' - let bindingSources - let exportSpecifiers = '' - let writeCases = '' - let index = 0 - for (const binding of bindings.values()) { - const directName = typeof binding === 'string' ? binding : undefined - const name = directName ?? binding.name - let namespaceName = 'namespace' - if (directName === undefined) { - originNamespaces ??= new Map() - namespaceName = originNamespaces.get(binding.origin) - if (namespaceName === undefined) { - namespaceName = `__ns${originNamespaces.size}` - originNamespaces.set(binding.origin, namespaceName) - originImports += `import * as ${namespaceName} from ${JSON.stringify(binding.origin)}\n` - } - } - const variableName = `$${index}` - const objectKey = JSON.stringify(name) - declarationNames += declarationNames === '' ? variableName : `, ${variableName}` - bindingNames += bindingNames === '' ? objectKey : `, ${objectKey}` - if (bindingSources !== undefined) bindingSources += ', ' - if (namespaceName !== 'namespace') { - bindingSources ??= 'undefined, '.repeat(index) - bindingSources += namespaceName - } else if (bindingSources !== undefined) { - bindingSources += 'undefined' - } - writeCases += ` case ${index++}: ${variableName} = value; break\n` - if (shouldReexport(name, realUrl)) { - const exportName = name === 'default' ? name : objectKey - exportSpecifiers += exportSpecifiers === '' - ? `${variableName} as ${exportName}` - : `, ${variableName} as ${exportName}` - } - } - const binder = declarationNames === '' - ? 'const __binder = new ModuleBinder(namespace)\n' - : `let ${declarationNames} -function __write (index, value) { - switch (index) { -${writeCases} } -} -const __binder = new ModuleBinder(namespace, [${bindingNames}], __write${bindingSources === undefined - ? '' - : `, [${bindingSources}]`}) -` - const reexports = exportSpecifiers === '' ? '' : `export { ${exportSpecifiers} }\n` - - return ` -import { register, ModuleBinder } from ${JSON.stringify(iitmURL)} -import * as namespace from ${JSON.stringify(realUrl)} -${originImports} -${binder} -${reexports} - -__binder.flush() - -register(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}) -` - } /** * Finalizes a successful wrap and builds its module source. @@ -687,7 +409,12 @@ register(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifie if (context.format === 'commonjs') { cjsInIitmChain.add(realUrl) } - return buildWrapperSource(realUrl, bindings, originalSpecifier) + return buildWrapperSource({ + realUrl, + bindings, + originalSpecifier, + runtimeSpecifier: iitmURL + }) } // Bookkeeping shared by the async and sync wrap paths when `processModule` diff --git a/lib/bundler-runtime.js b/lib/bundler-runtime.js new file mode 100644 index 00000000..396f7211 --- /dev/null +++ b/lib/bundler-runtime.js @@ -0,0 +1,9 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. + +'use strict' + +const { ModuleBinder } = require('./register.js') + +exports.ModuleBinder = ModuleBinder diff --git a/lib/get-exports.mjs b/lib/get-exports.mjs index c3090798..d396a54e 100644 --- a/lib/get-exports.mjs +++ b/lib/get-exports.mjs @@ -245,9 +245,12 @@ function * getCjsExports (url, context, source) { * module. */ export function * getExports (url, context) { - const cached = esmExportsCache.get(url) - if (cached !== undefined) { - return cached + const useCache = context.cache !== false + if (useCache) { + const cached = esmExportsCache.get(url) + if (cached !== undefined) { + return cached + } } // `[LOAD, ...]` gives us the possibility of getting the source from an @@ -303,7 +306,7 @@ export function * getExports (url, context) { const moduleExports = starReexports === undefined ? { exportNames } : { exportNames, starReexports } if (moduleFormat === 'module') { - esmExportsCache.set(url, moduleExports) + if (useCache) esmExportsCache.set(url, moduleExports) return moduleExports } @@ -314,7 +317,7 @@ export function * getExports (url, context) { if (exportNames.length === 0 && !hasModuleSyntax) { return yield * getCjsExports(url, context, source) } - esmExportsCache.set(url, moduleExports) + if (useCache) esmExportsCache.set(url, moduleExports) return moduleExports } catch (cause) { const err = new Error(`Failed to parse '${url}'`) diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs new file mode 100644 index 00000000..e4aaac06 --- /dev/null +++ b/lib/wrapper.mjs @@ -0,0 +1,304 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. + +'use strict' + +import { builtinModules } from 'module' +import { URL } from 'url' + +import { getExports } from './get-exports.mjs' +import { RESOLVE } from './io.mjs' + +// Depth at which `processModule` starts tracking visited URLs to break an +// `export *` cycle. Real re-export chains are only a few levels deep, so this +// is far beyond any legitimate graph yet well below the call-stack limit a +// cycle would otherwise hit. Below it the recursion pays only an integer +// compare per level and allocates no set. +const STAR_CYCLE_DEPTH = 100 + +/** @typedef {{ name: string, origin: string }} StarBinding */ +/** + * @typedef {object} ProcessResult + * @property {string[] | Map} bindings + * @property {Map | undefined} origins + */ + +function isBareSpecifier (specifier) { + // Relative and absolute paths are not bare specifiers. + if ( + specifier.startsWith('.') || + specifier.startsWith('/')) { + return false + } + + // Valid URLs are not bare specifiers. (file:, http:, node:, etc.) + + // eslint-disable-next-line no-prototype-builtins + if (URL.hasOwnProperty('canParse')) { + return !URL.canParse(specifier) + } + + const stackTraceLimit = Error.stackTraceLimit + try { + Error.stackTraceLimit = 0 + // eslint-disable-next-line no-new + new URL(specifier) + return false + } catch { + return true + } finally { + Error.stackTraceLimit = stackTraceLimit + } +} + +/** + * @param {string} name The exported name. + * @param {string} sourceUrl The URL of the module that defines the export. + */ +function shouldReexport (name, sourceUrl) { + return name !== 'module.exports' || + (!sourceUrl.startsWith('node:') && !builtinModules.includes(sourceUrl)) +} + +/** + * @param {string} name The exported name. + * @param {string} sourceUrl The URL of the module that defines the export. + */ +function shouldExcludeExport (name, sourceUrl) { + return name === 'default' || !shouldReexport(name, sourceUrl) +} + +/** + * Processes a module's exports and builds its wrapper bindings. + * + * Written as a "sans-io" generator (see `lib/io.mjs`): instead of calling the + * loader's resolve/load hooks directly it `yield`s `[RESOLVE, ...]` to resolve + * star re-exports and `[LOAD, ...]` (via {@link getExports}) to read source, + * and is driven by either {@link driveSync} (for + * `module.registerHooks`) or {@link driveAsync} (for `module.register`). The + * body is identical for both, so there is a single implementation to maintain. + * + * @param {object} params + * @param {string} params.srcUrl The full URL to the module to process. + * @param {LoadContext} params.context Provided by the loaders API. + * @param {boolean} [params.excludeDefault = false] Exclude the default export. + * @param {number} [params.depth = 0] Star-re-export recursion depth. Used to + * detect `export *` cycles (`a` re-exports `b`, `b` re-exports `a`) cheaply: + * the acyclic common case pays only an integer compare per level, and the + * cycle-tracking set is allocated only once recursion is implausibly deep. + * @param {Set} [params.seen] URLs currently on the recursion stack, + * created lazily once `depth` crosses {@link STAR_CYCLE_DEPTH}. A URL is added + * before descending into its subtree and removed once that subtree finishes, so + * it tracks the active path rather than every URL ever visited. + * @returns {Generator} + * A generator that yields I/O operations and ultimately returns the shimmed + * bindings for all the exports from the module and any transitive export all + * modules. `origins` (the defining module per `*`-sourced name) is `undefined` + * for a module with no `export *`. + */ +export function * processModule ({ srcUrl, context, excludeDefault = false, depth = 0, seen }) { + const { exportNames, starReexports } = yield * getExports(srcUrl, context) + + // Most modules have no export star. Keep that path array-backed so it pays + // neither merge bookkeeping nor a Map lookup for each direct export. + if (starReexports === undefined) { + if (!excludeDefault) { + return { bindings: exportNames, origins: undefined } + } + + const bindings = [] + for (const name of exportNames) { + if (shouldExcludeExport(name, srcUrl)) continue + bindings.push(name) + } + return { bindings, origins: undefined } + } + + const bindings = new Map() + + // Maps each live `*`-sourced name to the module that defined it. Its keys + // double as "this name came from a `*` re-export" (so an explicit export can + // override it), and its values let two `*` re-exports of the same name be told + // apart. Allocated on the first `export *`, never for a module without one; a + // single Map carries both facts so a star with no collision pays one structure + // and one write per name, not two. + let starOrigins + let ambiguousStars + let firstStarUrl + let processedStarUrls + + for (const name of exportNames) { + if (excludeDefault && shouldExcludeExport(name, srcUrl)) continue + bindings.set(name, name) + } + + for (const { specifier, parentURL } of starReexports) { + // Relative paths need to be resolved relative to the module declaring the star. + const newSpecifier = isBareSpecifier(specifier) ? specifier : new URL(specifier, parentURL).href + // We need to resolve bare specifiers to a full URL. We also need to + // resolve all sub-modules to get the `format`. We can't rely on the + // parent's `format` to know if this sub-module is ESM or CJS! + const result = yield [RESOLVE, newSpecifier, { parentURL }] + + // Most star modules have one target. Defer the collection until a second + // distinct target while still ignoring repeated declarations. + if (firstStarUrl === undefined) { + firstStarUrl = result.url + } else if (processedStarUrls === undefined) { + if (result.url === firstStarUrl) continue + processedStarUrls = [firstStarUrl, result.url] + } else { + if (processedStarUrls.includes(result.url)) continue + processedStarUrls.push(result.url) + } + + // First `*` re-export: allocate the origin bookkeeping lazily. + starOrigins ??= new Map() + + // `export *` graphs are normally only a handful of levels deep. A cycle + // (`a` re-exports `b`, `b` re-exports `a`) instead recurses without bound + // and exhausts memory. Rather than track every URL on the common shallow + // path, only start recording once the depth is implausibly large for a + // real graph; from there a re-export pointing back at a module already on + // the recursion stack is the cycle, and is skipped (its exports are + // collected by the in-progress ancestor frame). `seen` mirrors the stack, + // not every URL visited: a module reached and fully processed through one + // sibling branch must stay reachable through a later, more direct branch, + // so it is removed again once its subtree finishes. + if (depth >= STAR_CYCLE_DEPTH) { + seen ??= new Set() + if (seen.has(result.url)) continue + seen.add(result.url) + } + + try { + const sub = yield * processModule({ + srcUrl: result.url, + context: { ...context, format: result.format }, + excludeDefault: true, + depth: depth + 1, + seen + }) + + for (const binding of sub.bindings.values()) { + const directName = typeof binding === 'string' ? binding : undefined + const name = directName ?? binding.name + if (ambiguousStars?.has(name)) continue + + const origin = directName === undefined ? binding.origin : sub.origins?.get(name) ?? result.url + if (bindings.has(name)) { + // An explicit export shadows every star re-export. + if (!starOrigins.has(name)) continue + + if (starOrigins.get(name) === origin) { + // IITM's aggregate namespace sees the wrapped paths as ambiguous. + // Retain the defining URL so source generation can import it once. + bindings.set(name, { name, origin }) + } else { + bindings.delete(name) + starOrigins.delete(name) + ambiguousStars ??= new Set() + ambiguousStars.add(name) + } + } else { + starOrigins.set(name, origin) + bindings.set(name, binding) + } + } + } finally { + seen?.delete(result.url) + } + } + + return { bindings, origins: starOrigins } +} + +/** + * @param {object} options + * @param {string} options.realUrl The URL of the wrapped module. + * @param {string[] | Map} options.bindings Its exported bindings. + * @param {string} options.originalSpecifier The specifier used to import the module. + * @param {string} options.runtimeSpecifier The wrapper runtime import. + * @param {(url: string) => string} [options.mapImport] Maps module URLs to bundler-owned imports. + * @returns {string} + */ +export function buildWrapperSource ({ + realUrl, + bindings, + originalSpecifier, + runtimeSpecifier, + mapImport +}) { + const moduleSpecifier = mapImport?.(realUrl) ?? realUrl + // The wrapped module imports its namespace as `namespace`, which serves + // every export but the ones a same-origin `export *` collision forced onto + // their defining module (#171): the aggregate namespace drops those as + // ambiguous under iitm, so each such defining module gets its own alias the + // wrapper imports. Without such a collision nothing is added. + let originImports = '' + let originNamespaces + let declarationNames = '' + let bindingNames = '' + let bindingSources + let exportSpecifiers = '' + let writeCases = '' + let index = 0 + for (const binding of bindings.values()) { + const directName = typeof binding === 'string' ? binding : undefined + const name = directName ?? binding.name + let namespaceName = 'namespace' + if (directName === undefined) { + originNamespaces ??= new Map() + namespaceName = originNamespaces.get(binding.origin) + if (namespaceName === undefined) { + namespaceName = `__ns${originNamespaces.size}` + originNamespaces.set(binding.origin, namespaceName) + const originSpecifier = mapImport?.(binding.origin) ?? binding.origin + originImports += `import * as ${namespaceName} from ${JSON.stringify(originSpecifier)}\n` + } + } + const variableName = `$${index}` + const objectKey = JSON.stringify(name) + declarationNames += declarationNames === '' ? variableName : `, ${variableName}` + bindingNames += bindingNames === '' ? objectKey : `, ${objectKey}` + if (bindingSources !== undefined) bindingSources += ', ' + if (namespaceName !== 'namespace') { + bindingSources ??= 'undefined, '.repeat(index) + bindingSources += namespaceName + } else if (bindingSources !== undefined) { + bindingSources += 'undefined' + } + writeCases += ` case ${index++}: ${variableName} = value; break\n` + if (shouldReexport(name, realUrl)) { + const exportName = name === 'default' ? name : objectKey + exportSpecifiers += exportSpecifiers === '' + ? `${variableName} as ${exportName}` + : `, ${variableName} as ${exportName}` + } + } + const binder = declarationNames === '' + ? 'const __binder = new ModuleBinder(namespace)\n' + : `let ${declarationNames} +function __write (index, value) { +switch (index) { +${writeCases} } +} +const __binder = new ModuleBinder(namespace, [${bindingNames}], __write${bindingSources === undefined +? '' +: `, [${bindingSources}]`}) +` + const reexports = exportSpecifiers === '' ? '' : `export { ${exportSpecifiers} }\n` + + return ` +import { register, ModuleBinder } from ${JSON.stringify(runtimeSpecifier)} +import * as namespace from ${JSON.stringify(moduleSpecifier)} +${originImports} +${binder} +${reexports} + +__binder.flush() + +register(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}) +` +} diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs new file mode 100644 index 00000000..7c980b33 --- /dev/null +++ b/test/low-level/bundler.mjs @@ -0,0 +1,241 @@ +import { strictEqual, deepStrictEqual, match, doesNotMatch } from 'assert' +import { readFile, mkdtemp, writeFile, rm } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { pathToFileURL } from 'url' + +import Hook from '../../index.js' +import { createWrapperModule } from '../../bundler.mjs' + +const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href +const source = await readFile(new URL(moduleUrl), 'utf8') + +/** + * @returns {never} + */ +function unexpectedIo () { + throw new Error('I/O should not be used when source is provided') +} + +const wrapper = await createWrapperModule({ + module: { + url: moduleUrl, + format: 'module', + source, + specifier: './something.mjs' + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +strictEqual(wrapper.sideEffects, true) +deepStrictEqual(wrapper.watchFiles, [moduleUrl]) +strictEqual(wrapper.imports.length, 2) +strictEqual(wrapper.imports[0].specifier, './__iitm_runtime__.js') +strictEqual(wrapper.imports[0].kind, 'runtime') +strictEqual(wrapper.imports[0].external, false) +strictEqual(wrapper.imports[1].specifier, './__iitm_module_0__.js') +strictEqual(wrapper.imports[1].kind, 'module') +strictEqual(wrapper.imports[1].external, false) +match(wrapper.code, /from "\.\/__iitm_runtime__\.js"/) +match(wrapper.code, /from "\.\/__iitm_module_0__\.js"/) +match(wrapper.code, /__binder\.register\(\)/) +doesNotMatch(wrapper.code, /from "file:/) + +/** + * @param {object} exported + */ +function hookFoo (exported) { + exported.foo = 43 +} + +const hook = new Hook(['./something.mjs'], hookFoo) +let executableCode = wrapper.code +for (const { specifier, target } of wrapper.imports) { + executableCode = executableCode.replaceAll(JSON.stringify(specifier), JSON.stringify(target.url)) +} +const temporaryDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-')) +const wrapperUrl = pathToFileURL(join(temporaryDirectory, 'wrapper.mjs')).href +try { + await writeFile(new URL(wrapperUrl), executableCode) + const wrappedNamespace = await import(wrapperUrl) + strictEqual(wrappedNamespace.foo, 43) +} finally { + hook.unhook() + await rm(temporaryDirectory, { recursive: true, force: true }) +} + +const rebuilt = await createWrapperModule({ + module: { + url: moduleUrl, + format: 'module', + source: 'export const rebuilt = true', + specifier: './something.mjs' + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +match(rebuilt.code, /export \{ \$rebuilt as "rebuilt" \}/) +doesNotMatch(rebuilt.code, /\$foo/) + +/** + * @param {string} url + */ +function loadBuiltin (url) { + strictEqual(url, 'node:dns/promises') + return { format: 'builtin' } +} + +const builtinWrapper = await createWrapperModule({ + module: { + url: 'node:dns/promises', + format: 'builtin', + specifier: 'node:dns/promises' + }, + resolve: unexpectedIo, + load: loadBuiltin +}) + +strictEqual(builtinWrapper.imports[1].external, true) +strictEqual(builtinWrapper.imports[1].target.url, 'node:dns/promises') +doesNotMatch(builtinWrapper.code, /from "node:dns\/promises"/) + +const commonJsUrl = new URL('../fixtures/something.js', import.meta.url).href +const commonJsWrapper = await createWrapperModule({ + module: { + url: commonJsUrl, + format: 'commonjs', + source: await readFile(new URL(commonJsUrl), 'utf8'), + specifier: './something.js' + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +match(commonJsWrapper.code, /export \{ \$foo as "foo" \}/) +match(commonJsWrapper.code, /export \{ \$default as default \}/) + +const packageUrl = new URL('../../package.json', import.meta.url).href +const sourceWatchUrl = new URL('../fixtures/', import.meta.url).href + +/** + * @param {string} specifier + * @param {{ parentURL: string }} context + */ +function resolveModule (specifier, context) { + return { + url: new URL(specifier, context.parentURL).href, + format: 'module', + watchFiles: [packageUrl] + } +} + +/** + * @param {string} url + * @param {{ format: string }} context + */ +async function loadModule (url, context) { + return { + source: await readFile(new URL(url), 'utf8'), + format: context.format, + watchFiles: [sourceWatchUrl] + } +} + +const reexportUrl = new URL('../fixtures/reexport-same-source.mjs', import.meta.url).href +const reexportWrapper = await createWrapperModule({ + module: { + url: reexportUrl, + format: 'module', + specifier: './reexport-same-source.mjs' + }, + resolve: resolveModule, + load: loadModule +}) + +strictEqual(reexportWrapper.imports[0].kind, 'runtime') +strictEqual(reexportWrapper.imports[1].target.url, reexportUrl) +strictEqual(reexportWrapper.imports[2].specifier, './__iitm_module_1__.js') +strictEqual(reexportWrapper.imports[2].target.format, 'module') +strictEqual(reexportWrapper.watchFiles.includes(reexportUrl), true) +strictEqual(reexportWrapper.watchFiles.includes(packageUrl), true) +strictEqual(reexportWrapper.watchFiles.includes(sourceWatchUrl), true) +doesNotMatch(reexportWrapper.code, /from "file:/) + +/** + * @param {string} specifier + */ +function resolveCommonJsReexport (specifier) { + strictEqual(specifier, 'file:///virtual/something.js') + return { + url: commonJsUrl, + format: 'commonjs' + } +} + +const commonJsReexportWrapper = await createWrapperModule({ + module: { + url: 'file:///virtual/commonjs-reexport.mjs', + format: 'module', + source: "export * from './something.js'", + specifier: './commonjs-reexport.mjs' + }, + resolve: resolveCommonJsReexport, + load: loadModule +}) + +match(commonJsReexportWrapper.code, /export \{ \$foo as "foo" \}/) +doesNotMatch(commonJsReexportWrapper.code, /as default/) + +/** + * @param {string} specifier + */ +function resolveWithoutCanParse (specifier) { + return { + url: specifier === 'bare-package' ? moduleUrl : specifier, + format: 'module' + } +} + +await createWrapperModule({ + module: { + url: 'file:///virtual/can-parse-reexport.mjs', + format: 'module', + source: "export * from 'bare-package'", + specifier: './can-parse-reexport.mjs' + }, + resolve: resolveWithoutCanParse, + load: loadModule +}) + +const canParse = URL.canParse +delete URL.canParse +try { + await createWrapperModule({ + module: { + url: 'file:///virtual/bare-reexport.mjs', + format: 'module', + source: "export * from 'bare-package'", + specifier: './bare-reexport.mjs' + }, + resolve: resolveWithoutCanParse, + load: loadModule + }) + await createWrapperModule({ + module: { + url: 'file:///virtual/url-reexport.mjs', + format: 'module', + source: `export * from ${JSON.stringify(moduleUrl)}`, + specifier: './url-reexport.mjs' + }, + resolve: resolveWithoutCanParse, + load: loadModule + }) +} finally { + if (canParse === undefined) { + delete URL.canParse + } else { + URL.canParse = canParse + } +} diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts new file mode 100644 index 00000000..5c8ea9a5 --- /dev/null +++ b/test/typescript/bundler.test.mts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict' + +import { createWrapperModule } from '../../bundler.mjs' + +const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href +const wrapper = await createWrapperModule({ + module: { + url: moduleUrl, + format: 'module', + source: 'export const value = 42', + specifier: './something.mjs' + }, + resolve () { + throw new Error('Unexpected resolve') + }, + load () { + throw new Error('Unexpected load') + } +}) + +assert.equal(wrapper.sideEffects, true) +assert.equal(wrapper.imports[0].kind, 'runtime') From 22ac33081817e2a70978bd1ed651adcf7609b535 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 06:08:37 +0200 Subject: [PATCH 02/18] feat: instrument CommonJS through IITM Consumers currently need separate RITM and bundler-specific paths for CommonJS, which duplicates matching, metadata, and replacement behavior. 1. Make synchronous hooks opt into CommonJS and route both formats through the shared Hook registry. 2. Make bundler wrappers format-aware, preserve opaque adapter targets, and carry JSON metadata into runtime hooks. 3. Verify the contracts with native sync hooks, real esbuild and webpack bundles, and nft tracing. Refs: https://github.com/nodejs/import-in-the-middle/pull/270 Refs: https://github.com/DataDog/dd-trace-js/issues/9383 --- .eslintrc.yaml | 2 + README.md | 79 +++++-- bundler.d.mts | 50 +++-- bundler.mjs | 83 +++++-- create-hook.mjs | 199 +++++++++++++---- index.d.ts | 35 ++- index.js | 53 +++-- lib/bundler-runtime.js | 3 +- lib/get-exports.mjs | 12 +- lib/register.js | 81 ++++++- lib/source.mjs | 24 ++ lib/wrapper.mjs | 29 ++- package.json | 5 +- register-hooks.d.ts | 13 +- register-hooks.mjs | 3 + test/fixtures/inspectable-create-hook.mjs | 7 +- test/fixtures/sync-commonjs-cycle-a.cjs | 6 + test/fixtures/sync-commonjs-cycle-b.cjs | 5 + test/fixtures/sync-commonjs-semantics.cjs | 29 +++ test/fixtures/sync-commonjs-typescript.cts | 3 + test/fixtures/type-module/module.js | 1 + test/fixtures/type-module/package.json | 3 + test/low-level/bundler.mjs | 111 ++++++++-- test/other/v18-bundlers.mjs | 208 ++++++++++++++++++ test/other/v20-nft-runtime.mjs | 12 + test/register/v18.19-loader-url-escaping.mjs | 4 +- .../v22.15-sync-register-hooks-commonjs.mjs | 140 ++++++++++++ test/typescript/bundler.test.mts | 7 +- test/typescript/register-hooks.test.mts | 15 ++ 29 files changed, 1050 insertions(+), 172 deletions(-) create mode 100644 lib/source.mjs create mode 100644 test/fixtures/sync-commonjs-cycle-a.cjs create mode 100644 test/fixtures/sync-commonjs-cycle-b.cjs create mode 100644 test/fixtures/sync-commonjs-semantics.cjs create mode 100644 test/fixtures/sync-commonjs-typescript.cts create mode 100644 test/fixtures/type-module/module.js create mode 100644 test/fixtures/type-module/package.json create mode 100644 test/other/v18-bundlers.mjs create mode 100644 test/other/v20-nft-runtime.mjs create mode 100644 test/register/v22.15-sync-register-hooks-commonjs.mjs create mode 100644 test/typescript/register-hooks.test.mts diff --git a/.eslintrc.yaml b/.eslintrc.yaml index f95abcd8..a45ff2ed 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -30,3 +30,5 @@ ignorePatterns: - test/fixtures/reexport-same-source.mjs - test/fixtures/reexport-explicit-override.mjs - test/fixtures/reexport-nested-agg.mjs + - test/fixtures/type-module/module.js + - test/fixtures/sync-commonjs-typescript.cts diff --git a/README.md b/README.md index 55d5a334..87ffbe05 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # import-in-the-middle **`import-in-the-middle`** is a module loading interceptor inspired by -[`require-in-the-middle`](https://npm.im/require-in-the-middle), but -specifically for ESM modules. In fact, it can even modify modules after loading -time. +[`require-in-the-middle`](https://npm.im/require-in-the-middle). It supports ESM +through Node.js loader hooks and can also intercept CommonJS when synchronous +hooks are available. ## Usage @@ -117,29 +117,41 @@ node --import=./instrument.mjs ./my-app.mjs ## Bundler integrations -Bundlers can generate the same live-binding wrapper as the Node.js loader with -`createWrapperModule`: +Bundlers can generate ESM and CommonJS wrappers with `createWrapperModule`: ```js import { createWrapperModule } from 'import-in-the-middle/bundler.mjs' const wrapper = await createWrapperModule({ - module: { url, format, source, specifier }, + module: { url, format, source, specifier, target, data }, resolve, load }) ``` -`url` is the canonical `file:` or `node:` URL reported to hooks. `resolve` and -`load` adapt the bundler's resolver and source loader to the same URL-based -module graph. - -The result contains generated `code`, an `imports` manifest, `watchFiles`, and -`sideEffects: true`. The code imports only relative placeholder specifiers. The -bundler adapter provides it as a virtual module and maps each placeholder using -the manifest, so filesystem paths, virtual IDs, external modules, and cache -invalidation remain owned by the bundler. `watchFiles` are file URLs that the -adapter converts to its native watch-dependency format. +`url` is the canonical `file:` or `node:` URL reported to hooks. `specifier` is +the original request. `target` is the bundler's opaque resolved value, such as +an esbuild path/namespace/plugin-data object or a webpack resource. IITM passes +it back unchanged in the import manifest and to `load(target, context)`. + +`resolve(specifier, context)` returns `{ url, format, target, watchFiles }`. +IITM uses `url` and `format` to inspect re-exports while preserving `target` for +the adapter. This keeps namespaces, query strings, external decisions, and +loader state owned by the bundler. + +The result contains generated `code`, its `format`, an `imports` manifest, +`watchFiles`, and `sideEffects: true`. ESM code imports relative placeholder +specifiers. The adapter maps each placeholder to the corresponding manifest +entry. CommonJS code contains the original source and only requires the runtime +placeholder, so the bundler can still discover literal `require()` calls in the +module source. The adapter must resolve those calls from the original module's +context. `watchFiles` are canonical URLs that the adapter converts to its native +watch-dependency format. + +`data` is optional JSON-serializable consumer metadata. IITM embeds it in the +wrapper and passes it as the fourth argument to `Hook` callbacks. This lets an +adapter carry package names, versions, or other build-time facts into a bundle +without retaining build-machine paths. The runtime import in the manifest must be bundled with the wrapper. Keeping it external can create a second hook registry at runtime. It is CommonJS and must @@ -166,7 +178,10 @@ pulled into the ESM graph until [nodejs/node#59929][]. The fix shipped in import { register, supportsSyncHooks } from 'import-in-the-middle/register-hooks.mjs' if (supportsSyncHooks()) { - register({ include: ['package-i-want-to-include'] }) + register({ + include: ['package-i-want-to-include'], + commonjs: true + }) } else { // Fall back to the asynchronous loader, e.g. module.register('import-in-the-middle/hook.mjs'). } @@ -180,7 +195,10 @@ if (supportsSyncHooks()) { import { register } from 'import-in-the-middle/register-hooks.mjs' import { Hook } from 'import-in-the-middle' -register({ include: ['package-i-want-to-include'] }) +register({ + include: ['package-i-want-to-include'], + commonjs: true +}) Hook(['package-i-want-to-include'], (exported, name, baseDir) => { // Instrument the module @@ -193,14 +211,21 @@ node --import=./instrument.mjs ./my-app.mjs `register()` accepts the same `include` / `exclude` options as the asynchronous loader and throws on a Node.js version where `supportsSyncHooks()` is `false`. +Set `commonjs: true` to intercept CommonJS `require()` and CommonJS imported +from ESM through the same `Hook` registry. It is opt-in because consumers that +also install `require-in-the-middle` must disable one CommonJS path to avoid +instrumenting a module twice. ESM loaded through `require()` is intercepted by +the synchronous ESM wrapper. ### Custom matching with `shouldInclude` Instead of `include` / `exclude` lists, you can pass a `shouldInclude(url, specifier)` predicate to decide which modules are intercepted. It is called for every resolved -module with the resolved URL and the import specifier; return a truthy value to -intercept the module. When a predicate is provided it takes over the decision and -the `include` / `exclude` options are ignored. +module with the resolved URL and the import specifier. Return `true` to intercept +the module, or return `{ data }` to intercept it and pass consumer metadata to +the `Hook` callback. `data` must be JSON-serializable. When a predicate is +provided it takes over the decision and the `include` / `exclude` options are +ignored. This is useful when matching doesn't map cleanly onto bare specifiers, file URLs and regular expressions — for example a matcher built from your own configuration, or a @@ -210,11 +235,16 @@ decision that depends on more than the specifier. import { register } from 'import-in-the-middle/register-hooks.mjs' register({ + commonjs: true, shouldInclude (url, specifier) { - return specifier === 'package-i-want-to-include' || - url.includes('/node_modules/some-scope/') + if (specifier !== 'package-i-want-to-include') return false + return { data: { version: '1.2.3' } } } }) + +Hook(['package-i-want-to-include'], (exported, name, baseDir, data) => { + console.log(data.version) +}) ``` The predicate receives only the URL and the specifier, never a resolved file path. @@ -284,7 +314,8 @@ On Node.js versions where type stripping is not enabled by default, run with * You cannot add new exports to a module. You can only modify existing ones. * While bindings to module exports end up being "re-bound" when modified in a hook, dynamically imported modules cannot be altered after they're loaded. -* Modules loaded via `require` are not affected at all. +* Modules loaded via `require` are only affected by synchronous registration + with `commonjs: true`, or when the required target is ESM. * A module's set of export *names* is assumed to be stable for the lifetime of the process. `import-in-the-middle` reads a module's source once to lex its exports and reuses that export set on later loads of the same URL. An upstream diff --git a/bundler.d.mts b/bundler.d.mts index a589bbf6..bdd7128d 100644 --- a/bundler.d.mts +++ b/bundler.d.mts @@ -4,11 +4,25 @@ export type WrapperSource = string | ArrayBuffer | ArrayBufferView -export type BundlerModule = { +export type JsonValue = + | boolean + | number + | string + | JsonValue[] + | { [key: string]: JsonValue } + +export type ModuleTarget = { + url: string + format?: string +} + +export type BundlerModule = { url: string format: string specifier: string source?: WrapperSource + target?: Target + data?: Data } export type ModuleContext = { @@ -16,12 +30,10 @@ export type ModuleContext = { parentURL?: string } -export type ModuleTarget = { +export type ResolveResult = { url: string format?: string -} - -export type ResolveResult = ModuleTarget & { + target?: Target watchFiles?: Iterable } @@ -31,30 +43,38 @@ export type LoadResult = { watchFiles?: Iterable } -export type WrapperImport = { +export type WrapperImport = { specifier: string kind: 'module' | 'runtime' - target: ModuleTarget - external: boolean + url: string + format?: string + target: Target | ModuleTarget } -export type WrapperModule = { +export type WrapperModule = { code: string - imports: WrapperImport[] + format: 'module' | 'commonjs' + imports: WrapperImport[] watchFiles: string[] sideEffects: true } -export type CreateWrapperModuleOptions = { - module: BundlerModule +export type CreateWrapperModuleOptions< + Target = ModuleTarget, + Data extends JsonValue = JsonValue +> = { + module: BundlerModule resolve: ( specifier: string, context: ModuleContext - ) => ResolveResult | Promise + ) => ResolveResult | Promise> load: ( - url: string, + target: Target | ModuleTarget, context: ModuleContext ) => LoadResult | Promise } -export declare function createWrapperModule(options: CreateWrapperModuleOptions): Promise +export declare function createWrapperModule< + Target = ModuleTarget, + Data extends JsonValue = JsonValue +>(options: CreateWrapperModuleOptions): Promise> diff --git a/bundler.mjs b/bundler.mjs index 681446a3..52ed15a6 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -4,10 +4,12 @@ 'use strict' -import { builtinModules } from 'module' - import { driveAsync } from './lib/io.mjs' -import { buildWrapperSource, processModule } from './lib/wrapper.mjs' +import { + buildCommonJSWrapperSource, + buildWrapperSource, + processModule +} from './lib/wrapper.mjs' const RUNTIME_SPECIFIER = './__iitm_runtime__.js' const MODULE_SPECIFIER_PREFIX = './__iitm_module_' @@ -19,6 +21,8 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @property {string} format * @property {string} specifier * @property {string | ArrayBuffer | ArrayBufferView} [source] + * @property {unknown} [target] + * @property {unknown} [data] */ /** @@ -31,6 +35,7 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @typedef {object} ResolveResult * @property {string} url * @property {string} [format] + * @property {unknown} [target] * @property {Iterable} [watchFiles] */ @@ -38,8 +43,9 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @typedef {object} WrapperImport * @property {string} specifier * @property {'module' | 'runtime'} kind - * @property {{ url: string, format?: string }} target - * @property {boolean} external + * @property {string} url + * @property {string} [format] + * @property {unknown} target */ /** @@ -50,15 +56,16 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href */ /** - * Creates an ESM wrapper without embedding bundler-specific module identifiers. + * Creates a format-aware wrapper without embedding bundler-specific module identifiers. * * @param {object} options * @param {BundlerModule} options.module * @param {(specifier: string, context: ModuleContext) => * (ResolveResult | Promise)} options.resolve - * @param {(url: string, context: ModuleContext) => (LoadResult | Promise)} options.load + * @param {(target: unknown, context: ModuleContext) => (LoadResult | Promise)} options.load * @returns {Promise<{ * code: string, + * format: 'module' | 'commonjs', * imports: WrapperImport[], * watchFiles: string[], * sideEffects: true @@ -68,6 +75,10 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } const context = { format: moduleData.format, cache: false } const watchFiles = new Set() const formats = new Map([[moduleData.url, moduleData.format]]) + const targets = new Map([[ + moduleData.url, + moduleData.target ?? { url: moduleData.url, format: moduleData.format } + ]]) if (moduleData.url.startsWith('file:')) { watchFiles.add(moduleData.url) @@ -86,7 +97,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } } } - const result = await load(url, loadContext) + const result = await load(targets.get(url), loadContext) if (result.format !== undefined) { formats.set(url, result.format) } @@ -116,24 +127,56 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } watchFiles.add(watchFile) } } + targets.set(result.url, result.target ?? { url: result.url, format: result.format }) return result } - const { bindings } = await driveAsync( - processModule({ srcUrl: moduleData.url, context }), - { resolve: resolveModule, load: loadModule } - ) - /** @type {WrapperImport[]} */ const imports = [{ specifier: RUNTIME_SPECIFIER, kind: 'runtime', + url: runtimeUrl, + format: 'commonjs', target: { url: runtimeUrl, format: 'commonjs' - }, - external: false + } }] + + if (moduleData.format === 'commonjs' || moduleData.format === 'commonjs-typescript') { + let source = moduleData.source + if (source === undefined) { + const result = await loadModule(moduleData.url, context) + source = result.source + } + if (source === undefined) { + throw new TypeError(`The bundler load adapter returned no source for '${moduleData.url}'`) + } + + return { + code: buildCommonJSWrapperSource({ + realUrl: moduleData.url, + source, + originalSpecifier: moduleData.specifier, + data: moduleData.data, + runtimeSpecifier: RUNTIME_SPECIFIER + }), + format: 'commonjs', + imports, + watchFiles: Array.from(watchFiles), + sideEffects: true + } + } + + if (moduleData.format !== 'module' && moduleData.format !== 'module-typescript' && moduleData.format !== 'builtin') { + throw new TypeError(`Unsupported module format '${moduleData.format}'`) + } + + const { bindings } = await driveAsync( + processModule({ srcUrl: moduleData.url, context }), + { resolve: resolveModule, load: loadModule } + ) + const moduleSpecifiers = new Map() /** @@ -148,11 +191,9 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } imports.push({ specifier, kind: 'module', - target: { - url, - format: formats.get(url) - }, - external: url.startsWith('node:') || builtinModules.includes(url) + url, + format: formats.get(url), + target: targets.get(url) }) } return specifier @@ -162,12 +203,14 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } realUrl: moduleData.url, bindings, originalSpecifier: moduleData.specifier, + data: moduleData.data, runtimeSpecifier: RUNTIME_SPECIFIER, mapImport }) return { code, + format: 'module', imports, watchFiles: Array.from(watchFiles), sideEffects: true diff --git a/create-hook.mjs b/create-hook.mjs index b4b8badb..f0440da8 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -2,11 +2,19 @@ // // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. +import { readFileSync } from 'fs' +import { builtinModules } from 'module' +import { dirname, extname, join } from 'path' import { URL, fileURLToPath } from 'url' import { inspect } from 'util' -import { builtinModules } from 'module' import { driveSync, driveAsync } from './lib/io.mjs' -import { buildWrapperSource, processModule } from './lib/wrapper.mjs' +import './lib/register.js' +import { sourceToString } from './lib/source.mjs' +import { + buildCommonJSWrapperSource, + buildWrapperSource, + processModule +} from './lib/wrapper.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' // Re-exported for backwards compatibility: `supportsSyncHooks` now lives in its @@ -26,10 +34,19 @@ const HANDLED_FORMATS = new Set([ 'builtin', 'module', 'commonjs', 'module-typescript', 'commonjs-typescript' ]) const TRACE_WARNINGS = process.execArgv.includes('--trace-warnings') +const stripTypeScriptTypes = process.getBuiltinModule?.('module')?.stripTypeScriptTypes +const packageTypes = new Map() /** @typedef {import('node:module').LoadHookContext} LoadContext */ /** @typedef {import('node:module').LoadFnOutput} LoadResult */ -/** @typedef {string | { specifier: string, format: 'module-typescript' | 'commonjs-typescript' }} SpecifierData */ +/** @typedef {{ name: string, origin: string }} StarBinding */ +/** + * @typedef {object} SpecifierData + * @property {string} specifier + * @property {string} [format] + * @property {unknown} [data] + * @property {boolean} [commonjs] + */ function hasIitm (url) { // Fast path: avoid URL parsing on the hot path when there's clearly no iitm. @@ -74,6 +91,49 @@ function deleteIitm (url) { } +/** + * @param {string} url + * @returns {string | undefined} + */ +function getFileFormat (url) { + if (!url.startsWith('file:')) return undefined + const filename = fileURLToPath(url) + const extension = extname(filename) + if (extension === '.mjs') return 'module' + if (extension === '.cjs') return 'commonjs' + if (extension === '.mts') return 'module-typescript' + if (extension === '.cts') return 'commonjs-typescript' + if (extension !== '.js' && extension !== '.ts') return undefined + + let directory = dirname(filename) + const visited = [] + while (true) { + if (packageTypes.has(directory)) { + const type = packageTypes.get(directory) + for (const visitedDirectory of visited) packageTypes.set(visitedDirectory, type) + return type === 'module' + ? (extension === '.ts' ? 'module-typescript' : 'module') + : (extension === '.ts' ? 'commonjs-typescript' : 'commonjs') + } + + visited.push(directory) + try { + const type = JSON.parse(readFileSync(join(directory, 'package.json'), 'utf8')).type + packageTypes.set(directory, type) + continue + } catch (error) { + if (error.code !== 'ENOENT') return undefined + } + + const parent = dirname(directory) + if (parent === directory) { + for (const visitedDirectory of visited) packageTypes.set(visitedDirectory, undefined) + return extension === '.ts' ? 'commonjs-typescript' : 'commonjs' + } + directory = parent + } +} + /** * Determines whether the input is a bare specifier, file URL or a regular expression. * @@ -157,10 +217,12 @@ export function createHook (meta) { /** @type {Map} */ const specifiers = new Map() let cachedResolve - const iitmURL = new URL('lib/register.js', meta.url).toString() + const iitmURL = new URL('lib/register.js', meta.url).href + const iitmPath = fileURLToPath(iitmURL) let includeModules, excludeModules let shouldInclude = defaultShouldInclude let disableCjsSourceStripping = false + let hookCommonJS = false // Track CJS module URLs that IITM has wrapped. On Node 24+, CJS modules loaded // via loadCJSModule (in an ESM import chain) have their require() calls for @@ -215,6 +277,7 @@ export function createHook (meta) { function applyOptions (data) { includeModules = ensureArrayWithBareSpecifiersFileUrlsAndRegex(data.include, 'include') excludeModules = ensureArrayWithBareSpecifiersFileUrlsAndRegex(data.exclude, 'exclude') + hookCommonJS = data.commonjs === true // A consumer can supply its own matcher as `shouldInclude(url, specifier)`, // taking ownership of the include/exclude decision instead of expressing it @@ -262,7 +325,7 @@ export function createHook (meta) { // once the parent loader has turned the specifier into a resolved URL. The // only difference between the asynchronous and synchronous hooks is whether // that resolution was awaited, so all the wrapping decisions live here. - function finishResolve (result, specifier, context, parentURL) { + function finishResolve (result, specifier, context, parentURL, synchronous) { // Do not wrap the entrypoint module. Many CLIs check whether they are the // "main" module (e.g. require.main === module). Wrapping changes how they // are evaluated, and can make them exit without doing anything. @@ -279,24 +342,26 @@ export function createHook (meta) { return result } - // The synchronous hooks (`module.registerHooks`) fire for `require()` as well - // as `import`, but iitm only owns the ESM graph: CommonJS modules are - // instrumented separately through require-in-the-middle, and `require()` must - // return the native, mutable module value (e.g. graceful-fs does - // `Object.defineProperty(require('fs'), ...)`, which throws on a frozen ESM - // namespace). Node reports the active module system in `context.conditions` - // ('require' vs 'import'), so leave any require() resolution untouched. The - // asynchronous hook never sees the 'require' condition, so this is a no-op - // there and only affects the synchronous path. - if (context.conditions?.includes('require')) { + const isRequire = context.conditions?.includes('require') === true + let format = result.format + let isModule = format === 'module' || format === 'module-typescript' + + // Without the opt-in, keep CommonJS owned by require-in-the-middle. ESM + // loaded through require() can still use the synchronous ESM wrapper. + if (isRequire && !isModule && (!synchronous || !hookCommonJS)) { return result } - // `shouldInclude` is always set (the include/exclude list matcher by default, - // a consumer-provided predicate otherwise), so no nullish check is needed. - if (!shouldInclude(result.url, specifier)) { + const inclusion = shouldInclude(result.url, specifier) + if (!inclusion) { return result } + const data = typeof inclusion === 'object' ? inclusion.data : undefined + if (synchronous && hookCommonJS && format == null) { + format = getFileFormat(result.url) + isModule = format === 'module' || format === 'module-typescript' + } + const isCommonJS = format === 'commonjs' || format === 'commonjs-typescript' if (isIitm(parentURL, meta) || (parentURL && hasIitm(parentURL))) { return result @@ -333,11 +398,20 @@ export function createHook (meta) { } } + if (synchronous && hookCommonJS && (isCommonJS || (isRequire && !isModule))) { + specifiers.set(result.url, { + specifier, + format, + data, + commonjs: true + }) + return result + } + + if (isRequire && !isModule) return result + // Preserve the format before an outer loader can normalize it. - const specifierData = result.format === 'module-typescript' || result.format === 'commonjs-typescript' - ? { specifier, format: result.format } - : specifier - specifiers.set(result.url, specifierData) + specifiers.set(result.url, { specifier, format, data }) return { url: addIitm(result.url), @@ -345,7 +419,7 @@ export function createHook (meta) { // Node's synchronous resolver drops `format: 'builtin'` for bare builtin // specifiers (`require('crypto')` -> `node:crypto`), so restore it; // otherwise the load hook reads `node:crypto` from disk and throws ENOENT. - format: result.format ?? (result.url.startsWith('node:') ? 'builtin' : undefined) + format: format ?? (result.url.startsWith('node:') ? 'builtin' : undefined) } } @@ -367,7 +441,7 @@ export function createHook (meta) { } const result = await parentResolve(newSpecifier, context) - return finishResolve(result, specifier, context, parentURL) + return finishResolve(result, specifier, context, parentURL, false) } // Synchronous counterpart to `resolve`, for `module.registerHooks`. The @@ -391,19 +465,18 @@ export function createHook (meta) { } const result = nextResolve(newSpecifier, context) - return finishResolve(result, specifier, context, parentURL) + return finishResolve(result, specifier, context, parentURL, true) } - /** * Finalizes a successful wrap and builds its module source. * * @param {string} realUrl The URL of the wrapped module. * @param {LoadContext} context Its loader context. - * @param {string} originalSpecifier The original import specifier. + * @param {SpecifierData} specifierData The module's interception data. * @param {string[] | Map} bindings Its exported bindings. */ - function onWrapSuccess (realUrl, context, originalSpecifier, bindings) { + function onWrapSuccess (realUrl, context, specifierData, bindings) { specifiers.delete(realUrl) // context.format is set to 'commonjs' by getCjsExports during processModule. if (context.format === 'commonjs') { @@ -412,7 +485,8 @@ export function createHook (meta) { return buildWrapperSource({ realUrl, bindings, - originalSpecifier, + originalSpecifier: specifierData.specifier, + data: specifierData.data, runtimeSpecifier: iitmURL }) } @@ -433,6 +507,49 @@ export function createHook (meta) { emitWarning(err) } + /** + * @param {string} url + * @param {LoadResult} result + * @param {SpecifierData} specifierData + * @returns {LoadResult} + */ + function wrapCommonJS (url, result, specifierData) { + specifiers.delete(url) + const format = result.format ?? specifierData.format + let source = result.source + + if (url.startsWith('node:')) { + source = `module.exports = process.getBuiltinModule(${JSON.stringify(url.slice(5))})\n` + } else if ((format === 'commonjs' || format === 'commonjs-typescript') && source == null && url.startsWith('file:')) { + source = readFileSync(fileURLToPath(url)) + } + + if (source == null || (format !== 'commonjs' && format !== 'commonjs-typescript' && !url.startsWith('node:'))) { + return result + } + + try { + if (format === 'commonjs-typescript' && stripTypeScriptTypes !== undefined) { + source = stripTypeScriptTypes(sourceToString(source), { mode: 'strip' }) + } + return { + ...result, + format: 'commonjs', + source: buildCommonJSWrapperSource({ + realUrl: url, + source, + originalSpecifier: specifierData.specifier, + data: specifierData.data, + runtimeSpecifier: iitmPath + }), + shortCircuit: true + } + } catch (cause) { + onWrapFailure(url, cause) + return result + } + } + /** * @param {string} url * @param {LoadContext} context @@ -447,10 +564,8 @@ export function createHook (meta) { return parentGetSource(url, context) } - let originalSpecifier = specifierData let processContext = context - if (typeof specifierData !== 'string') { - originalSpecifier = specifierData.specifier + if (specifierData.format !== undefined) { processContext = { ...context, format: specifierData.format } } @@ -459,7 +574,7 @@ export function createHook (meta) { processModule({ srcUrl: realUrl, context: processContext }), { resolve: cachedResolve, load: parentGetSource } ) - return { source: onWrapSuccess(realUrl, processContext, originalSpecifier, bindings) } + return { source: onWrapSuccess(realUrl, processContext, specifierData, bindings) } } catch (cause) { onWrapFailure(realUrl, cause) // Revert back to the non-iitm URL @@ -487,10 +602,8 @@ export function createHook (meta) { return nextLoad(url, context) } - let originalSpecifier = specifierData let processContext = context - if (typeof specifierData !== 'string') { - originalSpecifier = specifierData.specifier + if (specifierData.format !== undefined) { processContext = { ...context, format: specifierData.format } } @@ -499,7 +612,7 @@ export function createHook (meta) { processModule({ srcUrl: realUrl, context: processContext }), { resolve: cachedResolve, load: nextLoad } ) - return { source: onWrapSuccess(realUrl, processContext, originalSpecifier, bindings) } + return { source: onWrapSuccess(realUrl, processContext, specifierData, bindings) } } catch (cause) { onWrapFailure(realUrl, cause) url = realUrl @@ -567,6 +680,18 @@ export function createHook (meta) { return nextLoad(deleteIitm(url), context) } + const specifierData = specifiers.get(url) + if (specifierData?.commonjs === true) { + let result + try { + result = nextLoad(url, context) + } catch (error) { + specifiers.delete(url) + throw error + } + return wrapCommonJS(url, result, specifierData) + } + if (cjsInIitmChain.has(url) && !disableCjsSourceStripping) { const result = nextLoad(url, context) if (result.format === 'commonjs' && result.source != null) { diff --git a/index.d.ts b/index.d.ts index d3152482..acdfd7c1 100644 --- a/index.d.ts +++ b/index.d.ts @@ -18,16 +18,22 @@ export type Namespace = { [key: string]: any } * starting from the package name. * @param {baseDir} string The absolute path of the module, if not provided in * `name`. - * @return any A value that can will be assigned to `exports.default`. This is - * equivalent to doing that assignment in the body of this function. + * @param {data} Data Optional metadata embedded by a loader or bundler. + * @return unknown For ESM, a value assigned to `exports.default` when present. + * For CommonJS, a value that replaces `module.exports`. */ -export type HookFn = (exported: Namespace, name: string, baseDir: string|void) => any +export type HookFn = ( + exported: Namespace, + name: string, + baseDir: string|void, + data?: Data +) => unknown export type Options = { internals?: boolean } -export declare class Hook { +export declare class Hook { /** * Creates a hook to be run on any already loaded modules and any that will * be loaded in the future. It will be run once per loaded module. If @@ -41,9 +47,9 @@ export declare class Hook { * unless they are mentioned specifically in the modules array. * @param {HookFunction} hookFn The function to be run on each module. */ - constructor (modules: Array, options: Options, hookFn: HookFn) - constructor (modules: Array, hookFn: HookFn) - constructor (hookFn: HookFn) + constructor (modules: Array, options: Options, hookFn: HookFn) + constructor (modules: Array, hookFn: HookFn) + constructor (hookFn: HookFn) /** * Disables this hook. It will no longer be run against any subsequently @@ -60,8 +66,17 @@ export default Hook * @param {url} string The absolute path of the module, as a `file:` URL string. * @param {exported} { [string]: any } An object representing the exported * items of a module. + * @param {specifier} string The original import or require specifier. + * @param {data} Data Optional metadata embedded by a loader or bundler. + * @param {format} string The intercepted module format. */ -export type HookFunction = (url: string, exported: Namespace) => void +export type HookFunction = ( + url: string, + exported: Namespace, + specifier: string, + data: Data|undefined, + format: 'module'|'commonjs' +) => unknown /** * Adds a hook to be run on any already loaded modules and any that will be @@ -73,7 +88,7 @@ export type HookFunction = (url: string, exported: Namespace) => void * single imported module, rather than with any filtering. * @param {HookFunction} hookFn The function to be run on each module. */ -export declare function addHook(hookFn: HookFunction): void +export declare function addHook(hookFn: HookFunction): void /** * Removes a hook that has been previously added with `addHook`. It will no @@ -83,7 +98,7 @@ export declare function addHook(hookFn: HookFunction): void * `Hook` class. * @param {HookFunction} hookFn The function to be removed. */ -export declare function removeHook(hookFn: HookFunction): void +export declare function removeHook(hookFn: HookFunction): void type CreateAddHookMessageChannelReturn = { addHookMessagePort: MessagePort, diff --git a/index.js b/index.js index 563fb240..aa8bab19 100644 --- a/index.js +++ b/index.js @@ -13,9 +13,9 @@ if (!isBuiltin) { } const { - importHooks, - specifiers, - toHook + addHook, + removeHook, + specifiers } = require('./lib/register') /** @@ -38,20 +38,18 @@ function isTurbopackSpecifier (specifier, baseDir) { return baseDir.endsWith(specifierWithoutTurbopackHash) } -function addHook (hook) { - importHooks.push(hook) - toHook.forEach(([name, namespace, specifier]) => hook(name, namespace, specifier)) -} - -function removeHook (hook) { - const index = importHooks.indexOf(hook) - if (index > -1) { - importHooks.splice(index, 1) - } -} - -function callHookFn (hookFn, namespace, name, baseDir) { - const newDefault = hookFn(namespace, name, baseDir) +/** + * @param {Function} hookFn + * @param {object} namespace + * @param {string} name + * @param {string|undefined} baseDir + * @param {unknown} data + * @param {'module'|'commonjs'} format + * @returns {unknown} + */ +function callHookFn (hookFn, namespace, name, baseDir, data, format) { + const newDefault = hookFn(namespace, name, baseDir, data) + if (format === 'commonjs') return newDefault if (newDefault && newDefault !== namespace) { // Only ESM modules that actually export `default` can have it reassigned. // Some hooks return a value unconditionally; avoid crashing when the module @@ -147,7 +145,7 @@ function Hook (modules, options, hookFn) { sendModulesToLoader(modules) } - this._iitmHook = (name, namespace, specifier) => { + this._iitmHook = (name, namespace, specifier, data, format) => { const loadUrl = name const isNodeUrl = loadUrl.startsWith('node:') let filePath, baseDir @@ -178,32 +176,39 @@ function Hook (modules, options, hookFn) { } } + let replacement if (modules) { for (const matchArg of modules) { + let result if (filePath && matchArg === filePath) { // abspath match - callHookFn(hookFn, namespace, filePath, undefined) + result = callHookFn(hookFn, namespace, filePath, undefined, data, format) } else if (matchArg === name) { if (!baseDir) { // built-in module (or unexpected non file:// name?) - callHookFn(hookFn, namespace, name, baseDir) + result = callHookFn(hookFn, namespace, name, baseDir, data, format) } else if (baseDir.endsWith(specifiers.get(loadUrl)) || isTurbopackSpecifier(specifiers.get(loadUrl), baseDir)) { // An import of the top-level module (e.g. `import 'ioredis'`). // Note: Slight behaviour difference from RITM. RITM uses // `require.resolve(name)` to see if filename is the module // main file, which will catch `require('ioredis/built/index.js')`. // The check here will not catch `import 'ioredis/built/index.js'`. - callHookFn(hookFn, namespace, name, baseDir) + result = callHookFn(hookFn, namespace, name, baseDir, data, format) } else if (internals) { const internalPath = name + path.sep + path.relative(baseDir, filePath) - callHookFn(hookFn, namespace, internalPath, baseDir) + result = callHookFn(hookFn, namespace, internalPath, baseDir, data, format) } } else if (matchArg === specifier) { - callHookFn(hookFn, namespace, specifier, baseDir) + result = callHookFn(hookFn, namespace, specifier, baseDir, data, format) + } + if (result !== undefined) { + namespace = result + replacement = result } } + return replacement } else { - callHookFn(hookFn, namespace, name, baseDir) + return callHookFn(hookFn, namespace, name, baseDir, data, format) } } diff --git a/lib/bundler-runtime.js b/lib/bundler-runtime.js index 396f7211..ed49fe55 100644 --- a/lib/bundler-runtime.js +++ b/lib/bundler-runtime.js @@ -4,6 +4,7 @@ 'use strict' -const { ModuleBinder } = require('./register.js') +const { ModuleBinder, registerCommonJS } = require('./register.js') exports.ModuleBinder = ModuleBinder +exports.registerCommonJS = registerCommonJS diff --git a/lib/get-exports.mjs b/lib/get-exports.mjs index d396a54e..20952e4e 100644 --- a/lib/get-exports.mjs +++ b/lib/get-exports.mjs @@ -7,6 +7,7 @@ import { builtinModules, createRequire } from 'module' import { fileURLToPath, pathToFileURL } from 'url' import { dirname, join } from 'path' import { LOAD } from './io.mjs' +import { sourceToString } from './source.mjs' const nodeMajor = Number(process.versions.node.split('.')[0]) export const hasModuleExportsCJSDefault = nodeMajor >= 23 @@ -263,16 +264,7 @@ export function * getExports (url, context) { // Loader hooks can return ArrayBuffer / TypedArray sources. Normalize to a // string for parsing. if (source && typeof source !== 'string') { - // Avoid copies where possible: - // - Buffer.from(Uint8Array) copies - // - Buffer.from(ArrayBuffer, offset, length) wraps the existing memory - if (Buffer.isBuffer(source)) { - source = source.toString('utf8') - } else if (ArrayBuffer.isView(source)) { - source = Buffer.from(source.buffer, source.byteOffset, source.byteLength).toString('utf8') - } else { - source = Buffer.from(source).toString('utf8') - } + source = sourceToString(source) } if (!source) { diff --git a/lib/register.js b/lib/register.js index ff5f1676..91fea86c 100644 --- a/lib/register.js +++ b/lib/register.js @@ -7,6 +7,16 @@ const binders = new WeakMap() const specifiers = new Map() const toHook = [] +/** + * @typedef {object} HookEntry + * @property {string} name + * @property {object} namespace + * @property {string} specifier + * @property {unknown} data + * @property {'module' | 'commonjs'} format + * @property {{ exports: unknown }} [module] + */ + /** * @param {object} source The module namespace. * @param {string | symbol} name The export name. @@ -41,18 +51,80 @@ function defineExport (target, name, descriptor) { const proxyHandler = { defineProperty: defineExport, set: setExport } +/** + * @param {(name: string, exports: unknown, specifier: string, data: unknown, + * format: 'module' | 'commonjs') => unknown} hook + * @param {HookEntry} entry + * @returns {void} + */ +function applyHook (hook, entry) { + const exports = entry.module === undefined ? entry.namespace : entry.module.exports + const replacement = hook(entry.name, exports, entry.specifier, entry.data, entry.format) + if (entry.module !== undefined && replacement !== undefined) { + entry.module.exports = replacement + } +} + +/** + * @param {(name: string, exports: unknown, specifier: string, data: unknown, + * format: 'module' | 'commonjs') => unknown} hook + * @returns {void} + */ +function addHook (hook) { + importHooks.push(hook) + for (const entry of toHook) { + applyHook(hook, entry) + } +} + +/** + * @param {Function} hook + * @returns {void} + */ +function removeHook (hook) { + const index = importHooks.indexOf(hook) + if (index !== -1) importHooks.splice(index, 1) +} + /** * @param {string} name The wrapped module URL. * @param {ModuleBinder} binder The wrapper's binding state. * @param {string} specifier The original import specifier. + * @param {unknown} [data] Consumer data associated with the module. */ -function register (name, binder, specifier) { +function register (name, binder, specifier, data) { const { namespace } = binder specifiers.set(name, specifier) binders.set(namespace, binder) const proxy = new Proxy(namespace, proxyHandler) - importHooks.forEach(hook => hook(name, proxy, specifier)) - toHook.push([name, proxy, specifier]) + const entry = { name, namespace: proxy, specifier, data, format: 'module' } + for (const hook of importHooks) { + applyHook(hook, entry) + } + toHook.push(entry) +} + +/** + * @param {string} name The wrapped module URL. + * @param {{ exports: unknown }} module Its CommonJS module object. + * @param {string} specifier The original require specifier. + * @param {unknown} [data] Consumer data associated with the module. + * @returns {void} + */ +function registerCommonJS (name, module, specifier, data) { + specifiers.set(name, specifier) + const entry = { + name, + namespace: module.exports, + specifier, + data, + format: 'commonjs', + module + } + for (const hook of importHooks) { + applyHook(hook, entry) + } + toHook.push(entry) } // Delays (ms) for re-reading exports that were still in their temporal dead zone @@ -213,7 +285,10 @@ class ModuleBinder { } exports.register = register +exports.registerCommonJS = registerCommonJS exports.ModuleBinder = ModuleBinder +exports.addHook = addHook exports.importHooks = importHooks +exports.removeHook = removeHook exports.specifiers = specifiers exports.toHook = toHook diff --git a/lib/source.mjs b/lib/source.mjs new file mode 100644 index 00000000..27100e2d --- /dev/null +++ b/lib/source.mjs @@ -0,0 +1,24 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. +// +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. + +/** + * @param {string | ArrayBuffer | ArrayBufferView} source + * @returns {string} + */ +export function sourceToString (source) { + if (typeof source === 'string') return source + if (Buffer.isBuffer(source)) return source.toString('utf8') + if (ArrayBuffer.isView(source)) { + return Buffer.from(source.buffer, source.byteOffset, source.byteLength).toString('utf8') + } + return Buffer.from(source).toString('utf8') +} + +/** + * @param {string} source + * @returns {string} + */ +export function commentShebang (source) { + return source.startsWith('#!') ? '//' + source.slice(2) : source +} diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index e4aaac06..116d4dee 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -9,6 +9,7 @@ import { URL } from 'url' import { getExports } from './get-exports.mjs' import { RESOLVE } from './io.mjs' +import { commentShebang, sourceToString } from './source.mjs' // Depth at which `processModule` starts tracking visited URLs to break an // `export *` cycle. Real re-export chains are only a few levels deep, so this @@ -219,6 +220,7 @@ export function * processModule ({ srcUrl, context, excludeDefault = false, dept * @param {string} options.realUrl The URL of the wrapped module. * @param {string[] | Map} options.bindings Its exported bindings. * @param {string} options.originalSpecifier The specifier used to import the module. + * @param {unknown} [options.data] Consumer data associated with the module. * @param {string} options.runtimeSpecifier The wrapper runtime import. * @param {(url: string) => string} [options.mapImport] Maps module URLs to bundler-owned imports. * @returns {string} @@ -227,6 +229,7 @@ export function buildWrapperSource ({ realUrl, bindings, originalSpecifier, + data, runtimeSpecifier, mapImport }) { @@ -299,6 +302,30 @@ ${reexports} __binder.flush() -register(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}) +register(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)}) ` } + +/** + * @param {object} options + * @param {string} options.realUrl + * @param {string | ArrayBuffer | ArrayBufferView} options.source + * @param {string} options.originalSpecifier + * @param {unknown} [options.data] + * @param {string} options.runtimeSpecifier + * @returns {string} + */ +export function buildCommonJSWrapperSource ({ + realUrl, + source, + originalSpecifier, + data, + runtimeSpecifier +}) { + source = commentShebang(sourceToString(source)) + + return `(function (exports, require, module, __filename, __dirname) {${source}\n` + + '}).call(module.exports, module.exports, require, module, __filename, __dirname)\n' + + `require(${JSON.stringify(runtimeSpecifier)}).registerCommonJS(` + + `${JSON.stringify(realUrl)}, module, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)})\n` +} diff --git a/package.json b/package.json index 436a25e2..493d3468 100644 --- a/package.json +++ b/package.json @@ -43,8 +43,10 @@ "@node-rs/crc32": "^1.10.6", "@react-email/components": "^0.0.19", "@types/node": "^18.0.6", + "@vercel/nft": "^1.10.2", "c8": "^7.14.0", "date-fns": "^3.6.0", + "esbuild": "^0.28.1", "eslint": "^8.57.1", "eslint-config-standard": "^17.1.0", "eslint-plugin-import": "^2.32.0", @@ -56,7 +58,8 @@ "openai": "4.47.2", "ts-node": "^10.9.2", "typescript": "^4.9.5", - "vue": "^3.5.26" + "vue": "^3.5.26", + "webpack": "^5.109.2" }, "dependencies": { "cjs-module-lexer": "^2.2.0", diff --git a/register-hooks.d.ts b/register-hooks.d.ts index e398adb5..e470c789 100644 --- a/register-hooks.d.ts +++ b/register-hooks.d.ts @@ -3,10 +3,19 @@ * `file:` URLs or regular expressions, matched against the module being * resolved. CJS source stripping remains enabled unless explicitly disabled. */ -export type RegisterHooksOptions = { +export type ModuleInclusion = { + data?: Data +} + +export type RegisterHooksOptions = { include?: Array exclude?: Array disableCjsSourceStripping?: boolean + commonjs?: boolean + shouldInclude?: ( + url: string, + specifier: string + ) => boolean | ModuleInclusion } /** @@ -34,7 +43,7 @@ export type RegisterHooksOptions = { * * @throws If {@link supportsSyncHooks} is `false` in the running Node.js. */ -export declare function register(options?: RegisterHooksOptions): void +export declare function register(options?: RegisterHooksOptions): void /** * Whether the running Node.js can correctly run the synchronous loader via diff --git a/register-hooks.mjs b/register-hooks.mjs index d67de611..d33274cd 100644 --- a/register-hooks.mjs +++ b/register-hooks.mjs @@ -39,6 +39,9 @@ let registered = false * @param {Array} [options.include] Only intercept these modules. * @param {Array} [options.exclude] Never intercept these modules. * @param {boolean} [options.disableCjsSourceStripping] Leave hook-provided CJS source unchanged. + * @param {boolean} [options.commonjs] Intercept CommonJS through the synchronous load hook. + * @param {(url: string, specifier: string) => boolean | { data?: unknown }} [options.shouldInclude] + * Custom inclusion predicate. Returning `{ data }` passes that value to the Hook callback. * @returns {void} */ export function register (options) { diff --git a/test/fixtures/inspectable-create-hook.mjs b/test/fixtures/inspectable-create-hook.mjs index eead5e3c..d9dba82b 100644 --- a/test/fixtures/inspectable-create-hook.mjs +++ b/test/fixtures/inspectable-create-hook.mjs @@ -6,14 +6,15 @@ let source = readFileSync(createHookUrl, 'utf8') /** * @param {string} _match + * @param {string} prefix * @param {string} relative */ -function resolveImport (_match, relative) { +function resolveImport (_match, prefix, relative) { const absolute = new URL('../../' + relative.slice(2), import.meta.url).href - return `from ${JSON.stringify(absolute)}` + return prefix + JSON.stringify(absolute) } -source = source.replace(/from '(\.\/[^']+)'/g, resolveImport) +source = source.replace(/(from |import )'(\.\/[^']+)'/g, resolveImport) source = source.replace( 'return { initialize, load, resolve, resolveSync, loadSync, applyOptions }', 'return { initialize, load, resolve, resolveSync, loadSync, applyOptions, specifiers }' diff --git a/test/fixtures/sync-commonjs-cycle-a.cjs b/test/fixtures/sync-commonjs-cycle-a.cjs new file mode 100644 index 00000000..c28015e3 --- /dev/null +++ b/test/fixtures/sync-commonjs-cycle-a.cjs @@ -0,0 +1,6 @@ +exports.name = 'a' + +const b = require('./sync-commonjs-cycle-b.cjs') + +exports.fromB = b.name +exports.seenByB = b.fromA diff --git a/test/fixtures/sync-commonjs-cycle-b.cjs b/test/fixtures/sync-commonjs-cycle-b.cjs new file mode 100644 index 00000000..8d7d8ca7 --- /dev/null +++ b/test/fixtures/sync-commonjs-cycle-b.cjs @@ -0,0 +1,5 @@ +exports.name = 'b' + +const a = require('./sync-commonjs-cycle-a.cjs') + +exports.fromA = a.name diff --git a/test/fixtures/sync-commonjs-semantics.cjs b/test/fixtures/sync-commonjs-semantics.cjs new file mode 100644 index 00000000..71759698 --- /dev/null +++ b/test/fixtures/sync-commonjs-semantics.cjs @@ -0,0 +1,29 @@ +#!/usr/bin/env node +'use strict' + +if (this !== exports) { + throw new Error('top-level this must be exports') +} + +if (arguments.length !== 5 || arguments[0] !== exports || arguments[2] !== module) { + throw new Error('CommonJS wrapper arguments changed') +} + +module.exports = { + argumentExports: arguments[0], + topLevelThis: this, + value: 42 +} + +/* eslint-disable n/no-exports-assign, no-global-assign */ +exports = undefined +require = undefined +module = undefined +__filename = undefined +__dirname = undefined +/* eslint-enable n/no-exports-assign, no-global-assign */ + +return + +// eslint-disable-next-line no-unreachable +module.exports.unreachable = true diff --git a/test/fixtures/sync-commonjs-typescript.cts b/test/fixtures/sync-commonjs-typescript.cts new file mode 100644 index 00000000..a9fbaa1d --- /dev/null +++ b/test/fixtures/sync-commonjs-typescript.cts @@ -0,0 +1,3 @@ +const value: number = 42 + +module.exports = { value } diff --git a/test/fixtures/type-module/module.js b/test/fixtures/type-module/module.js new file mode 100644 index 00000000..c16d7056 --- /dev/null +++ b/test/fixtures/type-module/module.js @@ -0,0 +1 @@ +export const value = 42 diff --git a/test/fixtures/type-module/package.json b/test/fixtures/type-module/package.json new file mode 100644 index 00000000..3dbc1ca5 --- /dev/null +++ b/test/fixtures/type-module/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 7c980b33..103e7cc7 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -1,8 +1,8 @@ -import { strictEqual, deepStrictEqual, match, doesNotMatch } from 'assert' +import { strictEqual, deepStrictEqual, match, doesNotMatch, rejects } from 'assert' import { readFile, mkdtemp, writeFile, rm } from 'fs/promises' import { tmpdir } from 'os' import { join } from 'path' -import { pathToFileURL } from 'url' +import { fileURLToPath, pathToFileURL } from 'url' import Hook from '../../index.js' import { createWrapperModule } from '../../bundler.mjs' @@ -29,14 +29,15 @@ const wrapper = await createWrapperModule({ }) strictEqual(wrapper.sideEffects, true) +strictEqual(wrapper.format, 'module') deepStrictEqual(wrapper.watchFiles, [moduleUrl]) strictEqual(wrapper.imports.length, 2) strictEqual(wrapper.imports[0].specifier, './__iitm_runtime__.js') strictEqual(wrapper.imports[0].kind, 'runtime') -strictEqual(wrapper.imports[0].external, false) +strictEqual(wrapper.imports[0].format, 'commonjs') strictEqual(wrapper.imports[1].specifier, './__iitm_module_0__.js') strictEqual(wrapper.imports[1].kind, 'module') -strictEqual(wrapper.imports[1].external, false) +strictEqual(wrapper.imports[1].url, moduleUrl) match(wrapper.code, /from "\.\/__iitm_runtime__\.js"/) match(wrapper.code, /from "\.\/__iitm_module_0__\.js"/) match(wrapper.code, /__binder\.register\(\)/) @@ -80,10 +81,10 @@ match(rebuilt.code, /export \{ \$rebuilt as "rebuilt" \}/) doesNotMatch(rebuilt.code, /\$foo/) /** - * @param {string} url + * @param {{ url: string }} target */ -function loadBuiltin (url) { - strictEqual(url, 'node:dns/promises') +function loadBuiltin (target) { + strictEqual(target.url, 'node:dns/promises') return { format: 'builtin' } } @@ -97,7 +98,7 @@ const builtinWrapper = await createWrapperModule({ load: loadBuiltin }) -strictEqual(builtinWrapper.imports[1].external, true) +strictEqual(builtinWrapper.imports[1].format, 'builtin') strictEqual(builtinWrapper.imports[1].target.url, 'node:dns/promises') doesNotMatch(builtinWrapper.code, /from "node:dns\/promises"/) @@ -107,14 +108,61 @@ const commonJsWrapper = await createWrapperModule({ url: commonJsUrl, format: 'commonjs', source: await readFile(new URL(commonJsUrl), 'utf8'), - specifier: './something.js' + specifier: './something.js', + target: { namespace: 'file', path: fileURLToPath(commonJsUrl) }, + data: { package: 'fixture', version: '1.0.0' } }, resolve: unexpectedIo, load: unexpectedIo }) -match(commonJsWrapper.code, /export \{ \$foo as "foo" \}/) -match(commonJsWrapper.code, /export \{ \$default as default \}/) +strictEqual(commonJsWrapper.format, 'commonjs') +strictEqual(commonJsWrapper.imports.length, 1) +strictEqual(commonJsWrapper.imports[0].kind, 'runtime') +match(commonJsWrapper.code, /registerCommonJS/) +match(commonJsWrapper.code, /"package":"fixture","version":"1\.0\.0"/) +doesNotMatch(commonJsWrapper.code, /^(?:import|export) /m) + +const loadedCommonJsTarget = { namespace: 'file', path: '/virtual/loaded.cjs' } +const loadedCommonJsWrapper = await createWrapperModule({ + module: { + url: 'file:///virtual/loaded.cjs', + format: 'commonjs', + specifier: './loaded.cjs', + target: loadedCommonJsTarget + }, + resolve: unexpectedIo, + load (target) { + strictEqual(target, loadedCommonJsTarget) + return { source: 'module.exports = 42' } + } +}) + +strictEqual(loadedCommonJsWrapper.format, 'commonjs') +match(loadedCommonJsWrapper.code, /module\.exports = 42/) + +await rejects(createWrapperModule({ + module: { + url: 'file:///virtual/missing-source.cjs', + format: 'commonjs', + specifier: './missing-source.cjs' + }, + resolve: unexpectedIo, + load () { + return {} + } +}), /returned no source/) + +await rejects(createWrapperModule({ + module: { + url: 'file:///virtual/data.json', + format: 'json', + source: '{}', + specifier: './data.json' + }, + resolve: unexpectedIo, + load: unexpectedIo +}), /Unsupported module format 'json'/) const packageUrl = new URL('../../package.json', import.meta.url).href const sourceWatchUrl = new URL('../fixtures/', import.meta.url).href @@ -132,32 +180,57 @@ function resolveModule (specifier, context) { } /** - * @param {string} url + * @param {{ url: string }} target * @param {{ format: string }} context */ -async function loadModule (url, context) { +async function loadModule (target, context) { return { - source: await readFile(new URL(url), 'utf8'), + source: await readFile(new URL(target.url), 'utf8'), format: context.format, watchFiles: [sourceWatchUrl] } } const reexportUrl = new URL('../fixtures/reexport-same-source.mjs', import.meta.url).href +const reexportTarget = { + namespace: 'file', + path: fileURLToPath(reexportUrl), + pluginData: { loader: 'source' } +} +const leafTargets = new Map() const reexportWrapper = await createWrapperModule({ module: { url: reexportUrl, format: 'module', - specifier: './reexport-same-source.mjs' + specifier: './reexport-same-source.mjs', + target: reexportTarget }, - resolve: resolveModule, - load: loadModule + resolve (specifier, context) { + const result = resolveModule(specifier, context) + const target = { + namespace: 'file', + path: fileURLToPath(result.url), + pluginData: { resolvedBy: 'fixture' } + } + leafTargets.set(result.url, target) + return { ...result, target } + }, + async load (target, context) { + if (target === reexportTarget) { + return loadModule({ url: reexportUrl }, context) + } + for (const [url, leafTarget] of leafTargets) { + if (target === leafTarget) return loadModule({ url }, context) + } + throw new Error('load received a target that was not returned by the adapter') + } }) +strictEqual(reexportWrapper.format, 'module') strictEqual(reexportWrapper.imports[0].kind, 'runtime') -strictEqual(reexportWrapper.imports[1].target.url, reexportUrl) +strictEqual(reexportWrapper.imports[1].target, reexportTarget) strictEqual(reexportWrapper.imports[2].specifier, './__iitm_module_1__.js') -strictEqual(reexportWrapper.imports[2].target.format, 'module') +strictEqual(reexportWrapper.imports[2].target, leafTargets.get(reexportWrapper.imports[2].url)) strictEqual(reexportWrapper.watchFiles.includes(reexportUrl), true) strictEqual(reexportWrapper.watchFiles.includes(packageUrl), true) strictEqual(reexportWrapper.watchFiles.includes(sourceWatchUrl), true) diff --git a/test/other/v18-bundlers.mjs b/test/other/v18-bundlers.mjs new file mode 100644 index 00000000..6e27aa6a --- /dev/null +++ b/test/other/v18-bundlers.mjs @@ -0,0 +1,208 @@ +import { deepStrictEqual, strictEqual } from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import * as esbuild from 'esbuild' +import webpack from 'webpack' + +import { createWrapperModule } from '../../bundler.mjs' + +const packageRoot = fileURLToPath(new URL('../../', import.meta.url)) +const indexPath = join(packageRoot, 'index.js') +const temporaryDirectory = await realpath(await mkdtemp(join(tmpdir(), 'iitm-bundlers-'))) + +try { + const originalPath = join(temporaryDirectory, 'original.mjs') + const originalCommonJsPath = join(temporaryDirectory, 'original.cjs') + const dependencyPath = join(temporaryDirectory, 'dependency.cjs') + await writeFile(originalPath, 'export const value = 42\n') + await writeFile(dependencyPath, 'module.exports = 42\n') + + const originalTarget = { + namespace: 'iitm-original', + path: originalPath, + pluginData: { owner: 'adapter' } + } + const wrappers = new Map([ + ['esm', await createWrapperModule({ + module: { + url: pathToFileURL(originalPath).href, + format: 'module', + source: await readFile(originalPath), + specifier: 'iitm-virtual-esm', + target: originalTarget, + data: { increment: 1 } + }, + resolve: unexpectedIo, + load: unexpectedIo + })], + ['commonjs', await createWrapperModule({ + module: { + url: pathToFileURL(originalCommonJsPath).href, + format: 'commonjs', + source: "module.exports = { value: require('./dependency.cjs') }\n", + specifier: 'iitm-virtual-commonjs', + target: { namespace: 'file', path: originalCommonJsPath }, + data: { increment: 2 } + }, + resolve: unexpectedIo, + load: unexpectedIo + })] + ]) + + const entrySource = ` +const Hook = require(${JSON.stringify(indexPath)}) + +new Hook((exports, name, baseDir, data) => { + if (data === undefined) return + exports.value += data.increment +}) + +Promise.all([ + import('iitm-virtual-esm'), + Promise.resolve(require('iitm-virtual-commonjs')) +]).then(([esm, commonjs]) => { + console.log(JSON.stringify({ esm: esm.value, commonjs: commonjs.value })) +}) +` + + await testEsbuild(entrySource, wrappers, originalTarget) + await testWebpack(entrySource, wrappers) +} finally { + await rm(temporaryDirectory, { recursive: true, force: true }) +} + +/** + * @returns {never} + */ +function unexpectedIo () { + throw new Error('Unexpected adapter I/O') +} + +/** + * @param {string} entrySource + * @param {Map>>} wrappers + * @param {{ namespace: string, path: string, pluginData: object }} originalTarget + */ +async function testEsbuild (entrySource, wrappers, originalTarget) { + const outfile = join(temporaryDirectory, 'esbuild.cjs') + await esbuild.build({ + bundle: true, + format: 'cjs', + platform: 'node', + outfile, + stdin: { + contents: entrySource, + loader: 'js', + resolveDir: temporaryDirectory + }, + plugins: [{ + name: 'iitm-test-adapter', + setup (build) { + build.onResolve({ filter: /^iitm-virtual-/ }, args => ({ + path: args.path === 'iitm-virtual-esm' ? 'esm' : 'commonjs', + namespace: 'iitm-wrapper' + })) + build.onResolve({ filter: /^\.\/__iitm_/, namespace: 'iitm-wrapper' }, args => { + const wrapper = wrappers.get(args.importer) + const entry = wrapper.imports.find(entry => entry.specifier === args.path) + if (entry.kind === 'runtime') return { path: fileURLToPath(entry.url) } + return entry.target + }) + build.onLoad({ filter: /.*/, namespace: 'iitm-wrapper' }, args => ({ + contents: wrappers.get(args.path).code, + loader: 'js', + resolveDir: temporaryDirectory + })) + build.onLoad({ filter: /.*/, namespace: originalTarget.namespace }, async args => { + strictEqual(args.path, originalTarget.path) + strictEqual(args.pluginData, originalTarget.pluginData) + return { contents: await readFile(args.path), loader: 'js' } + }) + } + }] + }) + + deepStrictEqual(runBundle(outfile), { esm: 43, commonjs: 44 }) +} + +/** + * @param {string} entrySource + * @param {Map>>} wrappers + */ +async function testWebpack (entrySource, wrappers) { + const webpackDirectory = join(temporaryDirectory, 'webpack') + const outputDirectory = join(webpackDirectory, 'dist') + const wrappersByContext = new Map() + await mkdir(outputDirectory, { recursive: true }) + + const entryPath = join(webpackDirectory, 'entry.cjs') + await writeFile(entryPath, entrySource) + for (const [name, wrapper] of wrappers) { + const directory = name === 'commonjs' ? temporaryDirectory : join(webpackDirectory, name) + const filename = join(directory, wrapper.format === 'module' ? 'wrapper.mjs' : 'wrapper.cjs') + await mkdir(directory, { recursive: true }) + await writeFile(filename, wrapper.code) + wrappersByContext.set(directory, { filename, wrapper }) + } + + const replacement = new webpack.NormalModuleReplacementPlugin( + /^(?:iitm-virtual-|\.\/__iitm_)/, + resource => { + if (resource.request === 'iitm-virtual-esm') { + resource.request = wrappersByContext.get(join(webpackDirectory, 'esm')).filename + return + } + if (resource.request === 'iitm-virtual-commonjs') { + resource.request = wrappersByContext.get(temporaryDirectory).filename + return + } + + const { wrapper } = wrappersByContext.get(resource.context) + const entry = wrapper.imports.find(entry => entry.specifier === resource.request) + resource.request = entry.kind === 'runtime' ? fileURLToPath(entry.url) : entry.target.path + } + ) + + const stats = await runWebpack({ + entry: entryPath, + mode: 'development', + target: 'node', + devtool: false, + output: { + path: outputDirectory, + filename: 'bundle.cjs', + chunkFilename: '[name].cjs' + }, + plugins: [replacement] + }) + const errors = stats.toJson({ all: false, errors: true }).errors + deepStrictEqual(errors, []) + deepStrictEqual(runBundle(join(outputDirectory, 'bundle.cjs')), { esm: 43, commonjs: 44 }) +} + +/** + * @param {import('webpack').Configuration} configuration + * @returns {Promise} + */ +function runWebpack (configuration) { + return new Promise((resolve, reject) => { + webpack(configuration, (error, stats) => { + if (error) return reject(error) + resolve(stats) + }) + }) +} + +/** + * @param {string} filename + * @returns {{ esm: number, commonjs: number }} + */ +function runBundle (filename) { + const result = spawnSync(process.execPath, [filename], { encoding: 'utf8' }) + strictEqual(result.status, 0, result.stderr) + return JSON.parse(result.stdout) +} diff --git a/test/other/v20-nft-runtime.mjs b/test/other/v20-nft-runtime.mjs new file mode 100644 index 00000000..3ad4469e --- /dev/null +++ b/test/other/v20-nft-runtime.mjs @@ -0,0 +1,12 @@ +import { ok, strictEqual } from 'node:assert/strict' +import { fileURLToPath } from 'node:url' + +import { nodeFileTrace } from '@vercel/nft' + +const packageRoot = fileURLToPath(new URL('../../', import.meta.url)) +const { fileList, warnings } = await nodeFileTrace(['register-hooks.mjs'], { base: packageRoot }) + +strictEqual(warnings.size, 0) +ok(fileList.has('register-hooks.mjs')) +ok(fileList.has('create-hook.mjs')) +ok(fileList.has('lib/register.js'), 'the generated wrapper runtime must remain reachable to file tracers') diff --git a/test/register/v18.19-loader-url-escaping.mjs b/test/register/v18.19-loader-url-escaping.mjs index 7b15430b..5b58a4dd 100644 --- a/test/register/v18.19-loader-url-escaping.mjs +++ b/test/register/v18.19-loader-url-escaping.mjs @@ -23,7 +23,9 @@ try { 'lib/get-esm-exports.mjs', 'lib/get-exports.mjs', 'lib/io.mjs', - 'lib/register.js' + 'lib/register.js', + 'lib/source.mjs', + 'lib/wrapper.mjs' ] const setupPromises = [] for (const filename of packageFiles) { diff --git a/test/register/v22.15-sync-register-hooks-commonjs.mjs b/test/register/v22.15-sync-register-hooks-commonjs.mjs new file mode 100644 index 00000000..3b94e5b9 --- /dev/null +++ b/test/register/v22.15-sync-register-hooks-commonjs.mjs @@ -0,0 +1,140 @@ +import { deepStrictEqual, ok, strictEqual } from 'node:assert/strict' +import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises' +import * as nodeModule from 'node:module' +import { tmpdir } from 'node:os' +import { basename, join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import Hook from '../../index.js' +import { register, supportsSyncHooks } from '../../register-hooks.mjs' + +if (!supportsSyncHooks()) { + console.log(`Skipping ${process.env.IITM_TEST_FILE || import.meta.url}: synchronous hooks unsupported on this Node.js`) + process.exit(0) +} + +const commonJsUrl = new URL('../fixtures/sync-commonjs-semantics.cjs', import.meta.url) +const cycleAUrl = new URL('../fixtures/sync-commonjs-cycle-a.cjs', import.meta.url) +const cycleBUrl = new URL('../fixtures/sync-commonjs-cycle-b.cjs', import.meta.url) +const commonJsTypeScriptUrl = new URL('../fixtures/sync-commonjs-typescript.cts', import.meta.url) +const importedCommonJsUrl = new URL('../fixtures/something.js', import.meta.url) +const requiredEsmUrl = new URL('../fixtures/something.mjs', import.meta.url) +const requiredPackageEsmUrl = new URL('../fixtures/type-module/module.js', import.meta.url) +const includedUrls = new Set([ + commonJsUrl.href, + cycleAUrl.href, + cycleBUrl.href, + commonJsTypeScriptUrl.href, + importedCommonJsUrl.href, + requiredEsmUrl.href, + requiredPackageEsmUrl.href, + 'node:fs' +]) + +register({ + commonjs: true, + shouldInclude (url) { + if (!includedUrls.has(url)) return false + return { data: { filename: url.startsWith('file:') ? basename(fileURLToPath(url)) : url } } + } +}) + +const require = nodeModule.createRequire(import.meta.url) +const commonJsFilename = fileURLToPath(commonJsUrl) +let commonJsHookCount = 0 + +const commonJsHook = new Hook([commonJsFilename], (exports, name, baseDir, data) => { + commonJsHookCount++ + strictEqual(name, commonJsFilename) + strictEqual(baseDir, undefined) + deepStrictEqual(data, { filename: 'sync-commonjs-semantics.cjs' }) + return { ...exports, hooked: true } +}) + +const first = require(commonJsFilename) +strictEqual(first.value, 42) +strictEqual(first.hooked, true) +strictEqual(first.unreachable, undefined) +strictEqual(first.topLevelThis, first.argumentExports) +strictEqual(require(commonJsFilename), first) +strictEqual(commonJsHookCount, 1) + +const lateHook = new Hook([commonJsFilename], exports => ({ ...exports, late: true })) +const late = require(commonJsFilename) +strictEqual(late.late, true) +strictEqual(late.value, 42) +lateHook.unhook() +commonJsHook.unhook() + +const cycleCounts = new Map() +const cycleHook = new Hook([ + fileURLToPath(cycleAUrl), + fileURLToPath(cycleBUrl) +], (exports, name) => { + cycleCounts.set(name, (cycleCounts.get(name) ?? 0) + 1) + return exports +}) +const cycle = require(fileURLToPath(cycleAUrl)) +deepStrictEqual(cycle, { name: 'a', fromB: 'b', seenByB: 'a' }) +strictEqual(cycleCounts.get(fileURLToPath(cycleAUrl)), 1) +strictEqual(cycleCounts.get(fileURLToPath(cycleBUrl)), 1) +cycleHook.unhook() + +const commonJsTypeScriptHook = new Hook([fileURLToPath(commonJsTypeScriptUrl)], exports => { + exports.value++ +}) +const commonJsTypeScript = require(fileURLToPath(commonJsTypeScriptUrl)) +strictEqual(commonJsTypeScript.value, 43) +commonJsTypeScriptHook.unhook() + +const importedHook = new Hook([fileURLToPath(importedCommonJsUrl)], exports => { + exports.foo = 43 +}) +const imported = await import(importedCommonJsUrl) +strictEqual(imported.foo, 43) +strictEqual(imported.default.foo, 43) +importedHook.unhook() + +const esmHook = new Hook([fileURLToPath(requiredEsmUrl)], (exports, name, baseDir, data) => { + deepStrictEqual(data, { filename: 'something.mjs' }) + exports.foo = 57 +}) +const requiredEsm = require(fileURLToPath(requiredEsmUrl)) +strictEqual(requiredEsm.foo, 57) +esmHook.unhook() + +const packageEsmHook = new Hook([fileURLToPath(requiredPackageEsmUrl)], exports => { + exports.value = 57 +}) +const requiredPackageEsm = require(fileURLToPath(requiredPackageEsmUrl)) +strictEqual(requiredPackageEsm.value, 57) +packageEsmHook.unhook() + +const fsHook = new Hook(['fs'], (exports, name, baseDir, data) => { + strictEqual(name, 'fs') + strictEqual(baseDir, undefined) + deepStrictEqual(data, { filename: 'node:fs' }) + exports[Symbol.for('iitm.sync-commonjs')] = true + return exports +}) +const fs = require('fs') +strictEqual(fs, require('node:fs')) +strictEqual(fs[Symbol.for('iitm.sync-commonjs')], true) +ok(Object.isExtensible(fs)) +fsHook.unhook() + +const temporaryDirectory = await realpath(await mkdtemp(join(tmpdir(), 'iitm-commonjs-'))) +try { + const packageLessFilename = join(temporaryDirectory, 'package-less.js') + const packageLessUrl = pathToFileURL(packageLessFilename) + await writeFile(packageLessFilename, 'module.exports = { value: 42 }\n') + includedUrls.add(packageLessUrl.href) + + const packageLessHook = new Hook([packageLessFilename], exports => { + exports.value++ + }) + strictEqual(require(packageLessFilename).value, 43) + packageLessHook.unhook() +} finally { + await rm(temporaryDirectory, { recursive: true, force: true }) +} diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts index 5c8ea9a5..da3f0061 100644 --- a/test/typescript/bundler.test.mts +++ b/test/typescript/bundler.test.mts @@ -3,12 +3,15 @@ import assert from 'node:assert/strict' import { createWrapperModule } from '../../bundler.mjs' const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href +const target = { namespace: 'file', path: moduleUrl } const wrapper = await createWrapperModule({ module: { url: moduleUrl, format: 'module', source: 'export const value = 42', - specifier: './something.mjs' + specifier: './something.mjs', + target, + data: { version: '1.0.0' } }, resolve () { throw new Error('Unexpected resolve') @@ -19,4 +22,6 @@ const wrapper = await createWrapperModule({ }) assert.equal(wrapper.sideEffects, true) +assert.equal(wrapper.format, 'module') assert.equal(wrapper.imports[0].kind, 'runtime') +assert.equal(wrapper.imports[1].target, target) diff --git a/test/typescript/register-hooks.test.mts b/test/typescript/register-hooks.test.mts new file mode 100644 index 00000000..939f4a08 --- /dev/null +++ b/test/typescript/register-hooks.test.mts @@ -0,0 +1,15 @@ +import assert from 'node:assert/strict' + +import type { RegisterHooksOptions } from '../../register-hooks.mjs' + +type Data = { version: string } + +const options: RegisterHooksOptions = { + commonjs: true, + shouldInclude (url) { + if (!url.startsWith('file:')) return false + return { data: { version: '1.0.0' } } + } +} + +assert.equal(options.commonjs, true) From ff4283885101f749157b009dfbaa72934a49eb92 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 12:50:24 +0200 Subject: [PATCH 03/18] refactor: isolate CommonJS instrumentation paths - Keep CommonJS and bundler metadata on opt-in IITM paths. - Remove bundler-specific dependencies, fixtures, and license churn. Existing ESM wrapper registration and callback paths must retain their hot-path behavior. The standalone register-hooks TypeScript test also bypassed the TypeScript loader on Node 18 and stopped the CI matrix. - npm test - npm run test:ts - npm run test:e2e - npm run lint --- .eslintrc.yaml | 2 - README.md | 84 ++-- bundler.d.mts | 49 +- bundler.mjs | 96 ++-- create-hook.mjs | 436 +++++++++++------- index.d.ts | 17 +- index.js | 157 +++++-- lib/bundler-runtime.js | 8 +- lib/get-exports.mjs | 12 +- lib/register.js | 67 ++- lib/source.mjs | 24 - lib/wrapper.mjs | 67 ++- package.json | 5 +- register-hooks.d.ts | 12 +- register-hooks.mjs | 16 +- test/fixtures/inspectable-create-hook.mjs | 7 +- test/fixtures/sync-commonjs-cycle-a.cjs | 6 - test/fixtures/sync-commonjs-cycle-b.cjs | 5 - test/fixtures/sync-commonjs-semantics.cjs | 29 -- test/fixtures/sync-commonjs-typescript.cts | 3 - test/fixtures/type-module/module.js | 1 - test/fixtures/type-module/package.json | 3 - test/low-level/bundler.mjs | 186 +++++--- test/other/v18-bundlers.mjs | 208 --------- test/other/v20-nft-runtime.mjs | 12 - test/register/v18.19-loader-url-escaping.mjs | 1 - .../v22.15-sync-register-hooks-commonjs.mjs | 272 ++++++----- test/typescript/bundler.test.mts | 4 - test/typescript/register-hooks.test.mts | 15 - 29 files changed, 891 insertions(+), 913 deletions(-) delete mode 100644 lib/source.mjs delete mode 100644 test/fixtures/sync-commonjs-cycle-a.cjs delete mode 100644 test/fixtures/sync-commonjs-cycle-b.cjs delete mode 100644 test/fixtures/sync-commonjs-semantics.cjs delete mode 100644 test/fixtures/sync-commonjs-typescript.cts delete mode 100644 test/fixtures/type-module/module.js delete mode 100644 test/fixtures/type-module/package.json delete mode 100644 test/other/v18-bundlers.mjs delete mode 100644 test/other/v20-nft-runtime.mjs delete mode 100644 test/typescript/register-hooks.test.mts diff --git a/.eslintrc.yaml b/.eslintrc.yaml index a45ff2ed..f95abcd8 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -30,5 +30,3 @@ ignorePatterns: - test/fixtures/reexport-same-source.mjs - test/fixtures/reexport-explicit-override.mjs - test/fixtures/reexport-nested-agg.mjs - - test/fixtures/type-module/module.js - - test/fixtures/sync-commonjs-typescript.cts diff --git a/README.md b/README.md index 87ffbe05..151f6ad8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,8 @@ # import-in-the-middle **`import-in-the-middle`** is a module loading interceptor inspired by -[`require-in-the-middle`](https://npm.im/require-in-the-middle). It supports ESM -through Node.js loader hooks and can also intercept CommonJS when synchronous -hooks are available. +[`require-in-the-middle`](https://npm.im/require-in-the-middle). It intercepts +ESM modules and can opt into CommonJS when synchronous hooks are available. ## Usage @@ -117,41 +116,33 @@ node --import=./instrument.mjs ./my-app.mjs ## Bundler integrations -Bundlers can generate ESM and CommonJS wrappers with `createWrapperModule`: +Bundlers can generate ESM and CommonJS wrappers with +`createWrapperModule`: ```js import { createWrapperModule } from 'import-in-the-middle/bundler.mjs' const wrapper = await createWrapperModule({ - module: { url, format, source, specifier, target, data }, + module: { url, format, source, specifier, data }, resolve, load }) ``` -`url` is the canonical `file:` or `node:` URL reported to hooks. `specifier` is -the original request. `target` is the bundler's opaque resolved value, such as -an esbuild path/namespace/plugin-data object or a webpack resource. IITM passes -it back unchanged in the import manifest and to `load(target, context)`. - -`resolve(specifier, context)` returns `{ url, format, target, watchFiles }`. -IITM uses `url` and `format` to inspect re-exports while preserving `target` for -the adapter. This keeps namespaces, query strings, external decisions, and -loader state owned by the bundler. - -The result contains generated `code`, its `format`, an `imports` manifest, -`watchFiles`, and `sideEffects: true`. ESM code imports relative placeholder -specifiers. The adapter maps each placeholder to the corresponding manifest -entry. CommonJS code contains the original source and only requires the runtime -placeholder, so the bundler can still discover literal `require()` calls in the -module source. The adapter must resolve those calls from the original module's -context. `watchFiles` are canonical URLs that the adapter converts to its native -watch-dependency format. - -`data` is optional JSON-serializable consumer metadata. IITM embeds it in the -wrapper and passes it as the fourth argument to `Hook` callbacks. This lets an -adapter carry package names, versions, or other build-time facts into a bundle -without retaining build-machine paths. +`url` is the canonical `file:` or `node:` URL reported to hooks. `resolve` and +`load` adapt the bundler's resolver and source loader to the same URL-based +module graph. + +The optional `data` value must be JSON-serializable. It is embedded in the +wrapper and passed as the fourth argument to `Hook` callbacks, allowing package +metadata needed by instrumentation to reach the bundled runtime. + +The result contains generated `code`, an `imports` manifest, `watchFiles`, and +`sideEffects: true`. The code imports only relative placeholder specifiers. The +bundler adapter provides it as a virtual module and maps each placeholder using +the manifest, so filesystem paths, virtual IDs, external modules, and cache +invalidation remain owned by the bundler. `watchFiles` are file URLs that the +adapter converts to its native watch-dependency format. The runtime import in the manifest must be bundled with the wrapper. Keeping it external can create a second hook registry at runtime. It is CommonJS and must @@ -178,10 +169,7 @@ pulled into the ESM graph until [nodejs/node#59929][]. The fix shipped in import { register, supportsSyncHooks } from 'import-in-the-middle/register-hooks.mjs' if (supportsSyncHooks()) { - register({ - include: ['package-i-want-to-include'], - commonjs: true - }) + register({ include: ['package-i-want-to-include'] }) } else { // Fall back to the asynchronous loader, e.g. module.register('import-in-the-middle/hook.mjs'). } @@ -195,10 +183,7 @@ if (supportsSyncHooks()) { import { register } from 'import-in-the-middle/register-hooks.mjs' import { Hook } from 'import-in-the-middle' -register({ - include: ['package-i-want-to-include'], - commonjs: true -}) +register({ include: ['package-i-want-to-include'], commonjs: true }) Hook(['package-i-want-to-include'], (exported, name, baseDir) => { // Instrument the module @@ -211,21 +196,17 @@ node --import=./instrument.mjs ./my-app.mjs `register()` accepts the same `include` / `exclude` options as the asynchronous loader and throws on a Node.js version where `supportsSyncHooks()` is `false`. -Set `commonjs: true` to intercept CommonJS `require()` and CommonJS imported -from ESM through the same `Hook` registry. It is opt-in because consumers that -also install `require-in-the-middle` must disable one CommonJS path to avoid -instrumenting a module twice. ESM loaded through `require()` is intercepted by -the synchronous ESM wrapper. +Set `commonjs: true` to intercept both `require()` and ESM loaded through +`require()`. This is opt-in so consumers can continue using +`require-in-the-middle` for CommonJS without double instrumentation. ### Custom matching with `shouldInclude` Instead of `include` / `exclude` lists, you can pass a `shouldInclude(url, specifier)` predicate to decide which modules are intercepted. It is called for every resolved -module with the resolved URL and the import specifier. Return `true` to intercept -the module, or return `{ data }` to intercept it and pass consumer metadata to -the `Hook` callback. `data` must be JSON-serializable. When a predicate is -provided it takes over the decision and the `include` / `exclude` options are -ignored. +module with the resolved URL and the import specifier; return a truthy value to +intercept the module. When a predicate is provided it takes over the decision and +the `include` / `exclude` options are ignored. This is useful when matching doesn't map cleanly onto bare specifiers, file URLs and regular expressions — for example a matcher built from your own configuration, or a @@ -235,16 +216,11 @@ decision that depends on more than the specifier. import { register } from 'import-in-the-middle/register-hooks.mjs' register({ - commonjs: true, shouldInclude (url, specifier) { - if (specifier !== 'package-i-want-to-include') return false - return { data: { version: '1.2.3' } } + return specifier === 'package-i-want-to-include' || + url.includes('/node_modules/some-scope/') } }) - -Hook(['package-i-want-to-include'], (exported, name, baseDir, data) => { - console.log(data.version) -}) ``` The predicate receives only the URL and the specifier, never a resolved file path. @@ -315,7 +291,7 @@ On Node.js versions where type stripping is not enabled by default, run with * While bindings to module exports end up being "re-bound" when modified in a hook, dynamically imported modules cannot be altered after they're loaded. * Modules loaded via `require` are only affected by synchronous registration - with `commonjs: true`, or when the required target is ESM. + with `commonjs: true`. * A module's set of export *names* is assumed to be stable for the lifetime of the process. `import-in-the-middle` reads a module's source once to lex its exports and reuses that export set on later loads of the same URL. An upstream diff --git a/bundler.d.mts b/bundler.d.mts index bdd7128d..5d6b2996 100644 --- a/bundler.d.mts +++ b/bundler.d.mts @@ -1,27 +1,18 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. - export type WrapperSource = string | ArrayBuffer | ArrayBufferView export type JsonValue = | boolean + | null | number | string | JsonValue[] | { [key: string]: JsonValue } -export type ModuleTarget = { - url: string - format?: string -} - -export type BundlerModule = { +export type BundlerModule = { url: string format: string specifier: string source?: WrapperSource - target?: Target data?: Data } @@ -30,10 +21,12 @@ export type ModuleContext = { parentURL?: string } -export type ResolveResult = { +export type ModuleTarget = { url: string format?: string - target?: Target +} + +export type ResolveResult = ModuleTarget & { watchFiles?: Iterable } @@ -43,38 +36,32 @@ export type LoadResult = { watchFiles?: Iterable } -export type WrapperImport = { +export type WrapperImport = { specifier: string kind: 'module' | 'runtime' - url: string - format?: string - target: Target | ModuleTarget + target: ModuleTarget + external: boolean } -export type WrapperModule = { +export type WrapperModule = { code: string - format: 'module' | 'commonjs' - imports: WrapperImport[] + imports: WrapperImport[] watchFiles: string[] sideEffects: true } -export type CreateWrapperModuleOptions< - Target = ModuleTarget, - Data extends JsonValue = JsonValue -> = { - module: BundlerModule +export type CreateWrapperModuleOptions = { + module: BundlerModule resolve: ( specifier: string, context: ModuleContext - ) => ResolveResult | Promise> + ) => ResolveResult | Promise load: ( - target: Target | ModuleTarget, + url: string, context: ModuleContext ) => LoadResult | Promise } -export declare function createWrapperModule< - Target = ModuleTarget, - Data extends JsonValue = JsonValue ->(options: CreateWrapperModuleOptions): Promise> +export declare function createWrapperModule( + options: CreateWrapperModuleOptions +): Promise diff --git a/bundler.mjs b/bundler.mjs index 52ed15a6..9ed31667 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -1,13 +1,12 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. - 'use strict' +import { builtinModules } from 'module' + import { driveAsync } from './lib/io.mjs' import { buildCommonJSWrapperSource, buildWrapperSource, + buildWrapperSourceWithData, processModule } from './lib/wrapper.mjs' @@ -21,7 +20,6 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @property {string} format * @property {string} specifier * @property {string | ArrayBuffer | ArrayBufferView} [source] - * @property {unknown} [target] * @property {unknown} [data] */ @@ -35,7 +33,6 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @typedef {object} ResolveResult * @property {string} url * @property {string} [format] - * @property {unknown} [target] * @property {Iterable} [watchFiles] */ @@ -43,9 +40,8 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @typedef {object} WrapperImport * @property {string} specifier * @property {'module' | 'runtime'} kind - * @property {string} url - * @property {string} [format] - * @property {unknown} target + * @property {{ url: string, format?: string }} target + * @property {boolean} external */ /** @@ -56,16 +52,15 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href */ /** - * Creates a format-aware wrapper without embedding bundler-specific module identifiers. + * Creates an ESM wrapper without embedding bundler-specific module identifiers. * * @param {object} options * @param {BundlerModule} options.module * @param {(specifier: string, context: ModuleContext) => * (ResolveResult | Promise)} options.resolve - * @param {(target: unknown, context: ModuleContext) => (LoadResult | Promise)} options.load + * @param {(url: string, context: ModuleContext) => (LoadResult | Promise)} options.load * @returns {Promise<{ * code: string, - * format: 'module' | 'commonjs', * imports: WrapperImport[], * watchFiles: string[], * sideEffects: true @@ -75,10 +70,6 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } const context = { format: moduleData.format, cache: false } const watchFiles = new Set() const formats = new Map([[moduleData.url, moduleData.format]]) - const targets = new Map([[ - moduleData.url, - moduleData.target ?? { url: moduleData.url, format: moduleData.format } - ]]) if (moduleData.url.startsWith('file:')) { watchFiles.add(moduleData.url) @@ -97,7 +88,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } } } - const result = await load(targets.get(url), loadContext) + const result = await load(url, loadContext) if (result.format !== undefined) { formats.set(url, result.format) } @@ -127,22 +118,9 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } watchFiles.add(watchFile) } } - targets.set(result.url, result.target ?? { url: result.url, format: result.format }) return result } - /** @type {WrapperImport[]} */ - const imports = [{ - specifier: RUNTIME_SPECIFIER, - kind: 'runtime', - url: runtimeUrl, - format: 'commonjs', - target: { - url: runtimeUrl, - format: 'commonjs' - } - }] - if (moduleData.format === 'commonjs' || moduleData.format === 'commonjs-typescript') { let source = moduleData.source if (source === undefined) { @@ -161,22 +139,35 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } data: moduleData.data, runtimeSpecifier: RUNTIME_SPECIFIER }), - format: 'commonjs', - imports, + imports: [{ + specifier: RUNTIME_SPECIFIER, + kind: 'runtime', + target: { + url: runtimeUrl, + format: 'commonjs' + }, + external: false + }], watchFiles: Array.from(watchFiles), sideEffects: true } } - if (moduleData.format !== 'module' && moduleData.format !== 'module-typescript' && moduleData.format !== 'builtin') { - throw new TypeError(`Unsupported module format '${moduleData.format}'`) - } - const { bindings } = await driveAsync( processModule({ srcUrl: moduleData.url, context }), { resolve: resolveModule, load: loadModule } ) + /** @type {WrapperImport[]} */ + const imports = [{ + specifier: RUNTIME_SPECIFIER, + kind: 'runtime', + target: { + url: runtimeUrl, + format: 'commonjs' + }, + external: false + }] const moduleSpecifiers = new Map() /** @@ -191,26 +182,35 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } imports.push({ specifier, kind: 'module', - url, - format: formats.get(url), - target: targets.get(url) + target: { + url, + format: formats.get(url) + }, + external: url.startsWith('node:') || builtinModules.includes(url) }) } return specifier } - const code = buildWrapperSource({ - realUrl: moduleData.url, - bindings, - originalSpecifier: moduleData.specifier, - data: moduleData.data, - runtimeSpecifier: RUNTIME_SPECIFIER, - mapImport - }) + const code = moduleData.data === undefined + ? buildWrapperSource({ + realUrl: moduleData.url, + bindings, + originalSpecifier: moduleData.specifier, + runtimeSpecifier: RUNTIME_SPECIFIER, + mapImport + }) + : buildWrapperSourceWithData({ + realUrl: moduleData.url, + bindings, + originalSpecifier: moduleData.specifier, + data: moduleData.data, + runtimeSpecifier: RUNTIME_SPECIFIER, + mapImport + }) return { code, - format: 'module', imports, watchFiles: Array.from(watchFiles), sideEffects: true diff --git a/create-hook.mjs b/create-hook.mjs index f0440da8..baf49556 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -2,19 +2,11 @@ // // This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. -import { readFileSync } from 'fs' -import { builtinModules } from 'module' -import { dirname, extname, join } from 'path' import { URL, fileURLToPath } from 'url' import { inspect } from 'util' +import { builtinModules } from 'module' import { driveSync, driveAsync } from './lib/io.mjs' -import './lib/register.js' -import { sourceToString } from './lib/source.mjs' -import { - buildCommonJSWrapperSource, - buildWrapperSource, - processModule -} from './lib/wrapper.mjs' +import { buildCommonJSWrapperSource, buildWrapperSource, processModule } from './lib/wrapper.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' // Re-exported for backwards compatibility: `supportsSyncHooks` now lives in its @@ -24,29 +16,20 @@ export { supportsSyncHooks } const isWin = process.platform === 'win32' - // FIXME: Typescript extensions are added temporarily until we find a better // way of supporting arbitrary extensions const EXTENSION_RE = /\.(js|mjs|cjs|ts|mts|cts)$/ -// The full es-module-lexer build handles erasable TypeScript syntax in the same -// pass as JavaScript, so the `-typescript` formats use the normal export path. +// The `-typescript` formats are listed unconditionally; getExports strips the +// types when the runtime supports it and otherwise falls back to onWrapFailure. const HANDLED_FORMATS = new Set([ 'builtin', 'module', 'commonjs', 'module-typescript', 'commonjs-typescript' ]) const TRACE_WARNINGS = process.execArgv.includes('--trace-warnings') -const stripTypeScriptTypes = process.getBuiltinModule?.('module')?.stripTypeScriptTypes -const packageTypes = new Map() +let packageTypes /** @typedef {import('node:module').LoadHookContext} LoadContext */ /** @typedef {import('node:module').LoadFnOutput} LoadResult */ -/** @typedef {{ name: string, origin: string }} StarBinding */ -/** - * @typedef {object} SpecifierData - * @property {string} specifier - * @property {string} [format] - * @property {unknown} [data] - * @property {boolean} [commonjs] - */ +/** @typedef {string | { specifier: string, format: 'module-typescript' | 'commonjs-typescript' }} SpecifierData */ function hasIitm (url) { // Fast path: avoid URL parsing on the hot path when there's clearly no iitm. @@ -90,50 +73,6 @@ function deleteIitm (url) { return resultUrl } - -/** - * @param {string} url - * @returns {string | undefined} - */ -function getFileFormat (url) { - if (!url.startsWith('file:')) return undefined - const filename = fileURLToPath(url) - const extension = extname(filename) - if (extension === '.mjs') return 'module' - if (extension === '.cjs') return 'commonjs' - if (extension === '.mts') return 'module-typescript' - if (extension === '.cts') return 'commonjs-typescript' - if (extension !== '.js' && extension !== '.ts') return undefined - - let directory = dirname(filename) - const visited = [] - while (true) { - if (packageTypes.has(directory)) { - const type = packageTypes.get(directory) - for (const visitedDirectory of visited) packageTypes.set(visitedDirectory, type) - return type === 'module' - ? (extension === '.ts' ? 'module-typescript' : 'module') - : (extension === '.ts' ? 'commonjs-typescript' : 'commonjs') - } - - visited.push(directory) - try { - const type = JSON.parse(readFileSync(join(directory, 'package.json'), 'utf8')).type - packageTypes.set(directory, type) - continue - } catch (error) { - if (error.code !== 'ENOENT') return undefined - } - - const parent = dirname(directory) - if (parent === directory) { - for (const visitedDirectory of visited) packageTypes.set(visitedDirectory, undefined) - return extension === '.ts' ? 'commonjs-typescript' : 'commonjs' - } - directory = parent - } -} - /** * Determines whether the input is a bare specifier, file URL or a regular expression. * @@ -203,26 +142,85 @@ function emitWarning (err) { process.emitWarning(warnMessage) } - function addIitm (url) { const urlObj = new URL(url) urlObj.searchParams.set('iitm', 'true') return urlObj.href } +/** + * @param {'.js'|'.ts'} extension + * @param {string|undefined} type + * @returns {'module'|'module-typescript'|'commonjs'|'commonjs-typescript'} + */ +function getPackageFormat (extension, type) { + if (type === 'module') return extension === '.ts' ? 'module-typescript' : 'module' + return extension === '.ts' ? 'commonjs-typescript' : 'commonjs' +} + +/** + * @param {string} url + * @returns {string|undefined} + */ +function getFileFormat (url) { + if (!url.startsWith('file:')) return undefined + const pathname = new URL(url).pathname + let extension + if (pathname.endsWith('.mjs')) extension = '.mjs' + else if (pathname.endsWith('.cjs')) extension = '.cjs' + else if (pathname.endsWith('.mts')) extension = '.mts' + else if (pathname.endsWith('.cts')) extension = '.cts' + else if (pathname.endsWith('.js')) extension = '.js' + else if (pathname.endsWith('.ts')) extension = '.ts' + else return undefined + + if (extension === '.mjs') return 'module' + if (extension === '.cjs') return 'commonjs' + if (extension === '.mts') return 'module-typescript' + if (extension === '.cts') return 'commonjs-typescript' + + packageTypes ??= new Map() + const visited = [] + let directory = new URL('.', url) + while (true) { + if (packageTypes.has(directory.href)) { + const type = packageTypes.get(directory.href) + for (const href of visited) packageTypes.set(href, type) + return getPackageFormat(extension, type) + } + + visited.push(directory.href) + try { + const source = process.getBuiltinModule('fs').readFileSync(new URL('package.json', directory), 'utf8') + const type = JSON.parse(source).type + packageTypes.set(directory.href, type) + continue + } catch (error) { + if (error.code !== 'ENOENT') return undefined + } + + const parent = new URL('../', directory) + if (parent.href === directory.href) { + for (const href of visited) packageTypes.set(href, undefined) + return getPackageFormat(extension, undefined) + } + directory = parent + } +} + /** * @param {{ url: string }} meta + * @param {boolean} [commonjs] Whether to create CommonJS-specific synchronous hooks. */ -export function createHook (meta) { +export function createHook (meta, commonjs) { /** @type {Map} */ const specifiers = new Map() + let commonJsSpecifiers let cachedResolve - const iitmURL = new URL('lib/register.js', meta.url).href - const iitmPath = fileURLToPath(iitmURL) + const iitmURL = new URL('lib/register.js', meta.url).toString() let includeModules, excludeModules let shouldInclude = defaultShouldInclude let disableCjsSourceStripping = false - let hookCommonJS = false // Track CJS module URLs that IITM has wrapped. On Node 24+, CJS modules loaded // via loadCJSModule (in an ESM import chain) have their require() calls for @@ -277,7 +275,7 @@ export function createHook (meta) { function applyOptions (data) { includeModules = ensureArrayWithBareSpecifiersFileUrlsAndRegex(data.include, 'include') excludeModules = ensureArrayWithBareSpecifiersFileUrlsAndRegex(data.exclude, 'exclude') - hookCommonJS = data.commonjs === true + disableCjsSourceStripping = data.disableCjsSourceStripping === true // A consumer can supply its own matcher as `shouldInclude(url, specifier)`, // taking ownership of the include/exclude decision instead of expressing it @@ -286,10 +284,6 @@ export function createHook (meta) { // default applies the include/exclude options. shouldInclude = typeof data.shouldInclude === 'function' ? data.shouldInclude : defaultShouldInclude - if (data.disableCjsSourceStripping === true) { - disableCjsSourceStripping = true - } - if (data.addHookMessagePort) { data.addHookMessagePort.on('message', (modules) => { if (includeModules === undefined) { @@ -325,7 +319,7 @@ export function createHook (meta) { // once the parent loader has turned the specifier into a resolved URL. The // only difference between the asynchronous and synchronous hooks is whether // that resolution was awaited, so all the wrapping decisions live here. - function finishResolve (result, specifier, context, parentURL, synchronous) { + function finishResolve (result, specifier, context, parentURL) { // Do not wrap the entrypoint module. Many CLIs check whether they are the // "main" module (e.g. require.main === module). Wrapping changes how they // are evaluated, and can make them exit without doing anything. @@ -342,26 +336,24 @@ export function createHook (meta) { return result } - const isRequire = context.conditions?.includes('require') === true - let format = result.format - let isModule = format === 'module' || format === 'module-typescript' - - // Without the opt-in, keep CommonJS owned by require-in-the-middle. ESM - // loaded through require() can still use the synchronous ESM wrapper. - if (isRequire && !isModule && (!synchronous || !hookCommonJS)) { + // The synchronous hooks (`module.registerHooks`) fire for `require()` as well + // as `import`, but iitm only owns the ESM graph: CommonJS modules are + // instrumented separately through require-in-the-middle, and `require()` must + // return the native, mutable module value (e.g. graceful-fs does + // `Object.defineProperty(require('fs'), ...)`, which throws on a frozen ESM + // namespace). Node reports the active module system in `context.conditions` + // ('require' vs 'import'), so leave any require() resolution untouched. The + // asynchronous hook never sees the 'require' condition, so this is a no-op + // there and only affects the synchronous path. + if (context.conditions?.includes('require')) { return result } - const inclusion = shouldInclude(result.url, specifier) - if (!inclusion) { + // `shouldInclude` is always set (the include/exclude list matcher by default, + // a consumer-provided predicate otherwise), so no nullish check is needed. + if (!shouldInclude(result.url, specifier)) { return result } - const data = typeof inclusion === 'object' ? inclusion.data : undefined - if (synchronous && hookCommonJS && format == null) { - format = getFileFormat(result.url) - isModule = format === 'module' || format === 'module-typescript' - } - const isCommonJS = format === 'commonjs' || format === 'commonjs-typescript' if (isIitm(parentURL, meta) || (parentURL && hasIitm(parentURL))) { return result @@ -398,20 +390,11 @@ export function createHook (meta) { } } - if (synchronous && hookCommonJS && (isCommonJS || (isRequire && !isModule))) { - specifiers.set(result.url, { - specifier, - format, - data, - commonjs: true - }) - return result - } - - if (isRequire && !isModule) return result - // Preserve the format before an outer loader can normalize it. - specifiers.set(result.url, { specifier, format, data }) + const specifierData = result.format === 'module-typescript' || result.format === 'commonjs-typescript' + ? { specifier, format: result.format } + : specifier + specifiers.set(result.url, specifierData) return { url: addIitm(result.url), @@ -419,7 +402,60 @@ export function createHook (meta) { // Node's synchronous resolver drops `format: 'builtin'` for bare builtin // specifiers (`require('crypto')` -> `node:crypto`), so restore it; // otherwise the load hook reads `node:crypto` from disk and throws ENOENT. - format: format ?? (result.url.startsWith('node:') ? 'builtin' : undefined) + format: result.format ?? (result.url.startsWith('node:') ? 'builtin' : undefined) + } + } + + /** + * @param {{ url: string, format?: string }} result + * @param {string} specifier + * @param {object} context + * @param {string} parentURL + * @returns {object} + */ + let finishRequireResolve + if (commonjs === true) { + finishRequireResolve = (result, specifier, context, parentURL) => { + if (parentURL === '') { + if (!EXTENSION_RE.test(result.url) && !hasIitm(result.url)) { + return { url: result.url, format: 'commonjs' } + } + return result + } + + if (result.format && !HANDLED_FORMATS.has(result.format)) return result + if (!shouldInclude(result.url, specifier)) return result + if (isIitm(parentURL, meta) || (parentURL && hasIitm(parentURL))) return result + if (cjsInIitmChain.has(parentURL)) { + cjsInIitmChain.add(result.url) + return result + } + if (result.url.endsWith('.node')) return result + + const importAttributes = context.importAttributes || context.importAssertions + if (importAttributes && importAttributes.type === 'json') return result + if (result.url === parentURL) { + return { + url: result.url, + shortCircuit: true, + format: result.format + } + } + + const format = result.format ?? (result.url.startsWith('node:') ? 'builtin' : getFileFormat(result.url)) + if (format === 'module' || format === 'module-typescript') { + const specifierData = format === 'module-typescript' ? { specifier, format } : specifier + specifiers.set(result.url, specifierData) + return { + url: addIitm(result.url), + shortCircuit: true, + format + } + } + + commonJsSpecifiers ??= new Map() + commonJsSpecifiers.set(result.url, { specifier, format }) + return result } } @@ -441,7 +477,7 @@ export function createHook (meta) { } const result = await parentResolve(newSpecifier, context) - return finishResolve(result, specifier, context, parentURL, false) + return finishResolve(result, specifier, context, parentURL) } // Synchronous counterpart to `resolve`, for `module.registerHooks`. The @@ -465,18 +501,45 @@ export function createHook (meta) { } const result = nextResolve(newSpecifier, context) - return finishResolve(result, specifier, context, parentURL, true) + return finishResolve(result, specifier, context, parentURL) } /** - * Finalizes a successful wrap and builds its module source. - * - * @param {string} realUrl The URL of the wrapped module. - * @param {LoadContext} context Its loader context. - * @param {SpecifierData} specifierData The module's interception data. - * @param {string[] | Map} bindings Its exported bindings. + * @param {string} specifier + * @param {object} context + * @param {Function} nextResolve + * @returns {object} */ - function onWrapSuccess (realUrl, context, specifierData, bindings) { + let resolveSyncCommonJS + if (commonjs === true) { + resolveSyncCommonJS = (specifier, context, nextResolve) => { + cachedResolve = nextResolve + + if (specifier === iitmURL) { + return { + url: specifier, + shortCircuit: true + } + } + + const { parentURL = '' } = context + const newSpecifier = deleteIitm(specifier) + if (process.platform === 'win32' && parentURL.indexOf('file:node') === 0) { + context.parentURL = '' + } + const result = nextResolve(newSpecifier, context) + if (!context.conditions?.includes('require')) { + return finishResolve(result, specifier, context, parentURL) + } + return finishRequireResolve(result, specifier, context, parentURL) + } + } + + // Bookkeeping shared by the async and sync wrap paths once `processModule` + // succeeds: free the specifier entry early, and remember CJS modules so their + // transitive require() chain bypasses iitm (see `load`). Returns the wrapper + // module source. + function onWrapSuccess (realUrl, context, originalSpecifier, bindings) { specifiers.delete(realUrl) // context.format is set to 'commonjs' by getCjsExports during processModule. if (context.format === 'commonjs') { @@ -485,8 +548,7 @@ export function createHook (meta) { return buildWrapperSource({ realUrl, bindings, - originalSpecifier: specifierData.specifier, - data: specifierData.data, + originalSpecifier, runtimeSpecifier: iitmURL }) } @@ -496,10 +558,6 @@ export function createHook (meta) { // (it just can't be Hook'ed) rather than taking down the whole app. We free // the specifier entry to avoid a leak, and log because a failure here is // usually an iitm bug and would otherwise be very tricky to debug. - /** - * @param {string} realUrl The URL whose wrapper could not be built. - * @param {unknown} cause The parse or wrapper-generation failure. - */ function onWrapFailure (realUrl, cause) { specifiers.delete(realUrl) const err = new Error(`'import-in-the-middle' failed to wrap '${realUrl}'`) @@ -510,43 +568,50 @@ export function createHook (meta) { /** * @param {string} url * @param {LoadResult} result - * @param {SpecifierData} specifierData + * @param {{ specifier: string, format?: string }} specifierData * @returns {LoadResult} */ - function wrapCommonJS (url, result, specifierData) { - specifiers.delete(url) - const format = result.format ?? specifierData.format - let source = result.source - - if (url.startsWith('node:')) { - source = `module.exports = process.getBuiltinModule(${JSON.stringify(url.slice(5))})\n` - } else if ((format === 'commonjs' || format === 'commonjs-typescript') && source == null && url.startsWith('file:')) { - source = readFileSync(fileURLToPath(url)) - } - - if (source == null || (format !== 'commonjs' && format !== 'commonjs-typescript' && !url.startsWith('node:'))) { - return result - } + let wrapCommonJS + if (commonjs === true) { + wrapCommonJS = (url, result, specifierData) => { + commonJsSpecifiers.delete(url) + const format = result.format ?? specifierData.format + let source = result.source + + if (url.startsWith('node:')) { + source = `module.exports = process.getBuiltinModule(${JSON.stringify(url)})\n` + } else if ((format === 'commonjs' || format === 'commonjs-typescript') && source == null && + url.startsWith('file:')) { + source = process.getBuiltinModule('fs').readFileSync(fileURLToPath(url)) + } - try { - if (format === 'commonjs-typescript' && stripTypeScriptTypes !== undefined) { - source = stripTypeScriptTypes(sourceToString(source), { mode: 'strip' }) + if (source == null || (format !== 'commonjs' && format !== 'commonjs-typescript' && + !url.startsWith('node:'))) { + return result } - return { - ...result, - format: 'commonjs', - source: buildCommonJSWrapperSource({ - realUrl: url, - source, - originalSpecifier: specifierData.specifier, - data: specifierData.data, - runtimeSpecifier: iitmPath - }), - shortCircuit: true + + try { + if (format === 'commonjs-typescript') { + const stripTypeScriptTypes = process.getBuiltinModule('module').stripTypeScriptTypes + if (stripTypeScriptTypes !== undefined) { + source = stripTypeScriptTypes(Buffer.isBuffer(source) ? source.toString('utf8') : source, { mode: 'strip' }) + } + } + return { + ...result, + format: 'commonjs', + source: buildCommonJSWrapperSource({ + realUrl: url, + source, + originalSpecifier: specifierData.specifier, + runtimeSpecifier: fileURLToPath(iitmURL) + }), + shortCircuit: true + } + } catch (cause) { + onWrapFailure(url, cause) + return result } - } catch (cause) { - onWrapFailure(url, cause) - return result } } @@ -564,8 +629,10 @@ export function createHook (meta) { return parentGetSource(url, context) } + let originalSpecifier = specifierData let processContext = context - if (specifierData.format !== undefined) { + if (typeof specifierData !== 'string') { + originalSpecifier = specifierData.specifier processContext = { ...context, format: specifierData.format } } @@ -574,7 +641,7 @@ export function createHook (meta) { processModule({ srcUrl: realUrl, context: processContext }), { resolve: cachedResolve, load: parentGetSource } ) - return { source: onWrapSuccess(realUrl, processContext, specifierData, bindings) } + return { source: onWrapSuccess(realUrl, processContext, originalSpecifier, bindings) } } catch (cause) { onWrapFailure(realUrl, cause) // Revert back to the non-iitm URL @@ -602,8 +669,10 @@ export function createHook (meta) { return nextLoad(url, context) } + let originalSpecifier = specifierData let processContext = context - if (specifierData.format !== undefined) { + if (typeof specifierData !== 'string') { + originalSpecifier = specifierData.specifier processContext = { ...context, format: specifierData.format } } @@ -612,7 +681,7 @@ export function createHook (meta) { processModule({ srcUrl: realUrl, context: processContext }), { resolve: cachedResolve, load: nextLoad } ) - return { source: onWrapSuccess(realUrl, processContext, specifierData, bindings) } + return { source: onWrapSuccess(realUrl, processContext, originalSpecifier, bindings) } } catch (cause) { onWrapFailure(realUrl, cause) url = realUrl @@ -680,18 +749,6 @@ export function createHook (meta) { return nextLoad(deleteIitm(url), context) } - const specifierData = specifiers.get(url) - if (specifierData?.commonjs === true) { - let result - try { - result = nextLoad(url, context) - } catch (error) { - specifiers.delete(url) - throw error - } - return wrapCommonJS(url, result, specifierData) - } - if (cjsInIitmChain.has(url) && !disableCjsSourceStripping) { const result = nextLoad(url, context) if (result.format === 'commonjs' && result.source != null) { @@ -706,5 +763,44 @@ export function createHook (meta) { return nextLoad(url, context) } + /** + * @param {string} url + * @param {LoadContext} context + * @param {(url: string, context?: Partial) => LoadResult} nextLoad + * @returns {LoadResult} + */ + let loadSyncCommonJS + if (commonjs === true) { + loadSyncCommonJS = (url, context, nextLoad) => { + if (hasIitm(url)) return loadSync(url, context, nextLoad) + + const specifierData = commonJsSpecifiers?.get(url) + if (specifierData !== undefined) { + let result + try { + result = nextLoad(url, context) + } catch (error) { + commonJsSpecifiers.delete(url) + throw error + } + return wrapCommonJS(url, result, specifierData) + } + + return loadSync(url, context, nextLoad) + } + } + + if (commonjs === true) { + return { + initialize, + load, + resolve, + resolveSync, + resolveSyncCommonJS, + loadSync, + loadSyncCommonJS, + applyOptions + } + } return { initialize, load, resolve, resolveSync, loadSync, applyOptions } } diff --git a/index.d.ts b/index.d.ts index acdfd7c1..cafd7d81 100644 --- a/index.d.ts +++ b/index.d.ts @@ -18,16 +18,17 @@ export type Namespace = { [key: string]: any } * starting from the package name. * @param {baseDir} string The absolute path of the module, if not provided in * `name`. - * @param {data} Data Optional metadata embedded by a loader or bundler. - * @return unknown For ESM, a value assigned to `exports.default` when present. - * For CommonJS, a value that replaces `module.exports`. + * @param {data} Data Optional metadata embedded by a bundler. + * @return any A value that can will be assigned to `exports.default`. This is + * equivalent to doing that assignment in the body of this function. For + * CommonJS modules, the value replaces `module.exports`. */ export type HookFn = ( exported: Namespace, name: string, baseDir: string|void, data?: Data -) => unknown +) => any export type Options = { internals?: boolean @@ -67,15 +68,15 @@ export default Hook * @param {exported} { [string]: any } An object representing the exported * items of a module. * @param {specifier} string The original import or require specifier. - * @param {data} Data Optional metadata embedded by a loader or bundler. - * @param {format} string The intercepted module format. + * @param {data} Data Optional metadata embedded by a bundler. + * @param {format} string The intercepted module format, when available. */ export type HookFunction = ( url: string, exported: Namespace, specifier: string, - data: Data|undefined, - format: 'module'|'commonjs' + data?: Data, + format?: 'module'|'commonjs' ) => unknown /** diff --git a/index.js b/index.js index aa8bab19..4934554c 100644 --- a/index.js +++ b/index.js @@ -13,11 +13,15 @@ if (!isBuiltin) { } const { - addHook, - removeHook, - specifiers + extendedHooks, + importHooks, + specifiers, + toHook, + toHookExtended } = require('./lib/register') +const hookExtensions = new WeakMap() + /** * Checks turbopack specifiers separately (for Next.js 16+). * @@ -38,26 +42,123 @@ function isTurbopackSpecifier (specifier, baseDir) { return baseDir.endsWith(specifierWithoutTurbopackHash) } +function addHook (hook, extendedHook = hook) { + importHooks.push(hook) + toHook.forEach(([name, namespace, specifier]) => hook(name, namespace, specifier)) + extendedHooks.push(extendedHook) + for (const entry of toHookExtended) { + const namespace = entry.module === undefined ? entry.namespace : entry.module.exports + const replacement = extendedHook(entry.name, namespace, entry.specifier, entry.data, entry.format) + if (entry.module !== undefined && replacement !== undefined) entry.module.exports = replacement + } +} + +function removeHook (hook, extendedHook = hook) { + const index = importHooks.indexOf(hook) + if (index > -1) { + importHooks.splice(index, 1) + } + const extendedIndex = extendedHooks.indexOf(extendedHook) + if (extendedIndex > -1) { + extendedHooks.splice(extendedIndex, 1) + } +} + +function callHookFn (hookFn, namespace, name, baseDir) { + const newDefault = hookFn(namespace, name, baseDir) + if (newDefault && newDefault !== namespace) { + // Only ESM modules that actually export `default` can have it reassigned. + // Some hooks return a value unconditionally; avoid crashing when the module + // has no default export (see issue #188). + if ('default' in namespace) { + namespace.default = newDefault + } + } +} + /** - * @param {Function} hookFn - * @param {object} namespace + * @param {(namespace: unknown, name: string, baseDir: string|undefined, data: unknown) => unknown} hookFn + * @param {unknown} namespace * @param {string} name * @param {string|undefined} baseDir * @param {unknown} data * @param {'module'|'commonjs'} format * @returns {unknown} */ -function callHookFn (hookFn, namespace, name, baseDir, data, format) { - const newDefault = hookFn(namespace, name, baseDir, data) - if (format === 'commonjs') return newDefault - if (newDefault && newDefault !== namespace) { - // Only ESM modules that actually export `default` can have it reassigned. - // Some hooks return a value unconditionally; avoid crashing when the module - // has no default export (see issue #188). - if ('default' in namespace) { - namespace.default = newDefault +function callExtendedHookFn (hookFn, namespace, name, baseDir, data, format) { + const replacement = hookFn(namespace, name, baseDir, data) + if (format === 'commonjs') return replacement + if (replacement && replacement !== namespace && 'default' in namespace) { + namespace.default = replacement + } +} + +/** + * @param {(namespace: unknown, name: string, baseDir: string|undefined, data: unknown) => unknown} hookFn + * @param {Array|null} modules + * @param {boolean} internals + * @param {string} name + * @param {unknown} namespace + * @param {string} specifier + * @param {unknown} data + * @param {'module'|'commonjs'} format + * @returns {unknown} + */ +function callExtendedHook (hookFn, modules, internals, name, namespace, specifier, data, format) { + const loadUrl = name + const isNodeUrl = loadUrl.startsWith('node:') + let filePath, baseDir + + if (isNodeUrl) { + const unprefixed = name.slice(5) + if (isBuiltin(unprefixed)) { + name = unprefixed + } + } else if (loadUrl.startsWith('file://')) { + const stackTraceLimit = Error.stackTraceLimit + Error.stackTraceLimit = 0 + try { + filePath = fileURLToPath(name) + name = filePath + } catch {} + Error.stackTraceLimit = stackTraceLimit + + if (filePath) { + const details = moduleDetailsFromPath(filePath) + if (details) { + name = details.name + baseDir = details.basedir + } } } + + let replacement + if (modules) { + for (const matchArg of modules) { + let result + if (filePath && matchArg === filePath) { + result = callExtendedHookFn(hookFn, namespace, filePath, undefined, data, format) + } else if (matchArg === name) { + if (!baseDir) { + result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) + } else if (baseDir.endsWith(specifiers.get(loadUrl)) || isTurbopackSpecifier(specifiers.get(loadUrl), baseDir)) { + result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) + } else if (internals) { + const internalPath = name + path.sep + path.relative(baseDir, filePath) + result = callExtendedHookFn(hookFn, namespace, internalPath, baseDir, data, format) + } + } else if (matchArg === specifier) { + result = callExtendedHookFn(hookFn, namespace, specifier, baseDir, data, format) + } + if (format === 'commonjs' && result !== undefined) { + namespace = result + replacement = result + } + } + return replacement + } + + return callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) } let sendModulesToLoader @@ -145,7 +246,7 @@ function Hook (modules, options, hookFn) { sendModulesToLoader(modules) } - this._iitmHook = (name, namespace, specifier, data, format) => { + this._iitmHook = (name, namespace, specifier) => { const loadUrl = name const isNodeUrl = loadUrl.startsWith('node:') let filePath, baseDir @@ -176,47 +277,43 @@ function Hook (modules, options, hookFn) { } } - let replacement if (modules) { for (const matchArg of modules) { - let result if (filePath && matchArg === filePath) { // abspath match - result = callHookFn(hookFn, namespace, filePath, undefined, data, format) + callHookFn(hookFn, namespace, filePath, undefined) } else if (matchArg === name) { if (!baseDir) { // built-in module (or unexpected non file:// name?) - result = callHookFn(hookFn, namespace, name, baseDir, data, format) + callHookFn(hookFn, namespace, name, baseDir) } else if (baseDir.endsWith(specifiers.get(loadUrl)) || isTurbopackSpecifier(specifiers.get(loadUrl), baseDir)) { // An import of the top-level module (e.g. `import 'ioredis'`). // Note: Slight behaviour difference from RITM. RITM uses // `require.resolve(name)` to see if filename is the module // main file, which will catch `require('ioredis/built/index.js')`. // The check here will not catch `import 'ioredis/built/index.js'`. - result = callHookFn(hookFn, namespace, name, baseDir, data, format) + callHookFn(hookFn, namespace, name, baseDir) } else if (internals) { const internalPath = name + path.sep + path.relative(baseDir, filePath) - result = callHookFn(hookFn, namespace, internalPath, baseDir, data, format) + callHookFn(hookFn, namespace, internalPath, baseDir) } } else if (matchArg === specifier) { - result = callHookFn(hookFn, namespace, specifier, baseDir, data, format) - } - if (result !== undefined) { - namespace = result - replacement = result + callHookFn(hookFn, namespace, specifier, baseDir) } } - return replacement } else { - return callHookFn(hookFn, namespace, name, baseDir, data, format) + callHookFn(hookFn, namespace, name, baseDir) } } - addHook(this._iitmHook) + const extendedHook = callExtendedHook.bind(undefined, hookFn, modules, internals) + hookExtensions.set(this, extendedHook) + addHook(this._iitmHook, extendedHook) } Hook.prototype.unhook = function () { - removeHook(this._iitmHook) + removeHook(this._iitmHook, hookExtensions.get(this)) + hookExtensions.delete(this) } module.exports = Hook diff --git a/lib/bundler-runtime.js b/lib/bundler-runtime.js index ed49fe55..3ea401aa 100644 --- a/lib/bundler-runtime.js +++ b/lib/bundler-runtime.js @@ -1,10 +1,8 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. - 'use strict' -const { ModuleBinder, registerCommonJS } = require('./register.js') +const { ModuleBinder, register, registerCommonJS, registerWithData } = require('./register.js') exports.ModuleBinder = ModuleBinder +exports.register = register exports.registerCommonJS = registerCommonJS +exports.registerWithData = registerWithData diff --git a/lib/get-exports.mjs b/lib/get-exports.mjs index 20952e4e..d396a54e 100644 --- a/lib/get-exports.mjs +++ b/lib/get-exports.mjs @@ -7,7 +7,6 @@ import { builtinModules, createRequire } from 'module' import { fileURLToPath, pathToFileURL } from 'url' import { dirname, join } from 'path' import { LOAD } from './io.mjs' -import { sourceToString } from './source.mjs' const nodeMajor = Number(process.versions.node.split('.')[0]) export const hasModuleExportsCJSDefault = nodeMajor >= 23 @@ -264,7 +263,16 @@ export function * getExports (url, context) { // Loader hooks can return ArrayBuffer / TypedArray sources. Normalize to a // string for parsing. if (source && typeof source !== 'string') { - source = sourceToString(source) + // Avoid copies where possible: + // - Buffer.from(Uint8Array) copies + // - Buffer.from(ArrayBuffer, offset, length) wraps the existing memory + if (Buffer.isBuffer(source)) { + source = source.toString('utf8') + } else if (ArrayBuffer.isView(source)) { + source = Buffer.from(source.buffer, source.byteOffset, source.byteLength).toString('utf8') + } else { + source = Buffer.from(source).toString('utf8') + } } if (!source) { diff --git a/lib/register.js b/lib/register.js index 91fea86c..8a140fcc 100644 --- a/lib/register.js +++ b/lib/register.js @@ -6,6 +6,8 @@ const importHooks = [] // TODO should this be a Set? const binders = new WeakMap() const specifiers = new Map() const toHook = [] +const extendedHooks = [] +const toHookExtended = [] /** * @typedef {object} HookEntry @@ -52,56 +54,37 @@ function defineExport (target, name, descriptor) { const proxyHandler = { defineProperty: defineExport, set: setExport } /** - * @param {(name: string, exports: unknown, specifier: string, data: unknown, - * format: 'module' | 'commonjs') => unknown} hook - * @param {HookEntry} entry - * @returns {void} - */ -function applyHook (hook, entry) { - const exports = entry.module === undefined ? entry.namespace : entry.module.exports - const replacement = hook(entry.name, exports, entry.specifier, entry.data, entry.format) - if (entry.module !== undefined && replacement !== undefined) { - entry.module.exports = replacement - } -} - -/** - * @param {(name: string, exports: unknown, specifier: string, data: unknown, - * format: 'module' | 'commonjs') => unknown} hook - * @returns {void} + * @param {string} name The wrapped module URL. + * @param {ModuleBinder} binder The wrapper's binding state. + * @param {string} specifier The original import specifier. */ -function addHook (hook) { - importHooks.push(hook) - for (const entry of toHook) { - applyHook(hook, entry) +function register (name, binder, specifier) { + const { namespace } = binder + specifiers.set(name, specifier) + binders.set(namespace, binder) + const proxy = new Proxy(namespace, proxyHandler) + for (const hook of importHooks) { + hook(name, proxy, specifier) } -} - -/** - * @param {Function} hook - * @returns {void} - */ -function removeHook (hook) { - const index = importHooks.indexOf(hook) - if (index !== -1) importHooks.splice(index, 1) + toHook.push([name, proxy, specifier]) } /** * @param {string} name The wrapped module URL. * @param {ModuleBinder} binder The wrapper's binding state. * @param {string} specifier The original import specifier. - * @param {unknown} [data] Consumer data associated with the module. + * @param {unknown} data Consumer data associated with the module. + * @returns {void} */ -function register (name, binder, specifier, data) { +function registerWithData (name, binder, specifier, data) { const { namespace } = binder specifiers.set(name, specifier) binders.set(namespace, binder) const proxy = new Proxy(namespace, proxyHandler) - const entry = { name, namespace: proxy, specifier, data, format: 'module' } - for (const hook of importHooks) { - applyHook(hook, entry) + for (const hook of extendedHooks) { + hook(name, proxy, specifier, data, 'module') } - toHook.push(entry) + toHookExtended.push({ name, namespace: proxy, specifier, data, format: 'module' }) } /** @@ -121,10 +104,11 @@ function registerCommonJS (name, module, specifier, data) { format: 'commonjs', module } - for (const hook of importHooks) { - applyHook(hook, entry) + for (const hook of extendedHooks) { + const replacement = hook(name, module.exports, specifier, data, 'commonjs') + if (replacement !== undefined) module.exports = replacement } - toHook.push(entry) + toHookExtended.push(entry) } // Delays (ms) for re-reading exports that were still in their temporal dead zone @@ -286,9 +270,10 @@ class ModuleBinder { exports.register = register exports.registerCommonJS = registerCommonJS +exports.registerWithData = registerWithData exports.ModuleBinder = ModuleBinder -exports.addHook = addHook +exports.extendedHooks = extendedHooks exports.importHooks = importHooks -exports.removeHook = removeHook exports.specifiers = specifiers exports.toHook = toHook +exports.toHookExtended = toHookExtended diff --git a/lib/source.mjs b/lib/source.mjs deleted file mode 100644 index 27100e2d..00000000 --- a/lib/source.mjs +++ /dev/null @@ -1,24 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2.0 License. -// -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021 Datadog, Inc. - -/** - * @param {string | ArrayBuffer | ArrayBufferView} source - * @returns {string} - */ -export function sourceToString (source) { - if (typeof source === 'string') return source - if (Buffer.isBuffer(source)) return source.toString('utf8') - if (ArrayBuffer.isView(source)) { - return Buffer.from(source.buffer, source.byteOffset, source.byteLength).toString('utf8') - } - return Buffer.from(source).toString('utf8') -} - -/** - * @param {string} source - * @returns {string} - */ -export function commentShebang (source) { - return source.startsWith('#!') ? '//' + source.slice(2) : source -} diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index 116d4dee..80b6bc5d 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -9,7 +9,6 @@ import { URL } from 'url' import { getExports } from './get-exports.mjs' import { RESOLVE } from './io.mjs' -import { commentShebang, sourceToString } from './source.mjs' // Depth at which `processModule` starts tracking visited URLs to break an // `export *` cycle. Real re-export chains are only a few levels deep, so this @@ -216,23 +215,27 @@ export function * processModule ({ srcUrl, context, excludeDefault = false, dept } /** - * @param {object} options - * @param {string} options.realUrl The URL of the wrapped module. - * @param {string[] | Map} options.bindings Its exported bindings. - * @param {string} options.originalSpecifier The specifier used to import the module. - * @param {unknown} [options.data] Consumer data associated with the module. - * @param {string} options.runtimeSpecifier The wrapper runtime import. - * @param {(url: string) => string} [options.mapImport] Maps module URLs to bundler-owned imports. + * @typedef {object} WrapperOptions + * @property {string} realUrl The URL of the wrapped module. + * @property {string[] | Map} bindings Its exported bindings. + * @property {string} originalSpecifier The specifier used to import the module. + * @property {string} runtimeSpecifier The wrapper runtime import. + * @property {(url: string) => string} [mapImport] Maps module URLs to bundler-owned imports. + */ + +/** + * @param {WrapperOptions & { data?: unknown }} options + * @param {boolean} withData Whether to use the extended bundler registry. * @returns {string} */ -export function buildWrapperSource ({ +function buildESMWrapperSource ({ realUrl, bindings, originalSpecifier, data, runtimeSpecifier, mapImport -}) { +}, withData) { const moduleSpecifier = mapImport?.(realUrl) ?? realUrl // The wrapped module imports its namespace as `namespace`, which serves // every export but the ones a same-origin `export *` collision forced onto @@ -284,17 +287,19 @@ export function buildWrapperSource ({ ? 'const __binder = new ModuleBinder(namespace)\n' : `let ${declarationNames} function __write (index, value) { -switch (index) { + switch (index) { ${writeCases} } } const __binder = new ModuleBinder(namespace, [${bindingNames}], __write${bindingSources === undefined -? '' -: `, [${bindingSources}]`}) + ? '' + : `, [${bindingSources}]`}) ` const reexports = exportSpecifiers === '' ? '' : `export { ${exportSpecifiers} }\n` + const registerName = withData ? 'registerWithData' : 'register' + const registrationData = withData ? `, ${JSON.stringify(data)}` : '' return ` -import { register, ModuleBinder } from ${JSON.stringify(runtimeSpecifier)} +import { ${registerName}, ModuleBinder } from ${JSON.stringify(runtimeSpecifier)} import * as namespace from ${JSON.stringify(moduleSpecifier)} ${originImports} ${binder} @@ -302,10 +307,39 @@ ${reexports} __binder.flush() -register(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)}) +${registerName}(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}${registrationData}) ` } +/** + * @param {WrapperOptions} options + * @returns {string} + */ +export function buildWrapperSource (options) { + return buildESMWrapperSource(options, false) +} + +/** + * @param {WrapperOptions & { data: unknown }} options + * @returns {string} + */ +export function buildWrapperSourceWithData (options) { + return buildESMWrapperSource(options, true) +} + +/** + * @param {string | ArrayBuffer | ArrayBufferView} source + * @returns {string} + */ +function sourceToString (source) { + if (typeof source === 'string') return source + if (Buffer.isBuffer(source)) return source.toString('utf8') + if (ArrayBuffer.isView(source)) { + return Buffer.from(source.buffer, source.byteOffset, source.byteLength).toString('utf8') + } + return Buffer.from(source).toString('utf8') +} + /** * @param {object} options * @param {string} options.realUrl @@ -322,7 +356,8 @@ export function buildCommonJSWrapperSource ({ data, runtimeSpecifier }) { - source = commentShebang(sourceToString(source)) + source = sourceToString(source) + if (source.startsWith('#!')) source = '//' + source.slice(2) return `(function (exports, require, module, __filename, __dirname) {${source}\n` + '}).call(module.exports, module.exports, require, module, __filename, __dirname)\n' + diff --git a/package.json b/package.json index 493d3468..436a25e2 100644 --- a/package.json +++ b/package.json @@ -43,10 +43,8 @@ "@node-rs/crc32": "^1.10.6", "@react-email/components": "^0.0.19", "@types/node": "^18.0.6", - "@vercel/nft": "^1.10.2", "c8": "^7.14.0", "date-fns": "^3.6.0", - "esbuild": "^0.28.1", "eslint": "^8.57.1", "eslint-config-standard": "^17.1.0", "eslint-plugin-import": "^2.32.0", @@ -58,8 +56,7 @@ "openai": "4.47.2", "ts-node": "^10.9.2", "typescript": "^4.9.5", - "vue": "^3.5.26", - "webpack": "^5.109.2" + "vue": "^3.5.26" }, "dependencies": { "cjs-module-lexer": "^2.2.0", diff --git a/register-hooks.d.ts b/register-hooks.d.ts index e470c789..80c365a9 100644 --- a/register-hooks.d.ts +++ b/register-hooks.d.ts @@ -3,19 +3,11 @@ * `file:` URLs or regular expressions, matched against the module being * resolved. CJS source stripping remains enabled unless explicitly disabled. */ -export type ModuleInclusion = { - data?: Data -} - -export type RegisterHooksOptions = { +export type RegisterHooksOptions = { include?: Array exclude?: Array disableCjsSourceStripping?: boolean commonjs?: boolean - shouldInclude?: ( - url: string, - specifier: string - ) => boolean | ModuleInclusion } /** @@ -43,7 +35,7 @@ export type RegisterHooksOptions = { * * @throws If {@link supportsSyncHooks} is `false` in the running Node.js. */ -export declare function register(options?: RegisterHooksOptions): void +export declare function register(options?: RegisterHooksOptions): void /** * Whether the running Node.js can correctly run the synchronous loader via diff --git a/register-hooks.mjs b/register-hooks.mjs index d33274cd..0dafbfea 100644 --- a/register-hooks.mjs +++ b/register-hooks.mjs @@ -1,4 +1,5 @@ import * as module from 'module' +import { createRequire } from 'module' import { createHook } from './create-hook.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' @@ -39,9 +40,7 @@ let registered = false * @param {Array} [options.include] Only intercept these modules. * @param {Array} [options.exclude] Never intercept these modules. * @param {boolean} [options.disableCjsSourceStripping] Leave hook-provided CJS source unchanged. - * @param {boolean} [options.commonjs] Intercept CommonJS through the synchronous load hook. - * @param {(url: string, specifier: string) => boolean | { data?: unknown }} [options.shouldInclude] - * Custom inclusion predicate. Returning `{ data }` passes that value to the Hook callback. + * @param {boolean} [options.commonjs] Intercept CommonJS modules. * @returns {void} */ export function register (options) { @@ -60,8 +59,17 @@ export function register (options) { } registered = true + const commonjs = options?.commonjs === true + const activeHook = commonjs ? createHook(import.meta, true) : hook if (options) { - hook.applyOptions(options) + activeHook.applyOptions(options) + } + + if (commonjs) { + const require = createRequire(import.meta.url) + require('./lib/register.js') + module.registerHooks({ resolve: activeHook.resolveSyncCommonJS, load: activeHook.loadSyncCommonJS }) + return } module.registerHooks({ resolve: hook.resolveSync, load: hook.loadSync }) diff --git a/test/fixtures/inspectable-create-hook.mjs b/test/fixtures/inspectable-create-hook.mjs index d9dba82b..eead5e3c 100644 --- a/test/fixtures/inspectable-create-hook.mjs +++ b/test/fixtures/inspectable-create-hook.mjs @@ -6,15 +6,14 @@ let source = readFileSync(createHookUrl, 'utf8') /** * @param {string} _match - * @param {string} prefix * @param {string} relative */ -function resolveImport (_match, prefix, relative) { +function resolveImport (_match, relative) { const absolute = new URL('../../' + relative.slice(2), import.meta.url).href - return prefix + JSON.stringify(absolute) + return `from ${JSON.stringify(absolute)}` } -source = source.replace(/(from |import )'(\.\/[^']+)'/g, resolveImport) +source = source.replace(/from '(\.\/[^']+)'/g, resolveImport) source = source.replace( 'return { initialize, load, resolve, resolveSync, loadSync, applyOptions }', 'return { initialize, load, resolve, resolveSync, loadSync, applyOptions, specifiers }' diff --git a/test/fixtures/sync-commonjs-cycle-a.cjs b/test/fixtures/sync-commonjs-cycle-a.cjs deleted file mode 100644 index c28015e3..00000000 --- a/test/fixtures/sync-commonjs-cycle-a.cjs +++ /dev/null @@ -1,6 +0,0 @@ -exports.name = 'a' - -const b = require('./sync-commonjs-cycle-b.cjs') - -exports.fromB = b.name -exports.seenByB = b.fromA diff --git a/test/fixtures/sync-commonjs-cycle-b.cjs b/test/fixtures/sync-commonjs-cycle-b.cjs deleted file mode 100644 index 8d7d8ca7..00000000 --- a/test/fixtures/sync-commonjs-cycle-b.cjs +++ /dev/null @@ -1,5 +0,0 @@ -exports.name = 'b' - -const a = require('./sync-commonjs-cycle-a.cjs') - -exports.fromA = a.name diff --git a/test/fixtures/sync-commonjs-semantics.cjs b/test/fixtures/sync-commonjs-semantics.cjs deleted file mode 100644 index 71759698..00000000 --- a/test/fixtures/sync-commonjs-semantics.cjs +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env node -'use strict' - -if (this !== exports) { - throw new Error('top-level this must be exports') -} - -if (arguments.length !== 5 || arguments[0] !== exports || arguments[2] !== module) { - throw new Error('CommonJS wrapper arguments changed') -} - -module.exports = { - argumentExports: arguments[0], - topLevelThis: this, - value: 42 -} - -/* eslint-disable n/no-exports-assign, no-global-assign */ -exports = undefined -require = undefined -module = undefined -__filename = undefined -__dirname = undefined -/* eslint-enable n/no-exports-assign, no-global-assign */ - -return - -// eslint-disable-next-line no-unreachable -module.exports.unreachable = true diff --git a/test/fixtures/sync-commonjs-typescript.cts b/test/fixtures/sync-commonjs-typescript.cts deleted file mode 100644 index a9fbaa1d..00000000 --- a/test/fixtures/sync-commonjs-typescript.cts +++ /dev/null @@ -1,3 +0,0 @@ -const value: number = 42 - -module.exports = { value } diff --git a/test/fixtures/type-module/module.js b/test/fixtures/type-module/module.js deleted file mode 100644 index c16d7056..00000000 --- a/test/fixtures/type-module/module.js +++ /dev/null @@ -1 +0,0 @@ -export const value = 42 diff --git a/test/fixtures/type-module/package.json b/test/fixtures/type-module/package.json deleted file mode 100644 index 3dbc1ca5..00000000 --- a/test/fixtures/type-module/package.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "type": "module" -} diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 103e7cc7..9dca9e2a 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -1,5 +1,6 @@ import { strictEqual, deepStrictEqual, match, doesNotMatch, rejects } from 'assert' import { readFile, mkdtemp, writeFile, rm } from 'fs/promises' +import { createRequire } from 'module' import { tmpdir } from 'os' import { join } from 'path' import { fileURLToPath, pathToFileURL } from 'url' @@ -7,6 +8,8 @@ import { fileURLToPath, pathToFileURL } from 'url' import Hook from '../../index.js' import { createWrapperModule } from '../../bundler.mjs' +const require = createRequire(import.meta.url) +const { registerWithData } = require('../../lib/bundler-runtime.js') const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href const source = await readFile(new URL(moduleUrl), 'utf8') @@ -22,32 +25,38 @@ const wrapper = await createWrapperModule({ url: moduleUrl, format: 'module', source, - specifier: './something.mjs' + specifier: './something.mjs', + data: { version: '1.0.0' } }, resolve: unexpectedIo, load: unexpectedIo }) strictEqual(wrapper.sideEffects, true) -strictEqual(wrapper.format, 'module') deepStrictEqual(wrapper.watchFiles, [moduleUrl]) strictEqual(wrapper.imports.length, 2) strictEqual(wrapper.imports[0].specifier, './__iitm_runtime__.js') strictEqual(wrapper.imports[0].kind, 'runtime') -strictEqual(wrapper.imports[0].format, 'commonjs') +strictEqual(wrapper.imports[0].external, false) strictEqual(wrapper.imports[1].specifier, './__iitm_module_0__.js') strictEqual(wrapper.imports[1].kind, 'module') -strictEqual(wrapper.imports[1].url, moduleUrl) +strictEqual(wrapper.imports[1].external, false) match(wrapper.code, /from "\.\/__iitm_runtime__\.js"/) match(wrapper.code, /from "\.\/__iitm_module_0__\.js"/) -match(wrapper.code, /__binder\.register\(\)/) +match(wrapper.code, /\nregisterWithData\(/) +match(wrapper.code, /\{"version":"1\.0\.0"\}\)/) doesNotMatch(wrapper.code, /from "file:/) /** * @param {object} exported + * @param {string} name + * @param {string|undefined} baseDir + * @param {object} data */ -function hookFoo (exported) { +function hookFoo (exported, name, baseDir, data) { + deepStrictEqual(data, { version: '1.0.0' }) exported.foo = 43 + return () => 44 } const hook = new Hook(['./something.mjs'], hookFoo) @@ -61,6 +70,7 @@ try { await writeFile(new URL(wrapperUrl), executableCode) const wrappedNamespace = await import(wrapperUrl) strictEqual(wrappedNamespace.foo, 43) + strictEqual(wrappedNamespace.default(), 44) } finally { hook.unhook() await rm(temporaryDirectory, { recursive: true, force: true }) @@ -78,13 +88,15 @@ const rebuilt = await createWrapperModule({ }) match(rebuilt.code, /export \{ \$rebuilt as "rebuilt" \}/) +match(rebuilt.code, /\nregister\(/) +doesNotMatch(rebuilt.code, /registerWithData/) doesNotMatch(rebuilt.code, /\$foo/) /** - * @param {{ url: string }} target + * @param {string} url */ -function loadBuiltin (target) { - strictEqual(target.url, 'node:dns/promises') +function loadBuiltin (url) { + strictEqual(url, 'node:dns/promises') return { format: 'builtin' } } @@ -98,7 +110,7 @@ const builtinWrapper = await createWrapperModule({ load: loadBuiltin }) -strictEqual(builtinWrapper.imports[1].format, 'builtin') +strictEqual(builtinWrapper.imports[1].external, true) strictEqual(builtinWrapper.imports[1].target.url, 'node:dns/promises') doesNotMatch(builtinWrapper.code, /from "node:dns\/promises"/) @@ -107,62 +119,93 @@ const commonJsWrapper = await createWrapperModule({ module: { url: commonJsUrl, format: 'commonjs', - source: await readFile(new URL(commonJsUrl), 'utf8'), + source: '#!/usr/bin/env node\nmodule.exports = { value: 42 }\nreturn\nmodule.exports.unreachable = true', specifier: './something.js', - target: { namespace: 'file', path: fileURLToPath(commonJsUrl) }, - data: { package: 'fixture', version: '1.0.0' } + data: { version: '1.0.0' } }, resolve: unexpectedIo, load: unexpectedIo }) -strictEqual(commonJsWrapper.format, 'commonjs') strictEqual(commonJsWrapper.imports.length, 1) strictEqual(commonJsWrapper.imports[0].kind, 'runtime') match(commonJsWrapper.code, /registerCommonJS/) -match(commonJsWrapper.code, /"package":"fixture","version":"1\.0\.0"/) doesNotMatch(commonJsWrapper.code, /^(?:import|export) /m) -const loadedCommonJsTarget = { namespace: 'file', path: '/virtual/loaded.cjs' } const loadedCommonJsWrapper = await createWrapperModule({ module: { - url: 'file:///virtual/loaded.cjs', + url: commonJsUrl, format: 'commonjs', - specifier: './loaded.cjs', - target: loadedCommonJsTarget + specifier: './something.js' }, resolve: unexpectedIo, - load (target) { - strictEqual(target, loadedCommonJsTarget) - return { source: 'module.exports = 42' } - } + load: async () => ({ source: Buffer.from('module.exports = 42') }) }) -strictEqual(loadedCommonJsWrapper.format, 'commonjs') match(loadedCommonJsWrapper.code, /module\.exports = 42/) -await rejects(createWrapperModule({ +const typedArrayCommonJsWrapper = await createWrapperModule({ module: { - url: 'file:///virtual/missing-source.cjs', + url: commonJsUrl, format: 'commonjs', - specifier: './missing-source.cjs' + source: new TextEncoder().encode('module.exports = 43'), + specifier: './something.js' }, resolve: unexpectedIo, - load () { - return {} - } -}), /returned no source/) + load: unexpectedIo +}) -await rejects(createWrapperModule({ +match(typedArrayCommonJsWrapper.code, /module\.exports = 43/) + +const arrayBufferCommonJsWrapper = await createWrapperModule({ module: { - url: 'file:///virtual/data.json', - format: 'json', - source: '{}', - specifier: './data.json' + url: commonJsUrl, + format: 'commonjs', + source: new TextEncoder().encode('module.exports = 44').buffer, + specifier: './something.js' }, resolve: unexpectedIo, load: unexpectedIo -}), /Unsupported module format 'json'/) +}) + +match(arrayBufferCommonJsWrapper.code, /module\.exports = 44/) + +await rejects(createWrapperModule({ + module: { + url: commonJsUrl, + format: 'commonjs', + specifier: './something.js' + }, + resolve: unexpectedIo, + load: async () => ({}) +}), { + name: 'TypeError', + message: `The bundler load adapter returned no source for '${commonJsUrl}'` +}) + +const commonJsHook = new Hook(['./something.js'], (exports, name, baseDir, data) => { + deepStrictEqual(data, { version: '1.0.0' }) + return { ...exports, hooked: true } +}) +let unfilteredCalls = 0 +const unfilteredHook = new Hook((exports, name, baseDir, data) => { + if (data?.version === '1.0.0') unfilteredCalls++ +}) +let commonJsCode = commonJsWrapper.code +for (const { specifier, target } of commonJsWrapper.imports) { + commonJsCode = commonJsCode.replaceAll(JSON.stringify(specifier), JSON.stringify(fileURLToPath(target.url))) +} +const commonJsDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-commonjs-')) +try { + const commonJsFilename = join(commonJsDirectory, 'wrapper.cjs') + await writeFile(commonJsFilename, commonJsCode) + deepStrictEqual(require(commonJsFilename), { value: 42, hooked: true }) + strictEqual(unfilteredCalls, 2) +} finally { + unfilteredHook.unhook() + commonJsHook.unhook() + await rm(commonJsDirectory, { recursive: true, force: true }) +} const packageUrl = new URL('../../package.json', import.meta.url).href const sourceWatchUrl = new URL('../fixtures/', import.meta.url).href @@ -179,58 +222,61 @@ function resolveModule (specifier, context) { } } +const hookedPackageUrl = new URL('../fixtures/node_modules/some-external-module/index.mjs', import.meta.url).href +let packageBaseDirectory +const packageHook = new Hook(['some-external-module'], (exports, name, baseDir, data) => { + packageBaseDirectory = baseDir + deepStrictEqual(data, { version: '2.0.0' }) +}) +registerWithData(hookedPackageUrl, {}, {}, {}, 'some-external-module', { version: '2.0.0' }) +strictEqual(packageBaseDirectory, fileURLToPath(new URL('.', hookedPackageUrl)).slice(0, -1)) +packageHook.unhook() + +const packageInternalUrl = new URL('../fixtures/node_modules/some-external-module/sub.mjs', import.meta.url).href +let packageInternalName +const packageInternalHook = new Hook(['some-external-module'], { internals: true }, (exports, name) => { + packageInternalName = name +}) +registerWithData(packageInternalUrl, {}, {}, {}, 'some-external-module/sub', undefined) +strictEqual(packageInternalName, join('some-external-module', 'sub.mjs')) +packageInternalHook.unhook() + +let invalidFileUrlName +const invalidFileUrlHook = new Hook((exports, name) => { + invalidFileUrlName = name +}) +invalidFileUrlName = undefined +registerWithData('file://%', {}, {}, {}, 'invalid', undefined) +strictEqual(invalidFileUrlName, 'file://%') +invalidFileUrlHook.unhook() + /** - * @param {{ url: string }} target + * @param {string} url * @param {{ format: string }} context */ -async function loadModule (target, context) { +async function loadModule (url, context) { return { - source: await readFile(new URL(target.url), 'utf8'), + source: await readFile(new URL(url), 'utf8'), format: context.format, watchFiles: [sourceWatchUrl] } } const reexportUrl = new URL('../fixtures/reexport-same-source.mjs', import.meta.url).href -const reexportTarget = { - namespace: 'file', - path: fileURLToPath(reexportUrl), - pluginData: { loader: 'source' } -} -const leafTargets = new Map() const reexportWrapper = await createWrapperModule({ module: { url: reexportUrl, format: 'module', - specifier: './reexport-same-source.mjs', - target: reexportTarget - }, - resolve (specifier, context) { - const result = resolveModule(specifier, context) - const target = { - namespace: 'file', - path: fileURLToPath(result.url), - pluginData: { resolvedBy: 'fixture' } - } - leafTargets.set(result.url, target) - return { ...result, target } + specifier: './reexport-same-source.mjs' }, - async load (target, context) { - if (target === reexportTarget) { - return loadModule({ url: reexportUrl }, context) - } - for (const [url, leafTarget] of leafTargets) { - if (target === leafTarget) return loadModule({ url }, context) - } - throw new Error('load received a target that was not returned by the adapter') - } + resolve: resolveModule, + load: loadModule }) -strictEqual(reexportWrapper.format, 'module') strictEqual(reexportWrapper.imports[0].kind, 'runtime') -strictEqual(reexportWrapper.imports[1].target, reexportTarget) +strictEqual(reexportWrapper.imports[1].target.url, reexportUrl) strictEqual(reexportWrapper.imports[2].specifier, './__iitm_module_1__.js') -strictEqual(reexportWrapper.imports[2].target, leafTargets.get(reexportWrapper.imports[2].url)) +strictEqual(reexportWrapper.imports[2].target.format, 'module') strictEqual(reexportWrapper.watchFiles.includes(reexportUrl), true) strictEqual(reexportWrapper.watchFiles.includes(packageUrl), true) strictEqual(reexportWrapper.watchFiles.includes(sourceWatchUrl), true) diff --git a/test/other/v18-bundlers.mjs b/test/other/v18-bundlers.mjs deleted file mode 100644 index 6e27aa6a..00000000 --- a/test/other/v18-bundlers.mjs +++ /dev/null @@ -1,208 +0,0 @@ -import { deepStrictEqual, strictEqual } from 'node:assert/strict' -import { spawnSync } from 'node:child_process' -import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { fileURLToPath, pathToFileURL } from 'node:url' - -import * as esbuild from 'esbuild' -import webpack from 'webpack' - -import { createWrapperModule } from '../../bundler.mjs' - -const packageRoot = fileURLToPath(new URL('../../', import.meta.url)) -const indexPath = join(packageRoot, 'index.js') -const temporaryDirectory = await realpath(await mkdtemp(join(tmpdir(), 'iitm-bundlers-'))) - -try { - const originalPath = join(temporaryDirectory, 'original.mjs') - const originalCommonJsPath = join(temporaryDirectory, 'original.cjs') - const dependencyPath = join(temporaryDirectory, 'dependency.cjs') - await writeFile(originalPath, 'export const value = 42\n') - await writeFile(dependencyPath, 'module.exports = 42\n') - - const originalTarget = { - namespace: 'iitm-original', - path: originalPath, - pluginData: { owner: 'adapter' } - } - const wrappers = new Map([ - ['esm', await createWrapperModule({ - module: { - url: pathToFileURL(originalPath).href, - format: 'module', - source: await readFile(originalPath), - specifier: 'iitm-virtual-esm', - target: originalTarget, - data: { increment: 1 } - }, - resolve: unexpectedIo, - load: unexpectedIo - })], - ['commonjs', await createWrapperModule({ - module: { - url: pathToFileURL(originalCommonJsPath).href, - format: 'commonjs', - source: "module.exports = { value: require('./dependency.cjs') }\n", - specifier: 'iitm-virtual-commonjs', - target: { namespace: 'file', path: originalCommonJsPath }, - data: { increment: 2 } - }, - resolve: unexpectedIo, - load: unexpectedIo - })] - ]) - - const entrySource = ` -const Hook = require(${JSON.stringify(indexPath)}) - -new Hook((exports, name, baseDir, data) => { - if (data === undefined) return - exports.value += data.increment -}) - -Promise.all([ - import('iitm-virtual-esm'), - Promise.resolve(require('iitm-virtual-commonjs')) -]).then(([esm, commonjs]) => { - console.log(JSON.stringify({ esm: esm.value, commonjs: commonjs.value })) -}) -` - - await testEsbuild(entrySource, wrappers, originalTarget) - await testWebpack(entrySource, wrappers) -} finally { - await rm(temporaryDirectory, { recursive: true, force: true }) -} - -/** - * @returns {never} - */ -function unexpectedIo () { - throw new Error('Unexpected adapter I/O') -} - -/** - * @param {string} entrySource - * @param {Map>>} wrappers - * @param {{ namespace: string, path: string, pluginData: object }} originalTarget - */ -async function testEsbuild (entrySource, wrappers, originalTarget) { - const outfile = join(temporaryDirectory, 'esbuild.cjs') - await esbuild.build({ - bundle: true, - format: 'cjs', - platform: 'node', - outfile, - stdin: { - contents: entrySource, - loader: 'js', - resolveDir: temporaryDirectory - }, - plugins: [{ - name: 'iitm-test-adapter', - setup (build) { - build.onResolve({ filter: /^iitm-virtual-/ }, args => ({ - path: args.path === 'iitm-virtual-esm' ? 'esm' : 'commonjs', - namespace: 'iitm-wrapper' - })) - build.onResolve({ filter: /^\.\/__iitm_/, namespace: 'iitm-wrapper' }, args => { - const wrapper = wrappers.get(args.importer) - const entry = wrapper.imports.find(entry => entry.specifier === args.path) - if (entry.kind === 'runtime') return { path: fileURLToPath(entry.url) } - return entry.target - }) - build.onLoad({ filter: /.*/, namespace: 'iitm-wrapper' }, args => ({ - contents: wrappers.get(args.path).code, - loader: 'js', - resolveDir: temporaryDirectory - })) - build.onLoad({ filter: /.*/, namespace: originalTarget.namespace }, async args => { - strictEqual(args.path, originalTarget.path) - strictEqual(args.pluginData, originalTarget.pluginData) - return { contents: await readFile(args.path), loader: 'js' } - }) - } - }] - }) - - deepStrictEqual(runBundle(outfile), { esm: 43, commonjs: 44 }) -} - -/** - * @param {string} entrySource - * @param {Map>>} wrappers - */ -async function testWebpack (entrySource, wrappers) { - const webpackDirectory = join(temporaryDirectory, 'webpack') - const outputDirectory = join(webpackDirectory, 'dist') - const wrappersByContext = new Map() - await mkdir(outputDirectory, { recursive: true }) - - const entryPath = join(webpackDirectory, 'entry.cjs') - await writeFile(entryPath, entrySource) - for (const [name, wrapper] of wrappers) { - const directory = name === 'commonjs' ? temporaryDirectory : join(webpackDirectory, name) - const filename = join(directory, wrapper.format === 'module' ? 'wrapper.mjs' : 'wrapper.cjs') - await mkdir(directory, { recursive: true }) - await writeFile(filename, wrapper.code) - wrappersByContext.set(directory, { filename, wrapper }) - } - - const replacement = new webpack.NormalModuleReplacementPlugin( - /^(?:iitm-virtual-|\.\/__iitm_)/, - resource => { - if (resource.request === 'iitm-virtual-esm') { - resource.request = wrappersByContext.get(join(webpackDirectory, 'esm')).filename - return - } - if (resource.request === 'iitm-virtual-commonjs') { - resource.request = wrappersByContext.get(temporaryDirectory).filename - return - } - - const { wrapper } = wrappersByContext.get(resource.context) - const entry = wrapper.imports.find(entry => entry.specifier === resource.request) - resource.request = entry.kind === 'runtime' ? fileURLToPath(entry.url) : entry.target.path - } - ) - - const stats = await runWebpack({ - entry: entryPath, - mode: 'development', - target: 'node', - devtool: false, - output: { - path: outputDirectory, - filename: 'bundle.cjs', - chunkFilename: '[name].cjs' - }, - plugins: [replacement] - }) - const errors = stats.toJson({ all: false, errors: true }).errors - deepStrictEqual(errors, []) - deepStrictEqual(runBundle(join(outputDirectory, 'bundle.cjs')), { esm: 43, commonjs: 44 }) -} - -/** - * @param {import('webpack').Configuration} configuration - * @returns {Promise} - */ -function runWebpack (configuration) { - return new Promise((resolve, reject) => { - webpack(configuration, (error, stats) => { - if (error) return reject(error) - resolve(stats) - }) - }) -} - -/** - * @param {string} filename - * @returns {{ esm: number, commonjs: number }} - */ -function runBundle (filename) { - const result = spawnSync(process.execPath, [filename], { encoding: 'utf8' }) - strictEqual(result.status, 0, result.stderr) - return JSON.parse(result.stdout) -} diff --git a/test/other/v20-nft-runtime.mjs b/test/other/v20-nft-runtime.mjs deleted file mode 100644 index 3ad4469e..00000000 --- a/test/other/v20-nft-runtime.mjs +++ /dev/null @@ -1,12 +0,0 @@ -import { ok, strictEqual } from 'node:assert/strict' -import { fileURLToPath } from 'node:url' - -import { nodeFileTrace } from '@vercel/nft' - -const packageRoot = fileURLToPath(new URL('../../', import.meta.url)) -const { fileList, warnings } = await nodeFileTrace(['register-hooks.mjs'], { base: packageRoot }) - -strictEqual(warnings.size, 0) -ok(fileList.has('register-hooks.mjs')) -ok(fileList.has('create-hook.mjs')) -ok(fileList.has('lib/register.js'), 'the generated wrapper runtime must remain reachable to file tracers') diff --git a/test/register/v18.19-loader-url-escaping.mjs b/test/register/v18.19-loader-url-escaping.mjs index 5b58a4dd..79c52865 100644 --- a/test/register/v18.19-loader-url-escaping.mjs +++ b/test/register/v18.19-loader-url-escaping.mjs @@ -24,7 +24,6 @@ try { 'lib/get-exports.mjs', 'lib/io.mjs', 'lib/register.js', - 'lib/source.mjs', 'lib/wrapper.mjs' ] const setupPromises = [] diff --git a/test/register/v22.15-sync-register-hooks-commonjs.mjs b/test/register/v22.15-sync-register-hooks-commonjs.mjs index 3b94e5b9..84ab1ae5 100644 --- a/test/register/v22.15-sync-register-hooks-commonjs.mjs +++ b/test/register/v22.15-sync-register-hooks-commonjs.mjs @@ -1,11 +1,12 @@ -import { deepStrictEqual, ok, strictEqual } from 'node:assert/strict' -import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises' -import * as nodeModule from 'node:module' +import { deepStrictEqual, match, strictEqual, throws } from 'node:assert/strict' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' import { tmpdir } from 'node:os' -import { basename, join } from 'node:path' +import { join } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import Hook from '../../index.js' +import { createHook } from '../../create-hook.mjs' import { register, supportsSyncHooks } from '../../register-hooks.mjs' if (!supportsSyncHooks()) { @@ -13,128 +14,193 @@ if (!supportsSyncHooks()) { process.exit(0) } -const commonJsUrl = new URL('../fixtures/sync-commonjs-semantics.cjs', import.meta.url) -const cycleAUrl = new URL('../fixtures/sync-commonjs-cycle-a.cjs', import.meta.url) -const cycleBUrl = new URL('../fixtures/sync-commonjs-cycle-b.cjs', import.meta.url) -const commonJsTypeScriptUrl = new URL('../fixtures/sync-commonjs-typescript.cts', import.meta.url) -const importedCommonJsUrl = new URL('../fixtures/something.js', import.meta.url) -const requiredEsmUrl = new URL('../fixtures/something.mjs', import.meta.url) -const requiredPackageEsmUrl = new URL('../fixtures/type-module/module.js', import.meta.url) -const includedUrls = new Set([ - commonJsUrl.href, - cycleAUrl.href, - cycleBUrl.href, - commonJsTypeScriptUrl.href, - importedCommonJsUrl.href, - requiredEsmUrl.href, - requiredPackageEsmUrl.href, - 'node:fs' -]) +const hookMeta = { url: new URL('../../register-hooks.mjs', import.meta.url).href } +const lowLevelHook = createHook(hookMeta, true) +const requireContext = { conditions: ['require'], parentURL: import.meta.url } + +/** + * @param {string} url + * @param {string} [format] + * @param {object} [context] + * @returns {object} + */ +function resolveAsRequire (url, format, context = requireContext) { + return lowLevelHook.resolveSyncCommonJS(url, context, () => ({ url, format })) +} + +deepStrictEqual(resolveAsRequire('file:///entry', undefined, { conditions: ['require'] }), { + url: 'file:///entry', + format: 'commonjs' +}) +deepStrictEqual(resolveAsRequire('file:///entry.cjs', undefined, { conditions: ['require'] }), { + url: 'file:///entry.cjs', + format: undefined +}) + +const ignored = { url: 'file:///ignored.json', format: 'json' } +strictEqual(lowLevelHook.resolveSyncCommonJS('ignored', requireContext, () => ignored), ignored) +strictEqual(resolveAsRequire('file:///native.node').url, 'file:///native.node') +strictEqual(resolveAsRequire('file:///data.json', undefined, { + ...requireContext, + importAttributes: { type: 'json' } +}).url, 'file:///data.json') +strictEqual(resolveAsRequire(import.meta.url, 'module', { + conditions: ['require'], + parentURL: import.meta.url +}).shortCircuit, true) +strictEqual(resolveAsRequire('https://example.com/module').url, 'https://example.com/module') +strictEqual(resolveAsRequire('file:///unknown.wasm').url, 'file:///unknown.wasm') + +const filteredHook = createHook(hookMeta, true) +filteredHook.applyOptions({ include: ['included'] }) +const filtered = { url: 'file:///filtered.cjs', format: 'commonjs' } +strictEqual(filteredHook.resolveSyncCommonJS('filtered', requireContext, () => filtered), filtered) +strictEqual(lowLevelHook.resolveSyncCommonJS('nested', { + conditions: ['require'], + parentURL: hookMeta.url +}, () => filtered), filtered) + +const formatDirectory = mkdtempSync(join(tmpdir(), 'iitm-formats-')) +try { + writeFileSync(join(formatDirectory, 'package.json'), '{"type":"module"}') + const moduleUrl = pathToFileURL(join(formatDirectory, 'module.js')).href + const moduleTypeScriptUrl = pathToFileURL(join(formatDirectory, 'module.ts')).href + strictEqual(resolveAsRequire(moduleUrl).format, 'module') + strictEqual(resolveAsRequire(moduleTypeScriptUrl).format, 'module-typescript') + strictEqual(resolveAsRequire(pathToFileURL(join(formatDirectory, 'module.mts')).href).format, 'module-typescript') + + const invalidDirectory = join(formatDirectory, 'invalid') + const invalidPackageUrl = pathToFileURL(join(invalidDirectory, 'module.js')).href + const invalidPackageDirectory = fileURLToPath(new URL('.', invalidPackageUrl)) + process.getBuiltinModule('fs').mkdirSync(invalidPackageDirectory) + writeFileSync(join(invalidPackageDirectory, 'package.json'), '{') + strictEqual(resolveAsRequire(invalidPackageUrl).url, invalidPackageUrl) +} finally { + rmSync(formatDirectory, { recursive: true, force: true }) +} + +strictEqual(resolveAsRequire('file:///iitm-default/module.ts').url, 'file:///iitm-default/module.ts') +strictEqual(resolveAsRequire('file:///iitm-default/module.js').url, 'file:///iitm-default/module.js') + +const fallbackDirectory = mkdtempSync(join(tmpdir(), 'iitm-commonjs-source-')) +const fallbackFilename = join(fallbackDirectory, 'module.cjs') +const fallbackUrl = pathToFileURL(fallbackFilename).href +try { + writeFileSync(fallbackFilename, 'module.exports = 42') + resolveAsRequire(fallbackUrl) + const fallback = lowLevelHook.loadSyncCommonJS(fallbackUrl, {}, () => ({ + format: 'commonjs', + source: undefined + })) + match(fallback.source, /module\.exports = 42/) + + const typeScriptUrl = pathToFileURL(join(fallbackDirectory, 'module.cts')).href + resolveAsRequire(typeScriptUrl, 'commonjs-typescript') + const typeScript = lowLevelHook.loadSyncCommonJS(typeScriptUrl, {}, () => ({ + format: 'commonjs-typescript', + source: Buffer.from('const value: number = 43; module.exports = value') + })) + match(typeScript.source, /module\.exports = value/) + + const skippedUrl = pathToFileURL(join(fallbackDirectory, 'skipped.cjs')).href + resolveAsRequire(skippedUrl, 'builtin') + const skipped = { format: 'builtin', source: 'module.exports = 44' } + strictEqual(lowLevelHook.loadSyncCommonJS(skippedUrl, {}, () => skipped), skipped) + + const chainHook = createHook(hookMeta, true) + const chainResolution = chainHook.resolveSyncCommonJS(fallbackUrl, { + conditions: ['import'], + parentURL: import.meta.url + }, () => ({ url: fallbackUrl, format: 'commonjs' })) + const chainLoad = chainHook.loadSyncCommonJS(chainResolution.url, { format: 'commonjs' }, () => ({ + format: 'commonjs', + source: 'module.exports = 45' + })) + match(chainLoad.source, /\nregister\(/) + const child = { url: pathToFileURL(join(fallbackDirectory, 'child.cjs')).href, format: 'commonjs' } + strictEqual(chainHook.resolveSyncCommonJS('child', { + conditions: ['require'], + parentURL: fallbackUrl + }, () => child), child) + + const invalidSourceUrl = pathToFileURL(join(fallbackDirectory, 'invalid-source.cjs')).href + resolveAsRequire(invalidSourceUrl, 'commonjs') + const invalidSource = { format: 'commonjs', source: {} } + const emitWarning = process.emitWarning + let warning + process.emitWarning = value => { warning = value } + try { + strictEqual(lowLevelHook.loadSyncCommonJS(invalidSourceUrl, {}, () => invalidSource), invalidSource) + } finally { + process.emitWarning = emitWarning + } + strictEqual(warning.cause instanceof TypeError, true) + + const failedUrl = pathToFileURL(join(fallbackDirectory, 'failed.cjs')).href + resolveAsRequire(failedUrl, 'commonjs') + throws(() => lowLevelHook.loadSyncCommonJS(failedUrl, {}, () => { + throw new Error('load failed') + }), /load failed/) + deepStrictEqual(lowLevelHook.loadSyncCommonJS(failedUrl, {}, () => ({ source: undefined })), { + source: undefined + }) +} finally { + rmSync(fallbackDirectory, { recursive: true, force: true }) +} +const commonJsUrl = new URL('../fixtures/something.js', import.meta.url) +const commonJsTypeScriptUrl = new URL('../fixtures/typescript-cjs-hook.cts', import.meta.url) +const esmUrl = new URL('../fixtures/something.mjs', import.meta.url) register({ commonjs: true, - shouldInclude (url) { - if (!includedUrls.has(url)) return false - return { data: { filename: url.startsWith('file:') ? basename(fileURLToPath(url)) : url } } - } + include: [commonJsUrl.href, commonJsTypeScriptUrl.href, esmUrl.href, 'fs', 'node:test'] }) -const require = nodeModule.createRequire(import.meta.url) +const require = createRequire(import.meta.url) const commonJsFilename = fileURLToPath(commonJsUrl) -let commonJsHookCount = 0 - -const commonJsHook = new Hook([commonJsFilename], (exports, name, baseDir, data) => { - commonJsHookCount++ - strictEqual(name, commonJsFilename) - strictEqual(baseDir, undefined) - deepStrictEqual(data, { filename: 'sync-commonjs-semantics.cjs' }) - return { ...exports, hooked: true } -}) +const commonJsHook = new Hook([commonJsFilename], exports => ({ + value: exports(), + foo: exports.foo +})) const first = require(commonJsFilename) -strictEqual(first.value, 42) -strictEqual(first.hooked, true) -strictEqual(first.unreachable, undefined) -strictEqual(first.topLevelThis, first.argumentExports) +deepStrictEqual(first, { value: 42, foo: 42 }) strictEqual(require(commonJsFilename), first) -strictEqual(commonJsHookCount, 1) const lateHook = new Hook([commonJsFilename], exports => ({ ...exports, late: true })) -const late = require(commonJsFilename) -strictEqual(late.late, true) -strictEqual(late.value, 42) +strictEqual(require(commonJsFilename).late, true) lateHook.unhook() commonJsHook.unhook() -const cycleCounts = new Map() -const cycleHook = new Hook([ - fileURLToPath(cycleAUrl), - fileURLToPath(cycleBUrl) -], (exports, name) => { - cycleCounts.set(name, (cycleCounts.get(name) ?? 0) + 1) - return exports +const commonJsTypeScriptFilename = fileURLToPath(commonJsTypeScriptUrl) +const commonJsTypeScriptHook = new Hook([commonJsTypeScriptFilename], exports => { + exports.epsilon++ }) -const cycle = require(fileURLToPath(cycleAUrl)) -deepStrictEqual(cycle, { name: 'a', fromB: 'b', seenByB: 'a' }) -strictEqual(cycleCounts.get(fileURLToPath(cycleAUrl)), 1) -strictEqual(cycleCounts.get(fileURLToPath(cycleBUrl)), 1) -cycleHook.unhook() - -const commonJsTypeScriptHook = new Hook([fileURLToPath(commonJsTypeScriptUrl)], exports => { - exports.value++ -}) -const commonJsTypeScript = require(fileURLToPath(commonJsTypeScriptUrl)) -strictEqual(commonJsTypeScript.value, 43) +strictEqual(require(commonJsTypeScriptFilename).epsilon, 6) commonJsTypeScriptHook.unhook() -const importedHook = new Hook([fileURLToPath(importedCommonJsUrl)], exports => { +const esmFilename = fileURLToPath(esmUrl) +const esmHook = new Hook([esmFilename], exports => { exports.foo = 43 }) -const imported = await import(importedCommonJsUrl) -strictEqual(imported.foo, 43) -strictEqual(imported.default.foo, 43) -importedHook.unhook() - -const esmHook = new Hook([fileURLToPath(requiredEsmUrl)], (exports, name, baseDir, data) => { - deepStrictEqual(data, { filename: 'something.mjs' }) - exports.foo = 57 -}) -const requiredEsm = require(fileURLToPath(requiredEsmUrl)) -strictEqual(requiredEsm.foo, 57) +strictEqual(require(esmFilename).foo, 43) esmHook.unhook() -const packageEsmHook = new Hook([fileURLToPath(requiredPackageEsmUrl)], exports => { - exports.value = 57 -}) -const requiredPackageEsm = require(fileURLToPath(requiredPackageEsmUrl)) -strictEqual(requiredPackageEsm.value, 57) -packageEsmHook.unhook() - -const fsHook = new Hook(['fs'], (exports, name, baseDir, data) => { - strictEqual(name, 'fs') - strictEqual(baseDir, undefined) - deepStrictEqual(data, { filename: 'node:fs' }) - exports[Symbol.for('iitm.sync-commonjs')] = true - return exports +const marker = Symbol('iitm-commonjs') +const fsHook = new Hook(['fs'], exports => { + exports[marker] = true }) const fs = require('fs') strictEqual(fs, require('node:fs')) -strictEqual(fs[Symbol.for('iitm.sync-commonjs')], true) -ok(Object.isExtensible(fs)) +strictEqual(fs[marker], true) +delete fs[marker] fsHook.unhook() -const temporaryDirectory = await realpath(await mkdtemp(join(tmpdir(), 'iitm-commonjs-'))) -try { - const packageLessFilename = join(temporaryDirectory, 'package-less.js') - const packageLessUrl = pathToFileURL(packageLessFilename) - await writeFile(packageLessFilename, 'module.exports = { value: 42 }\n') - includedUrls.add(packageLessUrl.href) - - const packageLessHook = new Hook([packageLessFilename], exports => { - exports.value++ - }) - strictEqual(require(packageLessFilename).value, 43) - packageLessHook.unhook() -} finally { - await rm(temporaryDirectory, { recursive: true, force: true }) -} +const nodeTestHook = new Hook(['node:test'], exports => { + exports[marker] = true +}) +const nodeTest = require('node:test') +strictEqual(nodeTest, process.getBuiltinModule('node:test')) +strictEqual(nodeTest[marker], true) +delete nodeTest[marker] +nodeTestHook.unhook() diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts index da3f0061..4a6062de 100644 --- a/test/typescript/bundler.test.mts +++ b/test/typescript/bundler.test.mts @@ -3,14 +3,12 @@ import assert from 'node:assert/strict' import { createWrapperModule } from '../../bundler.mjs' const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href -const target = { namespace: 'file', path: moduleUrl } const wrapper = await createWrapperModule({ module: { url: moduleUrl, format: 'module', source: 'export const value = 42', specifier: './something.mjs', - target, data: { version: '1.0.0' } }, resolve () { @@ -22,6 +20,4 @@ const wrapper = await createWrapperModule({ }) assert.equal(wrapper.sideEffects, true) -assert.equal(wrapper.format, 'module') assert.equal(wrapper.imports[0].kind, 'runtime') -assert.equal(wrapper.imports[1].target, target) diff --git a/test/typescript/register-hooks.test.mts b/test/typescript/register-hooks.test.mts deleted file mode 100644 index 939f4a08..00000000 --- a/test/typescript/register-hooks.test.mts +++ /dev/null @@ -1,15 +0,0 @@ -import assert from 'node:assert/strict' - -import type { RegisterHooksOptions } from '../../register-hooks.mjs' - -type Data = { version: string } - -const options: RegisterHooksOptions = { - commonjs: true, - shouldInclude (url) { - if (!url.startsWith('file:')) return false - return { data: { version: '1.0.0' } } - } -} - -assert.equal(options.commonjs, true) From 6f8fde58047938f8520f799056be1ed21720244b Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 13:20:17 +0200 Subject: [PATCH 04/18] feat: expose module format to Hook callbacks ## Summary Expose the loader-known ESM or CommonJS format as the optional fifth Hook callback argument. ## Why Consumers that support both formats cannot safely infer namespace semantics from user exports. ## Test plan - npm test - npm run test:ts - npm run test:e2e - npm run lint --- index.d.ts | 4 +++- index.js | 2 +- test/low-level/bundler.mjs | 17 ++++++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/index.d.ts b/index.d.ts index cafd7d81..7d5a4ffc 100644 --- a/index.d.ts +++ b/index.d.ts @@ -19,6 +19,7 @@ export type Namespace = { [key: string]: any } * @param {baseDir} string The absolute path of the module, if not provided in * `name`. * @param {data} Data Optional metadata embedded by a bundler. + * @param {format} string The intercepted module format, when available. * @return any A value that can will be assigned to `exports.default`. This is * equivalent to doing that assignment in the body of this function. For * CommonJS modules, the value replaces `module.exports`. @@ -27,7 +28,8 @@ export type HookFn = ( exported: Namespace, name: string, baseDir: string|void, - data?: Data + data?: Data, + format?: 'module'|'commonjs' ) => any export type Options = { diff --git a/index.js b/index.js index 4934554c..d5574fa8 100644 --- a/index.js +++ b/index.js @@ -86,7 +86,7 @@ function callHookFn (hookFn, namespace, name, baseDir) { * @returns {unknown} */ function callExtendedHookFn (hookFn, namespace, name, baseDir, data, format) { - const replacement = hookFn(namespace, name, baseDir, data) + const replacement = hookFn(namespace, name, baseDir, data, format) if (format === 'commonjs') return replacement if (replacement && replacement !== namespace && 'default' in namespace) { namespace.default = replacement diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 9dca9e2a..af1a30ce 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -52,9 +52,11 @@ doesNotMatch(wrapper.code, /from "file:/) * @param {string} name * @param {string|undefined} baseDir * @param {object} data + * @param {string} format */ -function hookFoo (exported, name, baseDir, data) { +function hookFoo (exported, name, baseDir, data, format) { deepStrictEqual(data, { version: '1.0.0' }) + strictEqual(format, 'module') exported.foo = 43 return () => 44 } @@ -183,10 +185,19 @@ await rejects(createWrapperModule({ message: `The bundler load adapter returned no source for '${commonJsUrl}'` }) -const commonJsHook = new Hook(['./something.js'], (exports, name, baseDir, data) => { +/** + * @param {object} exports + * @param {string} name + * @param {string|undefined} baseDir + * @param {object} data + * @param {string} format + */ +const commonJsHookFn = (exports, name, baseDir, data, format) => { deepStrictEqual(data, { version: '1.0.0' }) + strictEqual(format, 'commonjs') return { ...exports, hooked: true } -}) +} +const commonJsHook = new Hook(['./something.js'], commonJsHookFn) let unfilteredCalls = 0 const unfilteredHook = new Hook((exports, name, baseDir, data) => { if (data?.version === '1.0.0') unfilteredCalls++ From 9977a4d131a3b262fb6f70de4d53b4861241c1a3 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 14:00:53 +0200 Subject: [PATCH 05/18] fix: emit portable wrapper export names Emit valid identifier export names without string-literal syntax while preserving quoted names that require it. Webpack 5.54 accepts string-literal export names in the parser but crashes while analyzing the generated module. Most package exports are ordinary identifiers and do not need the newer syntax. - npm test - npm run test:ts - npm run test:e2e - npm run lint - webpack 5.54.0 ESM integration in dd-trace-js - webpack 5.109.2 ESM integration in dd-trace-js --- lib/wrapper.mjs | 3 ++- test/low-level/bundler.mjs | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index 80b6bc5d..ccbb3945 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -16,6 +16,7 @@ import { RESOLVE } from './io.mjs' // cycle would otherwise hit. Below it the recursion pays only an integer // compare per level and allocates no set. const STAR_CYCLE_DEPTH = 100 +const IDENTIFIER_NAME_REGEXP = /^[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}]*$/u /** @typedef {{ name: string, origin: string }} StarBinding */ /** @@ -277,7 +278,7 @@ function buildESMWrapperSource ({ } writeCases += ` case ${index++}: ${variableName} = value; break\n` if (shouldReexport(name, realUrl)) { - const exportName = name === 'default' ? name : objectKey + const exportName = IDENTIFIER_NAME_REGEXP.test(name) ? name : objectKey exportSpecifiers += exportSpecifiers === '' ? `${variableName} as ${exportName}` : `, ${variableName} as ${exportName}` diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index af1a30ce..27e23309 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -89,7 +89,7 @@ const rebuilt = await createWrapperModule({ load: unexpectedIo }) -match(rebuilt.code, /export \{ \$rebuilt as "rebuilt" \}/) +match(rebuilt.code, /export \{ \$rebuilt as rebuilt \}/) match(rebuilt.code, /\nregister\(/) doesNotMatch(rebuilt.code, /registerWithData/) doesNotMatch(rebuilt.code, /\$foo/) @@ -315,9 +315,22 @@ const commonJsReexportWrapper = await createWrapperModule({ load: loadModule }) -match(commonJsReexportWrapper.code, /export \{ \$foo as "foo" \}/) +match(commonJsReexportWrapper.code, /export \{ \$foo as foo \}/) doesNotMatch(commonJsReexportWrapper.code, /as default/) +const quotedExportWrapper = await createWrapperModule({ + module: { + url: 'file:///virtual/quoted-export.mjs', + format: 'module', + source: 'const value = 42; export { value as "quoted name" }', + specifier: './quoted-export.mjs' + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +match(quotedExportWrapper.code, /export \{ \$quoted_name as "quoted name" \}/) + /** * @param {string} specifier */ From d34b5a69b07d084d4ed524817f550db2a0d835d1 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 14:53:44 +0200 Subject: [PATCH 06/18] feat: add CommonJS bundler facade Bundler integrations need synchronous format detection before generating a wrapper, but that behavior was only available inside the ESM loader. The facade keeps format detection synchronous without eagerly loading the wrapper parser. --- README.md | 9 +++ bundler.d.ts | 7 ++ bundler.js | 17 +++++ create-hook.mjs | 64 +---------------- lib/get-node-module-format.js | 76 ++++++++++++++++++++ test/low-level/bundler.mjs | 36 +++++++++- test/register/v18.19-loader-url-escaping.mjs | 1 + test/typescript/bundler.test.mts | 2 + 8 files changed, 148 insertions(+), 64 deletions(-) create mode 100644 bundler.d.ts create mode 100644 bundler.js create mode 100644 lib/get-node-module-format.js diff --git a/README.md b/README.md index 151f6ad8..f855945a 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,15 @@ const wrapper = await createWrapperModule({ }) ``` +CommonJS integrations can use the lazy-loading facade. It also exposes the +module format detection used by the Node loader: + +```js +const { createWrapperModule, getNodeModuleFormat } = require('import-in-the-middle/bundler') + +const format = getNodeModuleFormat(url, packageJsonUrl, packageJson.type) +``` + `url` is the canonical `file:` or `node:` URL reported to hooks. `resolve` and `load` adapt the bundler's resolver and source loader to the same URL-based module graph. diff --git a/bundler.d.ts b/bundler.d.ts new file mode 100644 index 00000000..9ff71c04 --- /dev/null +++ b/bundler.d.ts @@ -0,0 +1,7 @@ +export * from './bundler.mjs' + +export declare function getNodeModuleFormat( + url: string, + packageJsonUrl?: string, + packageType?: string +): 'builtin' | 'module' | 'module-typescript' | 'commonjs' | 'commonjs-typescript' | undefined diff --git a/bundler.js b/bundler.js new file mode 100644 index 00000000..6a155110 --- /dev/null +++ b/bundler.js @@ -0,0 +1,17 @@ +'use strict' + +const getNodeModuleFormat = require('./lib/get-node-module-format.js') + +/** @type {typeof import('./bundler.mjs').createWrapperModule|undefined} */ +let createWrapperModuleImplementation + +/** + * @param {Parameters[0]} options + */ +async function createWrapperModule (options) { + createWrapperModuleImplementation ??= (await import('./bundler.mjs')).createWrapperModule + return createWrapperModuleImplementation(options) +} + +exports.createWrapperModule = createWrapperModule +exports.getNodeModuleFormat = getNodeModuleFormat diff --git a/create-hook.mjs b/create-hook.mjs index baf49556..79e10f5b 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -5,6 +5,7 @@ import { URL, fileURLToPath } from 'url' import { inspect } from 'util' import { builtinModules } from 'module' +import getNodeModuleFormat from './lib/get-node-module-format.js' import { driveSync, driveAsync } from './lib/io.mjs' import { buildCommonJSWrapperSource, buildWrapperSource, processModule } from './lib/wrapper.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' @@ -25,7 +26,6 @@ const HANDLED_FORMATS = new Set([ 'builtin', 'module', 'commonjs', 'module-typescript', 'commonjs-typescript' ]) const TRACE_WARNINGS = process.execArgv.includes('--trace-warnings') -let packageTypes /** @typedef {import('node:module').LoadHookContext} LoadContext */ /** @typedef {import('node:module').LoadFnOutput} LoadResult */ @@ -148,66 +148,6 @@ function addIitm (url) { return urlObj.href } -/** - * @param {'.js'|'.ts'} extension - * @param {string|undefined} type - * @returns {'module'|'module-typescript'|'commonjs'|'commonjs-typescript'} - */ -function getPackageFormat (extension, type) { - if (type === 'module') return extension === '.ts' ? 'module-typescript' : 'module' - return extension === '.ts' ? 'commonjs-typescript' : 'commonjs' -} - -/** - * @param {string} url - * @returns {string|undefined} - */ -function getFileFormat (url) { - if (!url.startsWith('file:')) return undefined - const pathname = new URL(url).pathname - let extension - if (pathname.endsWith('.mjs')) extension = '.mjs' - else if (pathname.endsWith('.cjs')) extension = '.cjs' - else if (pathname.endsWith('.mts')) extension = '.mts' - else if (pathname.endsWith('.cts')) extension = '.cts' - else if (pathname.endsWith('.js')) extension = '.js' - else if (pathname.endsWith('.ts')) extension = '.ts' - else return undefined - - if (extension === '.mjs') return 'module' - if (extension === '.cjs') return 'commonjs' - if (extension === '.mts') return 'module-typescript' - if (extension === '.cts') return 'commonjs-typescript' - - packageTypes ??= new Map() - const visited = [] - let directory = new URL('.', url) - while (true) { - if (packageTypes.has(directory.href)) { - const type = packageTypes.get(directory.href) - for (const href of visited) packageTypes.set(href, type) - return getPackageFormat(extension, type) - } - - visited.push(directory.href) - try { - const source = process.getBuiltinModule('fs').readFileSync(new URL('package.json', directory), 'utf8') - const type = JSON.parse(source).type - packageTypes.set(directory.href, type) - continue - } catch (error) { - if (error.code !== 'ENOENT') return undefined - } - - const parent = new URL('../', directory) - if (parent.href === directory.href) { - for (const href of visited) packageTypes.set(href, undefined) - return getPackageFormat(extension, undefined) - } - directory = parent - } -} - /** * @param {{ url: string }} meta * @param {boolean} [commonjs] Whether to create CommonJS-specific synchronous hooks. @@ -442,7 +382,7 @@ export function createHook (meta, commonjs) { } } - const format = result.format ?? (result.url.startsWith('node:') ? 'builtin' : getFileFormat(result.url)) + const format = result.format ?? getNodeModuleFormat(result.url) if (format === 'module' || format === 'module-typescript') { const specifierData = format === 'module-typescript' ? { specifier, format } : specifier specifiers.set(result.url, specifierData) diff --git a/lib/get-node-module-format.js b/lib/get-node-module-format.js new file mode 100644 index 00000000..8f3733c6 --- /dev/null +++ b/lib/get-node-module-format.js @@ -0,0 +1,76 @@ +'use strict' + +const { readFileSync } = process.getBuiltinModule?.('fs') ?? require('node:fs') + +let packageTypes + +/** @typedef {'builtin'|'module'|'module-typescript'|'commonjs'|'commonjs-typescript'} NodeModuleFormat */ + +/** + * @param {'.js'|'.ts'} extension + * @param {string|undefined} type + * @returns {NodeModuleFormat} + */ +function getPackageFormat (extension, type) { + if (type === 'module') return extension === '.ts' ? 'module-typescript' : 'module' + return extension === '.ts' ? 'commonjs-typescript' : 'commonjs' +} + +/** + * @param {string} url + * @param {string} [packageJsonUrl] + * @param {string} [packageType] + * @returns {NodeModuleFormat|undefined} + */ +module.exports = function getNodeModuleFormat (url, packageJsonUrl, packageType) { + if (url.startsWith('node:')) return 'builtin' + if (!url.startsWith('file:')) return undefined + const pathname = new URL(url).pathname + let extension + if (pathname.endsWith('.mjs')) extension = '.mjs' + else if (pathname.endsWith('.cjs')) extension = '.cjs' + else if (pathname.endsWith('.mts')) extension = '.mts' + else if (pathname.endsWith('.cts')) extension = '.cts' + else if (pathname.endsWith('.js')) extension = '.js' + else if (pathname.endsWith('.ts')) extension = '.ts' + else return undefined + + if (extension === '.mjs') return 'module' + if (extension === '.cjs') return 'commonjs' + if (extension === '.mts') return 'module-typescript' + if (extension === '.cts') return 'commonjs-typescript' + + packageTypes ??= new Map() + const packageDirectory = packageJsonUrl === undefined ? undefined : new URL('.', packageJsonUrl).href + const visited = [] + let directory = new URL('.', url) + while (true) { + if (directory.href === packageDirectory) { + return getPackageFormat(extension, packageType) + } + + if (packageDirectory === undefined && packageTypes.has(directory.href)) { + const type = packageTypes.get(directory.href) + for (const href of visited) packageTypes.set(href, type) + return getPackageFormat(extension, type) + } + + visited.push(directory.href) + try { + const source = readFileSync(new URL('package.json', directory), 'utf8') + const type = JSON.parse(source).type + packageTypes.set(directory.href, type) + if (packageDirectory !== undefined) return getPackageFormat(extension, type) + continue + } catch (error) { + if (error.code !== 'ENOENT') return undefined + } + + const parent = new URL('../', directory) + if (parent.href === directory.href) { + for (const href of visited) packageTypes.set(href, undefined) + return getPackageFormat(extension, undefined) + } + directory = parent + } +} diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 27e23309..b87495a5 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -1,5 +1,5 @@ import { strictEqual, deepStrictEqual, match, doesNotMatch, rejects } from 'assert' -import { readFile, mkdtemp, writeFile, rm } from 'fs/promises' +import { readFile, mkdir, mkdtemp, writeFile, rm } from 'fs/promises' import { createRequire } from 'module' import { tmpdir } from 'os' import { join } from 'path' @@ -9,6 +9,10 @@ import Hook from '../../index.js' import { createWrapperModule } from '../../bundler.mjs' const require = createRequire(import.meta.url) +const { + createWrapperModule: createCommonJSWrapperModule, + getNodeModuleFormat +} = require('../../bundler.js') const { registerWithData } = require('../../lib/bundler-runtime.js') const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href const source = await readFile(new URL(moduleUrl), 'utf8') @@ -20,7 +24,7 @@ function unexpectedIo () { throw new Error('I/O should not be used when source is provided') } -const wrapper = await createWrapperModule({ +const wrapper = await createCommonJSWrapperModule({ module: { url: moduleUrl, format: 'module', @@ -47,6 +51,34 @@ match(wrapper.code, /\nregisterWithData\(/) match(wrapper.code, /\{"version":"1\.0\.0"\}\)/) doesNotMatch(wrapper.code, /from "file:/) +const formatDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-format-')) +try { + const packageJsonUrl = pathToFileURL(join(formatDirectory, 'package.json')).href + strictEqual( + getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'seeded.js')).href, packageJsonUrl, 'module'), + 'module' + ) + await writeFile(join(formatDirectory, 'package.json'), '{"type":"module"}') + const nestedDirectory = join(formatDirectory, 'nested') + await mkdir(nestedDirectory) + await writeFile(join(nestedDirectory, 'package.json'), '{"type":"commonjs"}') + strictEqual( + getNodeModuleFormat(pathToFileURL(join(nestedDirectory, 'module.js')).href, packageJsonUrl, 'module'), + 'commonjs' + ) + strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.js')).href), 'module') + strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.ts')).href), 'module-typescript') + strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.mjs')).href), 'module') + strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.cjs')).href), 'commonjs') + strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.mts')).href), 'module-typescript') + strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.cts')).href), 'commonjs-typescript') +} finally { + await rm(formatDirectory, { recursive: true, force: true }) +} + +strictEqual(getNodeModuleFormat('node:fs'), 'builtin') +strictEqual(getNodeModuleFormat(moduleUrl.replace(/\.mjs$/, '.json')), undefined) + /** * @param {object} exported * @param {string} name diff --git a/test/register/v18.19-loader-url-escaping.mjs b/test/register/v18.19-loader-url-escaping.mjs index 79c52865..e8c234d0 100644 --- a/test/register/v18.19-loader-url-escaping.mjs +++ b/test/register/v18.19-loader-url-escaping.mjs @@ -22,6 +22,7 @@ try { 'supports-sync-hooks.mjs', 'lib/get-esm-exports.mjs', 'lib/get-exports.mjs', + 'lib/get-node-module-format.js', 'lib/io.mjs', 'lib/register.js', 'lib/wrapper.mjs' diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts index 4a6062de..ea3bc0ee 100644 --- a/test/typescript/bundler.test.mts +++ b/test/typescript/bundler.test.mts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict' +import { getNodeModuleFormat } from '../../bundler.js' import { createWrapperModule } from '../../bundler.mjs' const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href @@ -21,3 +22,4 @@ const wrapper = await createWrapperModule({ assert.equal(wrapper.sideEffects, true) assert.equal(wrapper.imports[0].kind, 'runtime') +assert.equal(getNodeModuleFormat(moduleUrl), 'module') From a1a2db14da9ae5d323094b177717403b89a8f5c6 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 15:25:04 +0200 Subject: [PATCH 07/18] fix: preserve CommonJS internal module names ## Summary Report package-relative names for CommonJS modules registered through extended hooks. ## Why CommonJS hooks historically receive package internals without opting into ESM internal interception. Bundler and synchronous loader wrappers need the same contract or file-specific instrumentation such as express/lib/express.js is skipped. ## Test plan - npm test - npm run test:ts - npm run lint --- index.js | 2 +- test/low-level/bundler.mjs | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index d5574fa8..e23ad8ab 100644 --- a/index.js +++ b/index.js @@ -143,7 +143,7 @@ function callExtendedHook (hookFn, modules, internals, name, namespace, specifie result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) } else if (baseDir.endsWith(specifiers.get(loadUrl)) || isTurbopackSpecifier(specifiers.get(loadUrl), baseDir)) { result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) - } else if (internals) { + } else if (internals || format === 'commonjs') { const internalPath = name + path.sep + path.relative(baseDir, filePath) result = callExtendedHookFn(hookFn, namespace, internalPath, baseDir, data, format) } diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index b87495a5..f8829c74 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -13,7 +13,7 @@ const { createWrapperModule: createCommonJSWrapperModule, getNodeModuleFormat } = require('../../bundler.js') -const { registerWithData } = require('../../lib/bundler-runtime.js') +const { registerCommonJS, registerWithData } = require('../../lib/bundler-runtime.js') const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href const source = await readFile(new URL(moduleUrl), 'utf8') @@ -284,6 +284,23 @@ registerWithData(packageInternalUrl, {}, {}, {}, 'some-external-module/sub', und strictEqual(packageInternalName, join('some-external-module', 'sub.mjs')) packageInternalHook.unhook() +let commonJsPackageName +const commonJsPackageHook = new Hook(['some-external-module'], (exports, name) => { + commonJsPackageName = name +}) +commonJsPackageName = undefined +registerCommonJS(hookedPackageUrl, { exports: {} }, 'some-external-module', undefined) +strictEqual(commonJsPackageName, 'some-external-module') +commonJsPackageName = undefined +registerCommonJS( + new URL('../fixtures/node_modules/some-external-module/sub.js', import.meta.url).href, + { exports: {} }, + './sub', + undefined +) +strictEqual(commonJsPackageName, join('some-external-module', 'sub.js')) +commonJsPackageHook.unhook() + let invalidFileUrlName const invalidFileUrlHook = new Hook((exports, name) => { invalidFileUrlName = name From 1325cebd1b3fbf1e34a417028ea551989813dd78 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 15:44:43 +0200 Subject: [PATCH 08/18] fix: preserve bundler CommonJS bindings ## Summary Keep require, module, exports, __filename, and __dirname bound to the bundler factory when createWrapperModule emits a CommonJS wrapper. ## Why Shadowing require in the generated factory prevents bundlers from resolving relative dependencies. The synchronous Node loader still uses explicit CommonJS parameters, while bundler wrappers preserve their outer bindings. ## Test plan - npm test - npm run test:ts - npm run lint --- bundler.mjs | 3 ++- lib/wrapper.mjs | 21 ++++++++++++++++----- test/low-level/bundler.mjs | 2 ++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/bundler.mjs b/bundler.mjs index 9ed31667..093a54e3 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -137,7 +137,8 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } source, originalSpecifier: moduleData.specifier, data: moduleData.data, - runtimeSpecifier: RUNTIME_SPECIFIER + runtimeSpecifier: RUNTIME_SPECIFIER, + preserveOuterBindings: true }), imports: [{ specifier: RUNTIME_SPECIFIER, diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index ccbb3945..969da870 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -341,6 +341,15 @@ function sourceToString (source) { return Buffer.from(source).toString('utf8') } +/** + * @param {string | ArrayBuffer | ArrayBufferView} source + * @returns {string} + */ +function prepareCommonJSSource (source) { + source = sourceToString(source) + return source.startsWith('#!') ? '//' + source.slice(2) : source +} + /** * @param {object} options * @param {string} options.realUrl @@ -348,6 +357,7 @@ function sourceToString (source) { * @param {string} options.originalSpecifier * @param {unknown} [options.data] * @param {string} options.runtimeSpecifier + * @param {boolean} [options.preserveOuterBindings] * @returns {string} */ export function buildCommonJSWrapperSource ({ @@ -355,13 +365,14 @@ export function buildCommonJSWrapperSource ({ source, originalSpecifier, data, - runtimeSpecifier + runtimeSpecifier, + preserveOuterBindings }) { - source = sourceToString(source) - if (source.startsWith('#!')) source = '//' + source.slice(2) + source = prepareCommonJSSource(source) - return `(function (exports, require, module, __filename, __dirname) {${source}\n` + - '}).call(module.exports, module.exports, require, module, __filename, __dirname)\n' + + const parameters = preserveOuterBindings ? '' : 'exports, require, module, __filename, __dirname' + const argumentsList = preserveOuterBindings ? '' : ', module.exports, require, module, __filename, __dirname' + return `(function (${parameters}) {${source}\n}).call(module.exports${argumentsList})\n` + `require(${JSON.stringify(runtimeSpecifier)}).registerCommonJS(` + `${JSON.stringify(realUrl)}, module, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)})\n` } diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index f8829c74..7c574025 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -163,8 +163,10 @@ const commonJsWrapper = await createWrapperModule({ strictEqual(commonJsWrapper.imports.length, 1) strictEqual(commonJsWrapper.imports[0].kind, 'runtime') +match(commonJsWrapper.code, /^\(function \(\) \{/) match(commonJsWrapper.code, /registerCommonJS/) doesNotMatch(commonJsWrapper.code, /^(?:import|export) /m) +doesNotMatch(commonJsWrapper.code, /function \(exports, require, module/) const loadedCommonJsWrapper = await createWrapperModule({ module: { From 569938708d82bc24b7c322e3e6fd394a08be2289 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 16:08:30 +0200 Subject: [PATCH 09/18] fix: avoid sync resolution in loader workers ## Summary Inject the filesystem reader into shared module-format detection instead of requiring it from the CommonJS helper. ## Why Node.js 21 routes a CommonJS builtin require from an active ESM loader through resolveSync, but its loader worker does not implement that method. Importing fs in the ESM caller and requiring it in the CommonJS facade keeps each path on its native module system. ## Test plan - npm test - npm run lint - Node.js 21.7.3 npm test - Node.js 21.7.3 test/other/double-loading.mjs --- bundler.js | 6 +- create-hook.mjs | 4 +- lib/get-node-module-format.js | 108 ++++++++++++++++++---------------- 3 files changed, 64 insertions(+), 54 deletions(-) diff --git a/bundler.js b/bundler.js index 6a155110..d4d9cbe4 100644 --- a/bundler.js +++ b/bundler.js @@ -1,6 +1,10 @@ 'use strict' -const getNodeModuleFormat = require('./lib/get-node-module-format.js') +const { readFileSync } = require('node:fs') + +const createGetNodeModuleFormat = require('./lib/get-node-module-format.js') + +const getNodeModuleFormat = createGetNodeModuleFormat(readFileSync) /** @type {typeof import('./bundler.mjs').createWrapperModule|undefined} */ let createWrapperModuleImplementation diff --git a/create-hook.mjs b/create-hook.mjs index 79e10f5b..3dfcae6b 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -5,7 +5,8 @@ import { URL, fileURLToPath } from 'url' import { inspect } from 'util' import { builtinModules } from 'module' -import getNodeModuleFormat from './lib/get-node-module-format.js' +import { readFileSync } from 'fs' +import createGetNodeModuleFormat from './lib/get-node-module-format.js' import { driveSync, driveAsync } from './lib/io.mjs' import { buildCommonJSWrapperSource, buildWrapperSource, processModule } from './lib/wrapper.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' @@ -16,6 +17,7 @@ import { supportsSyncHooks } from './supports-sync-hooks.mjs' export { supportsSyncHooks } const isWin = process.platform === 'win32' +const getNodeModuleFormat = createGetNodeModuleFormat(readFileSync) // FIXME: Typescript extensions are added temporarily until we find a better // way of supporting arbitrary extensions diff --git a/lib/get-node-module-format.js b/lib/get-node-module-format.js index 8f3733c6..65c57b1c 100644 --- a/lib/get-node-module-format.js +++ b/lib/get-node-module-format.js @@ -1,9 +1,5 @@ 'use strict' -const { readFileSync } = process.getBuiltinModule?.('fs') ?? require('node:fs') - -let packageTypes - /** @typedef {'builtin'|'module'|'module-typescript'|'commonjs'|'commonjs-typescript'} NodeModuleFormat */ /** @@ -17,60 +13,68 @@ function getPackageFormat (extension, type) { } /** - * @param {string} url - * @param {string} [packageJsonUrl] - * @param {string} [packageType] - * @returns {NodeModuleFormat|undefined} + * @param {typeof import('node:fs').readFileSync} readFileSync + * @returns {(url: string, packageJsonUrl?: string, packageType?: string) => NodeModuleFormat|undefined} */ -module.exports = function getNodeModuleFormat (url, packageJsonUrl, packageType) { - if (url.startsWith('node:')) return 'builtin' - if (!url.startsWith('file:')) return undefined - const pathname = new URL(url).pathname - let extension - if (pathname.endsWith('.mjs')) extension = '.mjs' - else if (pathname.endsWith('.cjs')) extension = '.cjs' - else if (pathname.endsWith('.mts')) extension = '.mts' - else if (pathname.endsWith('.cts')) extension = '.cts' - else if (pathname.endsWith('.js')) extension = '.js' - else if (pathname.endsWith('.ts')) extension = '.ts' - else return undefined +module.exports = function createGetNodeModuleFormat (readFileSync) { + let packageTypes - if (extension === '.mjs') return 'module' - if (extension === '.cjs') return 'commonjs' - if (extension === '.mts') return 'module-typescript' - if (extension === '.cts') return 'commonjs-typescript' + /** + * @param {string} url + * @param {string} [packageJsonUrl] + * @param {string} [packageType] + * @returns {NodeModuleFormat|undefined} + */ + return function getNodeModuleFormat (url, packageJsonUrl, packageType) { + if (url.startsWith('node:')) return 'builtin' + if (!url.startsWith('file:')) return undefined + const pathname = new URL(url).pathname + let extension + if (pathname.endsWith('.mjs')) extension = '.mjs' + else if (pathname.endsWith('.cjs')) extension = '.cjs' + else if (pathname.endsWith('.mts')) extension = '.mts' + else if (pathname.endsWith('.cts')) extension = '.cts' + else if (pathname.endsWith('.js')) extension = '.js' + else if (pathname.endsWith('.ts')) extension = '.ts' + else return undefined - packageTypes ??= new Map() - const packageDirectory = packageJsonUrl === undefined ? undefined : new URL('.', packageJsonUrl).href - const visited = [] - let directory = new URL('.', url) - while (true) { - if (directory.href === packageDirectory) { - return getPackageFormat(extension, packageType) - } + if (extension === '.mjs') return 'module' + if (extension === '.cjs') return 'commonjs' + if (extension === '.mts') return 'module-typescript' + if (extension === '.cts') return 'commonjs-typescript' - if (packageDirectory === undefined && packageTypes.has(directory.href)) { - const type = packageTypes.get(directory.href) - for (const href of visited) packageTypes.set(href, type) - return getPackageFormat(extension, type) - } + packageTypes ??= new Map() + const packageDirectory = packageJsonUrl === undefined ? undefined : new URL('.', packageJsonUrl).href + const visited = [] + let directory = new URL('.', url) + while (true) { + if (directory.href === packageDirectory) { + return getPackageFormat(extension, packageType) + } - visited.push(directory.href) - try { - const source = readFileSync(new URL('package.json', directory), 'utf8') - const type = JSON.parse(source).type - packageTypes.set(directory.href, type) - if (packageDirectory !== undefined) return getPackageFormat(extension, type) - continue - } catch (error) { - if (error.code !== 'ENOENT') return undefined - } + if (packageDirectory === undefined && packageTypes.has(directory.href)) { + const type = packageTypes.get(directory.href) + for (const href of visited) packageTypes.set(href, type) + return getPackageFormat(extension, type) + } + + visited.push(directory.href) + try { + const source = readFileSync(new URL('package.json', directory), 'utf8') + const type = JSON.parse(source).type + packageTypes.set(directory.href, type) + if (packageDirectory !== undefined) return getPackageFormat(extension, type) + continue + } catch (error) { + if (error.code !== 'ENOENT') return undefined + } - const parent = new URL('../', directory) - if (parent.href === directory.href) { - for (const href of visited) packageTypes.set(href, undefined) - return getPackageFormat(extension, undefined) + const parent = new URL('../', directory) + if (parent.href === directory.href) { + for (const href of visited) packageTypes.set(href, undefined) + return getPackageFormat(extension, undefined) + } + directory = parent } - directory = parent } } From 927ab563fa8eec9083130919f3a987695fe293f3 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 4 Aug 2026 16:38:57 +0200 Subject: [PATCH 10/18] refactor: key extended hooks by loader hook Key extended hooks by their existing loader hook instead of tracking a parallel array and WeakMap. The registries must be removed together. One key makes that invariant explicit and avoids extra per-Hook bookkeeping without changing legacy ESM dispatch. - npm test - npm run test:ts - npm run lint --- index.js | 15 ++++----------- lib/register.js | 6 +++--- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index e23ad8ab..4e40f03b 100644 --- a/index.js +++ b/index.js @@ -20,8 +20,6 @@ const { toHookExtended } = require('./lib/register') -const hookExtensions = new WeakMap() - /** * Checks turbopack specifiers separately (for Next.js 16+). * @@ -45,7 +43,7 @@ function isTurbopackSpecifier (specifier, baseDir) { function addHook (hook, extendedHook = hook) { importHooks.push(hook) toHook.forEach(([name, namespace, specifier]) => hook(name, namespace, specifier)) - extendedHooks.push(extendedHook) + extendedHooks.set(hook, extendedHook) for (const entry of toHookExtended) { const namespace = entry.module === undefined ? entry.namespace : entry.module.exports const replacement = extendedHook(entry.name, namespace, entry.specifier, entry.data, entry.format) @@ -53,15 +51,12 @@ function addHook (hook, extendedHook = hook) { } } -function removeHook (hook, extendedHook = hook) { +function removeHook (hook) { const index = importHooks.indexOf(hook) if (index > -1) { importHooks.splice(index, 1) } - const extendedIndex = extendedHooks.indexOf(extendedHook) - if (extendedIndex > -1) { - extendedHooks.splice(extendedIndex, 1) - } + extendedHooks.delete(hook) } function callHookFn (hookFn, namespace, name, baseDir) { @@ -307,13 +302,11 @@ function Hook (modules, options, hookFn) { } const extendedHook = callExtendedHook.bind(undefined, hookFn, modules, internals) - hookExtensions.set(this, extendedHook) addHook(this._iitmHook, extendedHook) } Hook.prototype.unhook = function () { - removeHook(this._iitmHook, hookExtensions.get(this)) - hookExtensions.delete(this) + removeHook(this._iitmHook) } module.exports = Hook diff --git a/lib/register.js b/lib/register.js index 8a140fcc..f277962e 100644 --- a/lib/register.js +++ b/lib/register.js @@ -6,7 +6,7 @@ const importHooks = [] // TODO should this be a Set? const binders = new WeakMap() const specifiers = new Map() const toHook = [] -const extendedHooks = [] +const extendedHooks = new Map() const toHookExtended = [] /** @@ -81,7 +81,7 @@ function registerWithData (name, binder, specifier, data) { specifiers.set(name, specifier) binders.set(namespace, binder) const proxy = new Proxy(namespace, proxyHandler) - for (const hook of extendedHooks) { + for (const hook of extendedHooks.values()) { hook(name, proxy, specifier, data, 'module') } toHookExtended.push({ name, namespace: proxy, specifier, data, format: 'module' }) @@ -104,7 +104,7 @@ function registerCommonJS (name, module, specifier, data) { format: 'commonjs', module } - for (const hook of extendedHooks) { + for (const hook of extendedHooks.values()) { const replacement = hook(name, module.exports, specifier, data, 'commonjs') if (replacement !== undefined) module.exports = replacement } From e37bb9d1dd307170dd265ae051b477894d05e903 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 8 Sep 2026 21:31:42 +0200 Subject: [PATCH 11/18] feat: preserve live exports in bundler wrappers Bundler wrappers copy every ESM export into a patchable local binding, so later source assignments stop reaching consumers of mutable exports. Selected exports retain their defining bindings while all other exports remain patchable through Hook. --- README.md | 12 +- bundler.d.mts | 11 + bundler.mjs | 62 +++-- index.d.ts | 20 +- index.js | 2 +- lib/get-esm-exports.mjs | 35 ++- lib/get-exports.mjs | 23 +- lib/register.js | 22 +- lib/wrapper.mjs | 231 +++++++++++++++- package.json | 4 +- .../test-nextjs-app/app/api/foo/route.ts | 4 +- .../test-nextjs-app/iitm-turbopack.d.mts | 8 + .../test-nextjs-app/iitm-turbopack.mjs | 29 ++ test/get-esm-exports/v20-get-esm-exports.mjs | 15 ++ .../turbopack-build-start.mjs | 123 ++++----- test/integration-tests/turbopack-dev.mjs | 137 +++++----- test/integration-tests/turbopack-server.mjs | 86 ++++++ test/integration-tests/turbopack-wrapper.mjs | 114 ++++++++ test/low-level/bundler.mjs | 254 ++++++++++++++++-- test/low-level/module-binder.mjs | 19 ++ test/other/v18-bundlers.mjs | 230 ++++++++++++++++ test/typescript/bundler.test.mts | 3 +- test/typescript/ts-node.test.mts | 16 +- 23 files changed, 1252 insertions(+), 208 deletions(-) create mode 100644 test/fixtures/test-nextjs-app/iitm-turbopack.d.mts create mode 100644 test/fixtures/test-nextjs-app/iitm-turbopack.mjs create mode 100644 test/integration-tests/turbopack-server.mjs create mode 100644 test/integration-tests/turbopack-wrapper.mjs create mode 100644 test/other/v18-bundlers.mjs diff --git a/README.md b/README.md index f855945a..de9da28a 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Bundlers can generate ESM and CommonJS wrappers with import { createWrapperModule } from 'import-in-the-middle/bundler.mjs' const wrapper = await createWrapperModule({ - module: { url, format, source, specifier, data }, + module: { url, format, source, specifier, data, passthroughExports }, resolve, load }) @@ -146,6 +146,16 @@ The optional `data` value must be JSON-serializable. It is embedded in the wrapper and passed as the fourth argument to `Hook` callbacks, allowing package metadata needed by instrumentation to reach the bundled runtime. +`passthroughExports` identifies ESM exports that must keep their original live +bindings. It accepts an iterable of names or a selector that receives all +resolved exports after IITM loads the module graph. Each resolved export has +the public `name`, the defining module `url`, and its `localName` when it has a +local ESM binding. IITM emits direct re-exports for selected names and exposes +their current values to `Hook` callbacks. Assignments from a callback do not +replace these bindings. Other exports remain patchable. This option has no +effect on CommonJS modules, and names that the module does not export are +ignored. + The result contains generated `code`, an `imports` manifest, `watchFiles`, and `sideEffects: true`. The code imports only relative placeholder specifiers. The bundler adapter provides it as a virtual module and maps each placeholder using diff --git a/bundler.d.mts b/bundler.d.mts index 5d6b2996..4c934036 100644 --- a/bundler.d.mts +++ b/bundler.d.mts @@ -8,12 +8,23 @@ export type JsonValue = | JsonValue[] | { [key: string]: JsonValue } +export type WrapperExport = { + name: string + url: string + localName?: string +} + +export type PassthroughExports = + | Iterable + | ((exports: readonly WrapperExport[]) => Iterable) + export type BundlerModule = { url: string format: string specifier: string source?: WrapperSource data?: Data + passthroughExports?: PassthroughExports } export type ModuleContext = { diff --git a/bundler.mjs b/bundler.mjs index 093a54e3..c8074b59 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -5,9 +5,9 @@ import { builtinModules } from 'module' import { driveAsync } from './lib/io.mjs' import { buildCommonJSWrapperSource, - buildWrapperSource, buildWrapperSourceWithData, - processModule + processModule, + resolveExportBindings } from './lib/wrapper.mjs' const RUNTIME_SPECIFIER = './__iitm_runtime__.js' @@ -21,6 +21,11 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @property {string} specifier * @property {string | ArrayBuffer | ArrayBufferView} [source] * @property {unknown} [data] + * @property {Iterable | ((exports: ReadonlyArray<{ + * name: string, + * url: string, + * localName?: string + * }>) => Iterable)} [passthroughExports] */ /** @@ -154,10 +159,30 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } } } - const { bindings } = await driveAsync( - processModule({ srcUrl: moduleData.url, context }), - { resolve: resolveModule, load: loadModule } - ) + const io = { resolve: resolveModule, load: loadModule } + const selectPassthroughExports = typeof moduleData.passthroughExports === 'function' + ? moduleData.passthroughExports + : undefined + const moduleExportsCache = selectPassthroughExports === undefined ? undefined : new Map() + const { bindings } = await driveAsync(processModule({ + srcUrl: moduleData.url, + context, + moduleExportsCache + }), io) + let selectedPassthroughExports = moduleData.passthroughExports + if (selectPassthroughExports !== undefined) { + const exportNames = Array.isArray(bindings) ? bindings.slice() : Array.from(bindings.keys()) + const exports = await driveAsync(resolveExportBindings({ + srcUrl: moduleData.url, + context, + exportNames, + moduleExportsCache + }), io) + selectedPassthroughExports = selectPassthroughExports(exports) + } + const passthroughExports = selectedPassthroughExports === undefined + ? undefined + : new Set(selectedPassthroughExports) /** @type {WrapperImport[]} */ const imports = [{ @@ -193,22 +218,15 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } return specifier } - const code = moduleData.data === undefined - ? buildWrapperSource({ - realUrl: moduleData.url, - bindings, - originalSpecifier: moduleData.specifier, - runtimeSpecifier: RUNTIME_SPECIFIER, - mapImport - }) - : buildWrapperSourceWithData({ - realUrl: moduleData.url, - bindings, - originalSpecifier: moduleData.specifier, - data: moduleData.data, - runtimeSpecifier: RUNTIME_SPECIFIER, - mapImport - }) + const code = buildWrapperSourceWithData({ + realUrl: moduleData.url, + bindings, + originalSpecifier: moduleData.specifier, + data: moduleData.data, + runtimeSpecifier: RUNTIME_SPECIFIER, + mapImport, + passthroughExports + }) return { code, diff --git a/index.d.ts b/index.d.ts index 7d5a4ffc..4d04b4d8 100644 --- a/index.d.ts +++ b/index.d.ts @@ -24,8 +24,8 @@ export type Namespace = { [key: string]: any } * equivalent to doing that assignment in the body of this function. For * CommonJS modules, the value replaces `module.exports`. */ -export type HookFn = ( - exported: Namespace, +export type HookFn = ( + exported: Exports, name: string, baseDir: string|void, data?: Data, @@ -36,7 +36,7 @@ export type Options = { internals?: boolean } -export declare class Hook { +export declare class Hook { /** * Creates a hook to be run on any already loaded modules and any that will * be loaded in the future. It will be run once per loaded module. If @@ -50,9 +50,9 @@ export declare class Hook { * unless they are mentioned specifically in the modules array. * @param {HookFunction} hookFn The function to be run on each module. */ - constructor (modules: Array, options: Options, hookFn: HookFn) - constructor (modules: Array, hookFn: HookFn) - constructor (hookFn: HookFn) + constructor (modules: Array, options: Options, hookFn: HookFn) + constructor (modules: Array, hookFn: HookFn) + constructor (hookFn: HookFn) /** * Disables this hook. It will no longer be run against any subsequently @@ -73,9 +73,9 @@ export default Hook * @param {data} Data Optional metadata embedded by a bundler. * @param {format} string The intercepted module format, when available. */ -export type HookFunction = ( +export type HookFunction = ( url: string, - exported: Namespace, + exported: Exports, specifier: string, data?: Data, format?: 'module'|'commonjs' @@ -91,7 +91,7 @@ export type HookFunction = ( * single imported module, rather than with any filtering. * @param {HookFunction} hookFn The function to be run on each module. */ -export declare function addHook(hookFn: HookFunction): void +export declare function addHook(hookFn: HookFunction): void /** * Removes a hook that has been previously added with `addHook`. It will no @@ -101,7 +101,7 @@ export declare function addHook(hookFn: HookFunction): voi * `Hook` class. * @param {HookFunction} hookFn The function to be removed. */ -export declare function removeHook(hookFn: HookFunction): void +export declare function removeHook(hookFn: HookFunction): void type CreateAddHookMessageChannelReturn = { addHookMessagePort: MessagePort, diff --git a/index.js b/index.js index 4e40f03b..3db89eee 100644 --- a/index.js +++ b/index.js @@ -138,7 +138,7 @@ function callExtendedHook (hookFn, modules, internals, name, namespace, specifie result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) } else if (baseDir.endsWith(specifiers.get(loadUrl)) || isTurbopackSpecifier(specifiers.get(loadUrl), baseDir)) { result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) - } else if (internals || format === 'commonjs') { + } else if (internals) { const internalPath = name + path.sep + path.relative(baseDir, filePath) result = callExtendedHookFn(hookFn, namespace, internalPath, baseDir, data, format) } diff --git a/lib/get-esm-exports.mjs b/lib/get-esm-exports.mjs index 4d2c275e..6bed8555 100644 --- a/lib/get-esm-exports.mjs +++ b/lib/get-esm-exports.mjs @@ -5,10 +5,20 @@ import { parse } from 'es-module-lexer' /** * @typedef {object} LexedExports * @property {string[] | Set} exportNames + * @property {Map} [exportDeclarations] * @property {Array<{ specifier: string, parentURL: string }> | undefined} starReexports * @property {boolean} hasModuleSyntax */ +/** + * @typedef {object} ExportDeclaration + * @property {'direct' | 'reexport'} type + * @property {string} name + * @property {string} [localName] + * @property {string} [importName] + * @property {string} [specifier] + */ + /** * Decodes an exported identifier the way the JS engine would. es-module-lexer * leaves Unicode escapes in bare identifier exports (`export const \u0061 = 1`) @@ -58,11 +68,13 @@ export default function getEsmExports (moduleSource) { * * @param {string} moduleSource The source code of the module to lex. * @param {string} [parentURL] The URL of the module declaring star re-exports. + * @param {boolean} [includeExportDeclarations] Include local and re-export binding metadata. * @returns {LexedExports} */ -export function lexEsm (moduleSource, parentURL) { +export function lexEsm (moduleSource, parentURL, includeExportDeclarations = false) { const legacy = parentURL === undefined const exportNames = legacy ? new Set() : [] + let exportDeclarations let starReexports const [, exports, , hasModuleSyntax] = parse(moduleSource) @@ -82,8 +94,27 @@ export function lexEsm (moduleSource, parentURL) { } else { exportNames.push(name) } + if (includeExportDeclarations) { + /** @type {ExportDeclaration} */ + const declaration = { type: exported.type, name } + if (exported.type === 'direct') { + if (typeof exported.localName === 'string') { + declaration.localName = decodeExportName(exported.localName) + } + } else { + declaration.specifier = exported.from + if (typeof exported.importName === 'string') { + declaration.importName = decodeExportName(exported.importName) + } + } + exportDeclarations ??= new Map() + exportDeclarations.set(name, declaration) + } } } - return { exportNames, starReexports, hasModuleSyntax } + /** @type {LexedExports} */ + const result = { exportNames, starReexports, hasModuleSyntax } + if (exportDeclarations !== undefined) result.exportDeclarations = exportDeclarations + return result } diff --git a/lib/get-exports.mjs b/lib/get-exports.mjs index d396a54e..2460acd0 100644 --- a/lib/get-exports.mjs +++ b/lib/get-exports.mjs @@ -12,7 +12,18 @@ const nodeMajor = Number(process.versions.node.split('.')[0]) export const hasModuleExportsCJSDefault = nodeMajor >= 23 /** @typedef {{ specifier: string, parentURL: string }} StarReexport */ -/** @typedef {{ exportNames: Iterable, starReexports?: StarReexport[] }} ModuleExports */ +/** + * @typedef {object} ModuleExports + * @property {Iterable} exportNames + * @property {Map} [exportDeclarations] + * @property {StarReexport[]} [starReexports] + */ let parserInitialized = false let stripTypeScriptTypes @@ -239,16 +250,17 @@ function * getCjsExports (url, context, source) { * get the exports of. * @param {object} context Context object as provided by the `load` hook from * the loaders API. + * @param {boolean} [includeExportDeclarations] Include local and re-export binding metadata. * * @returns {Generator} A generator that yields I/O * operations and ultimately returns the identifiers and star re-exports of the * module. */ -export function * getExports (url, context) { +export function * getExports (url, context, includeExportDeclarations = false) { const useCache = context.cache !== false if (useCache) { const cached = esmExportsCache.get(url) - if (cached !== undefined) { + if (cached !== undefined && (!includeExportDeclarations || cached.exportDeclarations !== undefined)) { return cached } } @@ -302,8 +314,11 @@ export function * getExports (url, context) { return yield * getCjsExports(url, context, source) } - const { exportNames, starReexports, hasModuleSyntax } = lexEsm(source, url) + const { exportNames, exportDeclarations, starReexports, hasModuleSyntax } = + lexEsm(source, url, includeExportDeclarations) + /** @type {ModuleExports} */ const moduleExports = starReexports === undefined ? { exportNames } : { exportNames, starReexports } + if (exportDeclarations !== undefined) moduleExports.exportDeclarations = exportDeclarations if (moduleFormat === 'module') { if (useCache) esmExportsCache.set(url, moduleExports) diff --git a/lib/register.js b/lib/register.js index f277962e..4fbb81fb 100644 --- a/lib/register.js +++ b/lib/register.js @@ -140,14 +140,34 @@ class ModuleBinder { * @param {string[]} [keys] Export names in wrapper-binding order. * @param {(index: number, value: unknown) => void} [write] Assigns a wrapper binding by index. * @param {object[]} [sources] Alternate namespaces for star-collision bindings. + * @param {string[]} [passthroughKeys] Read-only export names that retain their source bindings. + * @param {object[]} [passthroughSources] Alternate namespaces for read-only bindings. */ - constructor (source, keys, write, sources) { + constructor (source, keys, write, sources, passthroughKeys, passthroughSources) { this.#write = write if (keys !== undefined) { for (let index = 0; index < keys.length; index++) { this.#bind(keys[index], index, sources?.[index] ?? source) } } + if (passthroughKeys !== undefined) { + for (let index = 0; index < passthroughKeys.length; index++) { + this.#bindReadOnly(passthroughKeys[index], passthroughSources?.[index] ?? source) + } + } + } + + /** + * @param {string} key The export name. + * @param {object} source The binding's source namespace. + * @returns {void} + */ + #bindReadOnly (key, source) { + Object.defineProperty(this.namespace, key, { + configurable: true, + enumerable: true, + get: () => readExport(source, key) + }) } /** diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index 969da870..4ac0da10 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -19,6 +19,21 @@ const STAR_CYCLE_DEPTH = 100 const IDENTIFIER_NAME_REGEXP = /^[$_\p{ID_Start}][$_\u200C\u200D\p{ID_Continue}]*$/u /** @typedef {{ name: string, origin: string }} StarBinding */ +/** @typedef {{ name: string, url: string, localName?: string }} ResolvedExportBinding */ +/** + * @typedef {object} ExportDeclaration + * @property {'direct' | 'reexport'} type + * @property {string} name + * @property {string} [localName] + * @property {string} [importName] + * @property {string} [specifier] + */ +/** + * @typedef {object} ModuleExportsMetadata + * @property {Iterable} exportNames + * @property {Map} [exportDeclarations] + * @property {Array<{ specifier: string, parentURL: string }>} [starReexports] + */ /** * @typedef {object} ProcessResult * @property {string[] | Map} bindings @@ -92,14 +107,24 @@ function shouldExcludeExport (name, sourceUrl) { * created lazily once `depth` crosses {@link STAR_CYCLE_DEPTH}. A URL is added * before descending into its subtree and removed once that subtree finishes, so * it tracks the active path rather than every URL ever visited. + * @param {Map} [params.moduleExportsCache] Parsed exports used by binding resolution. * @returns {Generator} * A generator that yields I/O operations and ultimately returns the shimmed * bindings for all the exports from the module and any transitive export all * modules. `origins` (the defining module per `*`-sourced name) is `undefined` * for a module with no `export *`. */ -export function * processModule ({ srcUrl, context, excludeDefault = false, depth = 0, seen }) { - const { exportNames, starReexports } = yield * getExports(srcUrl, context) +export function * processModule ({ + srcUrl, + context, + excludeDefault = false, + depth = 0, + seen, + moduleExportsCache +}) { + const moduleExports = yield * getExports(srcUrl, context, moduleExportsCache !== undefined) + if (moduleExportsCache !== undefined) moduleExportsCache.set(srcUrl, moduleExports) + const { exportNames, starReexports } = moduleExports // Most modules have no export star. Keep that path array-backed so it pays // neither merge bookkeeping nor a Map lookup for each direct export. @@ -179,7 +204,8 @@ export function * processModule ({ srcUrl, context, excludeDefault = false, dept context: { ...context, format: result.format }, excludeDefault: true, depth: depth + 1, - seen + seen, + moduleExportsCache }) for (const binding of sub.bindings.values()) { @@ -215,6 +241,164 @@ export function * processModule ({ srcUrl, context, excludeDefault = false, dept return { bindings, origins: starOrigins } } +/** + * Resolves exported names to the local bindings that define them. + * + * @param {object} params + * @param {string} params.srcUrl The root module URL. + * @param {object} params.context The root module load context. + * @param {Iterable} params.exportNames The root module's resolved export names. + * @param {Map} params.moduleExportsCache Parsed exports collected during graph traversal. + * @returns {Generator} + */ +export function * resolveExportBindings ({ srcUrl, context, exportNames, moduleExportsCache }) { + const memo = new Map() + const pending = new Set() + const bindings = [] + for (const name of exportNames) { + const binding = yield * resolveExportBinding({ + srcUrl, + name, + context, + moduleExportsCache, + memo, + pending + }) + bindings.push(binding ?? { name, url: srcUrl }) + } + return bindings +} + +/** + * @param {object} params + * @param {string} params.srcUrl The module URL. + * @param {string} params.name The exported name to resolve. + * @param {object} params.context The module load context. + * @param {Map} params.moduleExportsCache Parsed module exports. + * @param {Map} params.memo Resolved bindings by module and export name. + * @param {Set} params.pending Bindings on the active resolution path. + * @returns {Generator} + */ +function * resolveExportBinding ({ srcUrl, name, context, moduleExportsCache, memo, pending }) { + const key = `${srcUrl}\0${name}` + if (memo.has(key)) { + const binding = memo.get(key) + return binding === false ? undefined : binding + } + if (pending.has(key)) return + pending.add(key) + + let binding + try { + const moduleExports = yield * loadModuleExports(srcUrl, context, moduleExportsCache) + const declaration = moduleExports.exportDeclarations?.get(name) + if (declaration?.type === 'direct') { + binding = { name, url: srcUrl } + if (declaration.localName !== undefined) binding.localName = declaration.localName + } else if (declaration?.type === 'reexport' && declaration.specifier !== undefined) { + const target = yield * resolveBindingTarget(declaration.specifier, srcUrl) + if (declaration.importName === undefined) { + binding = { name, url: target.url } + } else { + const imported = yield * resolveExportBinding({ + srcUrl: target.url, + name: declaration.importName, + context: { ...context, format: target.format }, + moduleExportsCache, + memo, + pending + }) + if (imported !== undefined) binding = { ...imported, name } + } + } else if (name !== 'default' && moduleExports.starReexports !== undefined) { + binding = yield * resolveStarExport({ + name, + context, + starReexports: moduleExports.starReexports, + moduleExportsCache, + memo, + pending + }) + } else if (hasExportName(moduleExports.exportNames, name)) { + binding = { name, url: srcUrl } + } + } finally { + pending.delete(key) + } + + memo.set(key, binding ?? false) + return binding +} + +/** + * @param {object} params + * @param {string} params.name The star-exported name to resolve. + * @param {object} params.context The parent module load context. + * @param {Array<{ specifier: string, parentURL: string }>} params.starReexports Star re-export declarations. + * @param {Map} params.moduleExportsCache Parsed module exports. + * @param {Map} params.memo Resolved bindings by module and export name. + * @param {Set} params.pending Bindings on the active resolution path. + * @returns {Generator} + */ +function * resolveStarExport ({ name, context, starReexports, moduleExportsCache, memo, pending }) { + let binding + for (const { specifier, parentURL } of starReexports) { + const target = yield * resolveBindingTarget(specifier, parentURL) + const candidate = yield * resolveExportBinding({ + srcUrl: target.url, + name, + context: { ...context, format: target.format }, + moduleExportsCache, + memo, + pending + }) + if (candidate === undefined) continue + if (binding === undefined) { + binding = candidate + } else if (binding.url !== candidate.url || binding.localName !== candidate.localName) { + return + } + } + return binding === undefined ? undefined : { ...binding, name } +} + +/** + * @param {string} specifier The re-exported module specifier. + * @param {string} parentURL The declaring module URL. + * @returns {Generator} + */ +function * resolveBindingTarget (specifier, parentURL) { + const request = isBareSpecifier(specifier) ? specifier : new URL(specifier, parentURL).href + return yield [RESOLVE, request, { parentURL }] +} + +/** + * @param {string} srcUrl The module URL. + * @param {object} context The module load context. + * @param {Map} moduleExportsCache Parsed module exports. + * @returns {Generator} + */ +function * loadModuleExports (srcUrl, context, moduleExportsCache) { + let moduleExports = moduleExportsCache.get(srcUrl) + if (moduleExports === undefined) { + moduleExports = yield * getExports(srcUrl, context, true) + moduleExportsCache.set(srcUrl, moduleExports) + } + return moduleExports +} + +/** + * @param {Iterable} exportNames The module's exported names. + * @param {string} name The expected export name. + * @returns {boolean} + */ +function hasExportName (exportNames, name) { + for (const exportName of exportNames) { + if (exportName === name) return true + } + return false +} + /** * @typedef {object} WrapperOptions * @property {string} realUrl The URL of the wrapped module. @@ -222,6 +406,7 @@ export function * processModule ({ srcUrl, context, excludeDefault = false, dept * @property {string} originalSpecifier The specifier used to import the module. * @property {string} runtimeSpecifier The wrapper runtime import. * @property {(url: string) => string} [mapImport] Maps module URLs to bundler-owned imports. + * @property {ReadonlySet} [passthroughExports] Exports that retain their source bindings. */ /** @@ -235,7 +420,8 @@ function buildESMWrapperSource ({ originalSpecifier, data, runtimeSpecifier, - mapImport + mapImport, + passthroughExports }, withData) { const moduleSpecifier = mapImport?.(realUrl) ?? realUrl // The wrapped module imports its namespace as `namespace`, which serves @@ -249,12 +435,17 @@ function buildESMWrapperSource ({ let bindingNames = '' let bindingSources let exportSpecifiers = '' + let passthroughNames = '' + let passthroughReexports = '' + let passthroughSources + let passthroughIndex = 0 let writeCases = '' let index = 0 for (const binding of bindings.values()) { const directName = typeof binding === 'string' ? binding : undefined const name = directName ?? binding.name let namespaceName = 'namespace' + let sourceSpecifier = moduleSpecifier if (directName === undefined) { originNamespaces ??= new Map() namespaceName = originNamespaces.get(binding.origin) @@ -263,10 +454,29 @@ function buildESMWrapperSource ({ originNamespaces.set(binding.origin, namespaceName) const originSpecifier = mapImport?.(binding.origin) ?? binding.origin originImports += `import * as ${namespaceName} from ${JSON.stringify(originSpecifier)}\n` + sourceSpecifier = originSpecifier + } else { + sourceSpecifier = mapImport?.(binding.origin) ?? binding.origin } } - const variableName = `$${index}` const objectKey = JSON.stringify(name) + const exportName = IDENTIFIER_NAME_REGEXP.test(name) ? name : objectKey + if (passthroughExports?.has(name)) { + passthroughNames += passthroughNames === '' ? objectKey : `, ${objectKey}` + if (passthroughSources !== undefined) passthroughSources += ', ' + if (namespaceName !== 'namespace') { + passthroughSources ??= 'undefined, '.repeat(passthroughIndex) + passthroughSources += namespaceName + } else if (passthroughSources !== undefined) { + passthroughSources += 'undefined' + } + passthroughIndex++ + if (shouldReexport(name, realUrl)) { + passthroughReexports += `export { ${exportName} } from ${JSON.stringify(sourceSpecifier)}\n` + } + continue + } + const variableName = `$${index}` declarationNames += declarationNames === '' ? variableName : `, ${variableName}` bindingNames += bindingNames === '' ? objectKey : `, ${objectKey}` if (bindingSources !== undefined) bindingSources += ', ' @@ -278,14 +488,16 @@ function buildESMWrapperSource ({ } writeCases += ` case ${index++}: ${variableName} = value; break\n` if (shouldReexport(name, realUrl)) { - const exportName = IDENTIFIER_NAME_REGEXP.test(name) ? name : objectKey exportSpecifiers += exportSpecifiers === '' ? `${variableName} as ${exportName}` : `, ${variableName} as ${exportName}` } } + const passthroughSourceArguments = passthroughSources === undefined ? '' : `, [${passthroughSources}]` const binder = declarationNames === '' - ? 'const __binder = new ModuleBinder(namespace)\n' + ? passthroughNames === '' + ? 'const __binder = new ModuleBinder(namespace)\n' + : `const __binder = new ModuleBinder(namespace, undefined, undefined, undefined, [${passthroughNames}]${passthroughSourceArguments})\n` : `let ${declarationNames} function __write (index, value) { switch (index) { @@ -293,7 +505,9 @@ ${writeCases} } } const __binder = new ModuleBinder(namespace, [${bindingNames}], __write${bindingSources === undefined ? '' - : `, [${bindingSources}]`}) + : `, [${bindingSources}]`}${passthroughNames === '' + ? '' + : `${bindingSources === undefined ? ', undefined' : ''}, [${passthroughNames}]${passthroughSourceArguments}`}) ` const reexports = exportSpecifiers === '' ? '' : `export { ${exportSpecifiers} }\n` const registerName = withData ? 'registerWithData' : 'register' @@ -305,6 +519,7 @@ import * as namespace from ${JSON.stringify(moduleSpecifier)} ${originImports} ${binder} ${reexports} +${passthroughReexports} __binder.flush() diff --git a/package.json b/package.json index 436a25e2..d1d59a81 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@types/node": "^18.0.6", "c8": "^7.14.0", "date-fns": "^3.6.0", + "esbuild": "^0.28.1", "eslint": "^8.57.1", "eslint-config-standard": "^17.1.0", "eslint-plugin-import": "^2.32.0", @@ -56,7 +57,8 @@ "openai": "4.47.2", "ts-node": "^10.9.2", "typescript": "^4.9.5", - "vue": "^3.5.26" + "vue": "^3.5.26", + "webpack": "^5.109.2" }, "dependencies": { "cjs-module-lexer": "^2.2.0", diff --git a/test/fixtures/test-nextjs-app/app/api/foo/route.ts b/test/fixtures/test-nextjs-app/app/api/foo/route.ts index 7be61e0a..b2cacc33 100644 --- a/test/fixtures/test-nextjs-app/app/api/foo/route.ts +++ b/test/fixtures/test-nextjs-app/app/api/foo/route.ts @@ -1,8 +1,10 @@ import camelcase from 'camelcase' import { NextResponse } from 'next/server' +import { runIitmWrapper } from '../../../iitm-turbopack.mjs' + export async function GET (request: Request) { camelcase('foo bar') - return NextResponse.json({}) + return NextResponse.json(runIitmWrapper()) } diff --git a/test/fixtures/test-nextjs-app/iitm-turbopack.d.mts b/test/fixtures/test-nextjs-app/iitm-turbopack.d.mts new file mode 100644 index 00000000..849a9940 --- /dev/null +++ b/test/fixtures/test-nextjs-app/iitm-turbopack.d.mts @@ -0,0 +1,8 @@ +export type IitmWrapperResult = { + initialLive: number + live: number + stable: number + hookedLive: number +} + +export declare function runIitmWrapper(): IitmWrapperResult diff --git a/test/fixtures/test-nextjs-app/iitm-turbopack.mjs b/test/fixtures/test-nextjs-app/iitm-turbopack.mjs new file mode 100644 index 00000000..84031e20 --- /dev/null +++ b/test/fixtures/test-nextjs-app/iitm-turbopack.mjs @@ -0,0 +1,29 @@ +import Hook from '../../../index.js' +import * as wrapped from './iitm-wrapper.mjs' + +/** + * @returns {{ initialLive: number, live: number, stable: number, hookedLive: number }} + */ +export function runIitmWrapper () { + let hookedExports + /** @param {Record} exports The intercepted exports. */ + function hookExports (exports) { + hookedExports = exports + exports.live = 100 + exports.stable = 43 + } + + const hook = new Hook(['iitm-turbopack-live'], hookExports) + try { + const initialLive = wrapped.live + wrapped.increment() + return { + initialLive, + live: wrapped.live, + stable: wrapped.stable, + hookedLive: hookedExports.live + } + } finally { + hook.unhook() + } +} diff --git a/test/get-esm-exports/v20-get-esm-exports.mjs b/test/get-esm-exports/v20-get-esm-exports.mjs index d9d2dc15..4c966225 100644 --- a/test/get-esm-exports/v20-get-esm-exports.mjs +++ b/test/get-esm-exports/v20-get-esm-exports.mjs @@ -32,6 +32,21 @@ assert.deepEqual(lexEsm('export const direct = 1; export * from "dependency"', ' starReexports: [{ specifier: 'dependency', parentURL: 'file:///parent.mjs' }], hasModuleSyntax: true }) +assert.deepEqual(lexEsm(` + export let direct = 1 + const local = 2 + export { local as alias } + export { remote as renamed } from './dependency.mjs' + import { imported as detached } from './dependency.mjs' + export { detached } + export * as namespace from './dependency.mjs' +`, 'file:///parent.mjs', true).exportDeclarations, new Map([ + ['direct', { type: 'direct', name: 'direct', localName: 'direct' }], + ['alias', { type: 'direct', name: 'alias', localName: 'local' }], + ['renamed', { type: 'reexport', name: 'renamed', importName: 'remote', specifier: './dependency.mjs' }], + ['detached', { type: 'reexport', name: 'detached', importName: 'imported', specifier: './dependency.mjs' }], + ['namespace', { type: 'reexport', name: 'namespace', specifier: './dependency.mjs' }] +])) // // Generate fixture data // fixture.split('\n').forEach(line => { diff --git a/test/integration-tests/turbopack-build-start.mjs b/test/integration-tests/turbopack-build-start.mjs index 347326a2..19598904 100644 --- a/test/integration-tests/turbopack-build-start.mjs +++ b/test/integration-tests/turbopack-build-start.mjs @@ -1,81 +1,56 @@ -import { spawn, execSync } from 'child_process' -import { existsSync } from 'fs' -import { strictEqual } from 'assert' -import { fileURLToPath } from 'url' -import path from 'path' - -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const appDir = path.resolve(__dirname, '..', 'fixtures', 'test-nextjs-app') -const iitmDir = path.resolve(__dirname, '..', '..') -const hookSetup = path.join(appDir, 'iitm-hook-setup.cjs') -const nextBin = path.join(appDir, 'node_modules', '.bin', 'next') -const PORT = 3100 - -if (!existsSync(path.join(appDir, 'node_modules'))) { +import { deepStrictEqual } from 'node:assert/strict' +import { execFileSync, execSync } from 'node:child_process' +import { existsSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { runTurbopackServer } from './turbopack-server.mjs' +import { prepareTurbopackWrapper, removeTurbopackWrapper } from './turbopack-wrapper.mjs' + +const directory = path.dirname(fileURLToPath(import.meta.url)) +const appDirectory = path.resolve(directory, '..', 'fixtures', 'test-nextjs-app') +const iitmDirectory = path.resolve(directory, '..', '..') +const hookSetup = path.join(appDirectory, 'iitm-hook-setup.cjs') +const nextBin = path.join(appDirectory, 'node_modules', '.bin', 'next') +const port = 3100 + +if (!existsSync(path.join(appDirectory, 'node_modules'))) { console.log('Installing Next.js app dependencies') - execSync('npm install', { cwd: appDir }) + execSync('npm install', { cwd: appDirectory }) } -console.log('Running `next build`') -await new Promise((resolve, reject) => { - const build = spawn(nextBin, ['build'], { - cwd: appDir, - env: { ...process.env, NODE_OPTIONS: '' } - }) - build.on('error', reject) - build.on('close', code => { - if (code === 0) resolve() - else reject(new Error(`next build exited with code ${code}`)) - }) -}) - -const hookTriggered = await new Promise((resolve, reject) => { - console.log(`Starting Next.js server on port ${PORT} via \`next start\``) - const server = spawn(nextBin, ['start', '--port', String(PORT)], { - cwd: appDir, - env: { - ...process.env, - NODE_OPTIONS: `--no-warnings --experimental-loader ${path.join(iitmDir, 'hook.mjs')} --require ${hookSetup}`, - IITM_PATH: path.join(iitmDir, 'index.js') - } +await prepareTurbopackWrapper(appDirectory, 41) +try { + console.log('Running `next build --turbopack`') + execFileSync(nextBin, ['build', '--turbopack'], { + cwd: appDirectory, + env: { ...process.env, NODE_OPTIONS: '' }, + stdio: 'inherit' }) - let output = '' - let hookSeen = false - let requestMade = false - - function onData (chunk) { - const text = chunk.toString() - output += text - - if (!requestMade && /ready/i.test(text)) { - requestMade = true - console.log('Server is ready, hitting /api/foo') - fetch(`http://localhost:${PORT}/api/foo`).catch(() => {}) - } - - if (!hookSeen && output.includes('IITM_HOOK_TRIGGERED:camelcase')) { - hookSeen = true - server.kill() - resolve(true) - } - } + console.log(`Starting Next.js server on port ${port} via \`next start\``) + await runTurbopackServer({ + appDirectory, + arguments: ['start', '--port', String(port)], + hookSetup, + iitmDirectory, + nextBin, + port + }, verifyRoute) +} finally { + await removeTurbopackWrapper(appDirectory) +} - server.stdout.on('data', onData) - server.stderr.on('data', onData) - server.on('error', reject) - server.on('close', () => { - if (!hookSeen) { - reject(new Error(`Hook was not triggered. Output:\n${output}`)) - } +/** + * @param {string} url The application route URL. + * @returns {Promise} + */ +async function verifyRoute (url) { + const response = await fetch(url) + deepStrictEqual(await response.json(), { + initialLive: 41, + live: 42, + stable: 43, + hookedLive: 42 }) - - setTimeout(() => { - if (!hookSeen) { - server.kill() - reject(new Error(`Timed out waiting for hook. Output:\n${output}`)) - } - }, 30_000) -}) - -strictEqual(hookTriggered, true) +} diff --git a/test/integration-tests/turbopack-dev.mjs b/test/integration-tests/turbopack-dev.mjs index ff451cc5..a82bf9e4 100644 --- a/test/integration-tests/turbopack-dev.mjs +++ b/test/integration-tests/turbopack-dev.mjs @@ -1,71 +1,86 @@ -import { spawn, execSync } from 'child_process' -import { existsSync } from 'fs' -import { strictEqual } from 'assert' -import { fileURLToPath } from 'url' -import path from 'path' +import { deepStrictEqual } from 'node:assert/strict' +import { execSync } from 'node:child_process' +import { existsSync } from 'node:fs' +import path from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { fileURLToPath } from 'node:url' -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const appDir = path.resolve(__dirname, '..', 'fixtures', 'test-nextjs-app') -const iitmDir = path.resolve(__dirname, '..', '..') -const hookSetup = path.join(appDir, 'iitm-hook-setup.cjs') -const PORT = 3099 +import { runTurbopackServer } from './turbopack-server.mjs' +import { + prepareTurbopackWrapper, + removeTurbopackWrapper, + updateTurbopackSource +} from './turbopack-wrapper.mjs' -if (!existsSync(path.join(appDir, 'node_modules'))) { +const directory = path.dirname(fileURLToPath(import.meta.url)) +const appDirectory = path.resolve(directory, '..', 'fixtures', 'test-nextjs-app') +const iitmDirectory = path.resolve(directory, '..', '..') +const hookSetup = path.join(appDirectory, 'iitm-hook-setup.cjs') +const nextBin = path.join(appDirectory, 'node_modules', '.bin', 'next') +const port = 3099 + +if (!existsSync(path.join(appDirectory, 'node_modules'))) { console.log('Installing Next.js app dependencies') - execSync('npm install', { cwd: appDir }) + execSync('npm install', { cwd: appDirectory }) } -const hookTriggered = await new Promise((resolve, reject) => { - console.log(`Starting Next.js server on port ${PORT} via \`next dev\``) - const server = spawn( - path.join(appDir, 'node_modules', '.bin', 'next'), - ['dev', '--port', String(PORT)], - { - cwd: appDir, - env: { - ...process.env, - NODE_OPTIONS: `--no-warnings --experimental-loader ${path.join(iitmDir, 'hook.mjs')} --require ${hookSetup}`, - IITM_PATH: path.join(iitmDir, 'index.js') - } - } - ) - - let output = '' - let hookSeen = false - let requestMade = false +await prepareTurbopackWrapper(appDirectory, 41) +try { + console.log(`Starting Next.js server on port ${port} via \`next dev --turbopack\``) + await runTurbopackServer({ + appDirectory, + arguments: ['dev', '--turbopack', '--port', String(port)], + hookSetup, + iitmDirectory, + nextBin, + port + }, verifyRoute) +} finally { + await removeTurbopackWrapper(appDirectory) +} - function onData (chunk) { - const text = chunk.toString() - output += text +/** + * @param {string} url The application route URL. + * @returns {Promise} + */ +async function verifyRoute (url) { + deepStrictEqual(await fetchResult(url), expectedResult(41)) + await updateTurbopackSource(appDirectory, 51) + deepStrictEqual(await waitForRebuild(url), expectedResult(51)) +} - if (!requestMade && /ready/i.test(text)) { - requestMade = true - console.log('Server is ready, hitting /api/foo') - fetch(`http://localhost:${PORT}/api/foo`).catch(() => {}) - } +/** + * @param {string} url The application route URL. + * @returns {Promise>} + */ +async function fetchResult (url) { + const response = await fetch(url) + return response.json() +} - if (!hookSeen && output.includes('IITM_HOOK_TRIGGERED:camelcase')) { - hookSeen = true - server.kill() - resolve(true) - } +/** + * @param {string} url The application route URL. + * @returns {Promise>} + */ +async function waitForRebuild (url) { + let result + for (let attempt = 0; attempt < 100; attempt++) { + await delay(100) + result = await fetchResult(url) + if (result.initialLive === 51) return result } + throw new Error(`Turbopack did not rebuild the wrapper dependency: ${JSON.stringify(result)}`) +} - server.stdout.on('data', onData) - server.stderr.on('data', onData) - server.on('error', reject) - server.on('close', () => { - if (!hookSeen) { - reject(new Error(`Hook was not triggered. Output:\n${output}`)) - } - }) - - setTimeout(() => { - if (!hookSeen) { - server.kill() - reject(new Error(`Timed out waiting for hook. Output:\n${output}`)) - } - }, 30_000) -}) - -strictEqual(hookTriggered, true) +/** + * @param {number} initialLive The dependency's initial live export. + * @returns {{ initialLive: number, live: number, stable: number, hookedLive: number }} + */ +function expectedResult (initialLive) { + return { + initialLive, + live: initialLive + 1, + stable: 43, + hookedLive: initialLive + 1 + } +} diff --git a/test/integration-tests/turbopack-server.mjs b/test/integration-tests/turbopack-server.mjs new file mode 100644 index 00000000..228a4bf2 --- /dev/null +++ b/test/integration-tests/turbopack-server.mjs @@ -0,0 +1,86 @@ +import { spawn } from 'node:child_process' +import { join } from 'node:path' + +/** + * @param {object} options The Next.js server options. + * @param {string} options.appDirectory The Next.js application directory. + * @param {string[]} options.arguments The Next.js command arguments. + * @param {string} options.hookSetup The CommonJS hook setup path. + * @param {string} options.iitmDirectory The IITM package directory. + * @param {string} options.nextBin The Next.js executable path. + * @param {number} options.port The server port. + * @param {(url: string) => Promise} verify Verifies the application route. + * @returns {Promise} + */ +export function runTurbopackServer (options, verify) { + return new Promise((resolve, reject) => { + const server = spawn(options.nextBin, options.arguments, { + cwd: options.appDirectory, + env: { + ...process.env, + NODE_OPTIONS: `--no-warnings --experimental-loader ${join(options.iitmDirectory, 'hook.mjs')} --require ${options.hookSetup}`, + IITM_PATH: join(options.iitmDirectory, 'index.js') + } + }) + + let output = '' + let hookSeen = false + let verificationStarted = false + let verificationComplete = false + let settled = false + + /** @param {Error} [error] The verification failure. */ + function finish (error) { + if (settled || (error === undefined && (!hookSeen || !verificationComplete))) return + settled = true + clearTimeout(timeout) + server.kill() + if (error === undefined) resolve() + else reject(error) + } + + async function verifyRoute () { + try { + await verify(`http://localhost:${options.port}/api/foo`) + verificationComplete = true + finish() + } catch (error) { + finish(error) + } + } + + /** @param {Buffer} chunk The server output chunk. */ + function onData (chunk) { + const text = chunk.toString() + output += text + + if (!verificationStarted && /ready/i.test(text)) { + verificationStarted = true + verifyRoute() + } + if (!hookSeen && output.includes('IITM_HOOK_TRIGGERED:camelcase')) { + hookSeen = true + finish() + } + } + + /** @param {Error} error The child-process failure. */ + function onError (error) { + finish(error) + } + + function onClose () { + if (!settled) finish(new Error(`Turbopack server exited before verification completed. Output:\n${output}`)) + } + + function onTimeout () { + finish(new Error(`Timed out waiting for Turbopack. Output:\n${output}`)) + } + + server.stdout.on('data', onData) + server.stderr.on('data', onData) + server.on('error', onError) + server.on('close', onClose) + const timeout = setTimeout(onTimeout, 30_000) + }) +} diff --git a/test/integration-tests/turbopack-wrapper.mjs b/test/integration-tests/turbopack-wrapper.mjs new file mode 100644 index 00000000..c512fa00 --- /dev/null +++ b/test/integration-tests/turbopack-wrapper.mjs @@ -0,0 +1,114 @@ +import { readFile, rm, writeFile } from 'node:fs/promises' +import { dirname, relative, sep } from 'node:path' +import { pathToFileURL, fileURLToPath } from 'node:url' + +import { createWrapperModule } from '../../bundler.mjs' + +const generatedNames = [ + 'iitm-dependency.mjs', + 'iitm-original.mjs', + 'iitm-wrapper.mjs' +] + +/** + * @param {string} appDirectory The Next.js application directory. + * @param {number} initialValue The dependency's initial live export. + * @returns {Promise>>} + */ +export async function prepareTurbopackWrapper (appDirectory, initialValue) { + const originalPath = `${appDirectory}/iitm-original.mjs` + const wrapperPath = `${appDirectory}/iitm-wrapper.mjs` + const originalSource = `export { live, increment } from './iitm-dependency.mjs' +export const stable = 42 +` + await Promise.all([ + writeFile(originalPath, originalSource), + updateTurbopackSource(appDirectory, initialValue) + ]) + + const wrapper = await createWrapperModule({ + module: { + url: pathToFileURL(originalPath).href, + format: 'module', + source: originalSource, + specifier: 'iitm-turbopack-live', + data: { bundler: 'turbopack' }, + passthroughExports: selectLiveExport + }, + resolve: resolveModule, + load: loadModule + }) + + let code = wrapper.code + for (const entry of wrapper.imports) { + const specifier = relativeImport(wrapperPath, entry.target.url) + code = code.replaceAll(JSON.stringify(entry.specifier), JSON.stringify(specifier)) + } + await writeFile(wrapperPath, code) + return wrapper +} + +/** + * @param {string} appDirectory The Next.js application directory. + * @param {number} initialValue The dependency's initial live export. + * @returns {Promise} + */ +export async function updateTurbopackSource (appDirectory, initialValue) { + const source = `export let live = ${initialValue} +export function increment () { live++ } +` + await writeFile(`${appDirectory}/iitm-dependency.mjs`, source) +} + +/** + * @param {string} appDirectory The Next.js application directory. + * @returns {Promise} + */ +export async function removeTurbopackWrapper (appDirectory) { + await Promise.all(generatedNames.map(name => rm(`${appDirectory}/${name}`, { force: true }))) +} + +/** + * @param {string} specifier The imported module specifier. + * @param {{ parentURL?: string }} context The importing module context. + * @returns {{ url: string, format: 'module', watchFiles: string[] }} + */ +function resolveModule (specifier, context) { + const url = new URL(specifier, context.parentURL).href + return { url, format: 'module', watchFiles: [url] } +} + +/** + * @param {string} url The resolved module URL. + * @returns {Promise<{ source: Buffer, format: 'module', watchFiles: string[] }>} + */ +async function loadModule (url) { + return { + source: await readFile(new URL(url)), + format: 'module', + watchFiles: [url] + } +} + +/** + * @param {string} wrapperPath The generated wrapper path. + * @param {string} targetUrl The manifest target URL. + * @returns {string} + */ +function relativeImport (wrapperPath, targetUrl) { + let specifier = relative(dirname(wrapperPath), fileURLToPath(targetUrl)).split(sep).join('/') + if (!specifier.startsWith('.')) specifier = `./${specifier}` + return specifier +} + +/** + * @param {ReadonlyArray<{ name: string, url: string, localName?: string }>} exports The resolved exports. + * @returns {string[]} + */ +function selectLiveExport (exports) { + const binding = exports.find(({ name }) => name === 'live') + if (binding?.localName !== 'live' || !binding.url.endsWith('/iitm-dependency.mjs')) { + throw new Error('IITM did not resolve the live re-export') + } + return [binding.name] +} diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 7c574025..e83a7f81 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -1,4 +1,5 @@ import { strictEqual, deepStrictEqual, match, doesNotMatch, rejects } from 'assert' +import { spawnSync } from 'child_process' import { readFile, mkdir, mkdtemp, writeFile, rm } from 'fs/promises' import { createRequire } from 'module' import { tmpdir } from 'os' @@ -13,8 +14,9 @@ const { createWrapperModule: createCommonJSWrapperModule, getNodeModuleFormat } = require('../../bundler.js') -const { registerCommonJS, registerWithData } = require('../../lib/bundler-runtime.js') +const { ModuleBinder, registerCommonJS, registerWithData } = require('../../lib/bundler-runtime.js') const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href +const reexportLeafUrl = new URL('../fixtures/reexport-same-source-leaf.mjs', import.meta.url).href const source = await readFile(new URL(moduleUrl), 'utf8') /** @@ -24,6 +26,44 @@ function unexpectedIo () { throw new Error('I/O should not be used when source is provided') } +/** + * @returns {never} + */ +function unexpectedPassthroughSelection () { + throw new Error('Unexpected passthrough export selection') +} + +/** + * @param {ReadonlyArray<{ name: string, url: string, localName?: string }>} exports The resolved exports. + * @returns {string[]} + */ +function selectLiveExport (exports) { + deepStrictEqual(exports, [ + { name: 'live', url: liveModuleUrl, localName: 'live' }, + { name: 'stable', url: liveModuleUrl, localName: 'stable' }, + { name: 'increment', url: liveModuleUrl, localName: 'increment' } + ]) + return ['live'] +} + +/** + * @param {ReadonlyArray<{ name: string, url: string, localName?: string }>} exports The resolved star exports. + * @returns {string[]} + */ +function selectValExport (exports) { + deepStrictEqual(exports, [{ name: 'val', url: reexportLeafUrl, localName: 'val' }]) + return exports.map(({ name }) => name) +} + +/** + * @param {string} name The canonical module URL. + * @param {string} specifier The original import specifier. + * @param {unknown} data Consumer data associated with the module. + */ +function registerModuleWithData (name, specifier, data) { + registerWithData(name, new ModuleBinder({}), specifier, data) +} + const wrapper = await createCommonJSWrapperModule({ module: { url: moduleUrl, @@ -51,6 +91,20 @@ match(wrapper.code, /\nregisterWithData\(/) match(wrapper.code, /\{"version":"1\.0\.0"\}\)/) doesNotMatch(wrapper.code, /from "file:/) +const emptyPassthroughWrapper = await createWrapperModule({ + module: { + url: moduleUrl, + format: 'module', + source, + specifier: './something.mjs', + data: { version: '1.0.0' }, + passthroughExports: [] + }, + resolve: unexpectedIo, + load: unexpectedIo +}) +deepStrictEqual(emptyPassthroughWrapper, wrapper) + const formatDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-format-')) try { const packageJsonUrl = pathToFileURL(join(formatDirectory, 'package.json')).href @@ -110,21 +164,150 @@ try { await rm(temporaryDirectory, { recursive: true, force: true }) } +const liveDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-live-')) +const liveSource = `export let live = 1 +export const stable = 2 +export function increment () { live++ } +` +const liveModuleUrl = `data:text/javascript,${encodeURIComponent(liveSource)}` +const liveWrapper = await createWrapperModule({ + module: { + url: liveModuleUrl, + format: 'module', + source: liveSource, + specifier: 'iitm-live', + data: { live: true }, + passthroughExports: selectLiveExport + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +match(liveWrapper.code, /export \{ live \} from "\.\/__iitm_module_0__\.js"/) +doesNotMatch(liveWrapper.code, / as live/) + +let liveCode = liveWrapper.code +for (const { specifier, target } of liveWrapper.imports) { + liveCode = liveCode.replaceAll(JSON.stringify(specifier), JSON.stringify(target.url)) +} +try { + const liveWrapperUrl = pathToFileURL(join(liveDirectory, 'wrapper.mjs')).href + const liveRunnerUrl = pathToFileURL(join(liveDirectory, 'runner.mjs')).href + await writeFile(new URL(liveWrapperUrl), liveCode) + await writeFile(new URL(liveRunnerUrl), `import Hook from ${JSON.stringify(new URL('../../index.js', import.meta.url).href)} + +let hookedExports +/** @param {Record} exported The wrapper exports. */ +function hookLive (exported) { + hookedExports = exported + exported.live = 99 + exported.stable = 43 +} + +const hook = new Hook(['iitm-live'], hookLive) +const [wrapper, source] = await Promise.all([ + import(${JSON.stringify(liveWrapperUrl)}), + import(${JSON.stringify(liveModuleUrl)}) +]) +const initial = [wrapper.live, wrapper.stable, hookedExports.live] +const sameIncrement = wrapper.increment === source.increment +wrapper.increment() +console.log(JSON.stringify({ + initial, + sameIncrement, + sourceLive: source.live, + wrapperLive: wrapper.live, + hookedLive: hookedExports.live +})) +hook.unhook() +`) + const result = spawnSync(process.execPath, [fileURLToPath(liveRunnerUrl)], { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: '' } + }) + strictEqual(result.status, 0, result.stderr) + deepStrictEqual(JSON.parse(result.stdout), { + initial: [1, 43, 1], + sameIncrement: true, + sourceLive: 2, + wrapperLive: 2, + hookedLive: 2 + }) +} finally { + await rm(liveDirectory, { recursive: true, force: true }) +} + +const staticPassthroughWrapper = await createWrapperModule({ + module: { + url: 'file:///virtual/static-passthrough.mjs', + format: 'module', + source: "export { live } from './unresolved.mjs'", + specifier: './static-passthrough.mjs', + passthroughExports: ['live'] + }, + resolve: unexpectedIo, + load: unexpectedIo +}) +match(staticPassthroughWrapper.code, /export \{ live \} from "\.\/__iitm_module_0__\.js"/) + +await rejects(createWrapperModule({ + module: { + url: 'file:///virtual/passthrough-error.mjs', + format: 'module', + source: 'export const value = 42', + specifier: './passthrough-error.mjs', + passthroughExports: unexpectedPassthroughSelection + }, + resolve: unexpectedIo, + load: unexpectedIo +}), { + message: 'Unexpected passthrough export selection' +}) + +const rebuiltSource = 'export const rebuilt = true' +const rebuiltUrl = `data:text/javascript,${encodeURIComponent(rebuiltSource)}` const rebuilt = await createWrapperModule({ module: { - url: moduleUrl, + url: rebuiltUrl, format: 'module', - source: 'export const rebuilt = true', - specifier: './something.mjs' + source: rebuiltSource, + specifier: 'iitm-rebuilt' }, resolve: unexpectedIo, load: unexpectedIo }) -match(rebuilt.code, /export \{ \$rebuilt as rebuilt \}/) -match(rebuilt.code, /\nregister\(/) -doesNotMatch(rebuilt.code, /registerWithData/) -doesNotMatch(rebuilt.code, /\$foo/) +match(rebuilt.code, /export \{ \$0 as rebuilt \}/) +match(rebuilt.code, /\nregisterWithData\(/) +doesNotMatch(rebuilt.code, /\nregister\(/) +doesNotMatch(rebuilt.code, /"foo"/) + +let rebuiltFormat +/** + * @param {object} exported + * @param {string} name + * @param {string|undefined} baseDir + * @param {unknown} data + * @param {string} format + */ +function captureRebuiltFormat (exported, name, baseDir, data, format) { + strictEqual(exported.rebuilt, true) + strictEqual(name, 'iitm-rebuilt') + strictEqual(baseDir, undefined) + strictEqual(data, undefined) + rebuiltFormat = format +} +const rebuiltHook = new Hook(['iitm-rebuilt'], captureRebuiltFormat) +try { + let rebuiltCode = rebuilt.code + for (const { specifier, target } of rebuilt.imports) { + rebuiltCode = rebuiltCode.replaceAll(JSON.stringify(specifier), JSON.stringify(target.url)) + } + await import(`data:text/javascript,${encodeURIComponent(rebuiltCode)}`) + strictEqual(rebuiltFormat, 'module') +} finally { + rebuiltHook.unhook() +} /** * @param {string} url @@ -155,7 +338,8 @@ const commonJsWrapper = await createWrapperModule({ format: 'commonjs', source: '#!/usr/bin/env node\nmodule.exports = { value: 42 }\nreturn\nmodule.exports.unreachable = true', specifier: './something.js', - data: { version: '1.0.0' } + data: { version: '1.0.0' }, + passthroughExports: unexpectedPassthroughSelection }, resolve: unexpectedIo, load: unexpectedIo @@ -273,7 +457,7 @@ const packageHook = new Hook(['some-external-module'], (exports, name, baseDir, packageBaseDirectory = baseDir deepStrictEqual(data, { version: '2.0.0' }) }) -registerWithData(hookedPackageUrl, {}, {}, {}, 'some-external-module', { version: '2.0.0' }) +registerModuleWithData(hookedPackageUrl, 'some-external-module', { version: '2.0.0' }) strictEqual(packageBaseDirectory, fileURLToPath(new URL('.', hookedPackageUrl)).slice(0, -1)) packageHook.unhook() @@ -282,7 +466,7 @@ let packageInternalName const packageInternalHook = new Hook(['some-external-module'], { internals: true }, (exports, name) => { packageInternalName = name }) -registerWithData(packageInternalUrl, {}, {}, {}, 'some-external-module/sub', undefined) +registerModuleWithData(packageInternalUrl, 'some-external-module/sub', undefined) strictEqual(packageInternalName, join('some-external-module', 'sub.mjs')) packageInternalHook.unhook() @@ -300,15 +484,22 @@ registerCommonJS( './sub', undefined ) -strictEqual(commonJsPackageName, join('some-external-module', 'sub.js')) +strictEqual(commonJsPackageName, undefined) commonJsPackageHook.unhook() +let commonJsInternalName +const commonJsInternalHook = new Hook(['some-external-module'], { internals: true }, (exports, name) => { + commonJsInternalName = name +}) +strictEqual(commonJsInternalName, join('some-external-module', 'sub.js')) +commonJsInternalHook.unhook() + let invalidFileUrlName const invalidFileUrlHook = new Hook((exports, name) => { invalidFileUrlName = name }) invalidFileUrlName = undefined -registerWithData('file://%', {}, {}, {}, 'invalid', undefined) +registerModuleWithData('file://%', 'invalid', undefined) strictEqual(invalidFileUrlName, 'file://%') invalidFileUrlHook.unhook() @@ -329,7 +520,8 @@ const reexportWrapper = await createWrapperModule({ module: { url: reexportUrl, format: 'module', - specifier: './reexport-same-source.mjs' + specifier: './reexport-same-source.mjs', + passthroughExports: selectValExport }, resolve: resolveModule, load: loadModule @@ -343,6 +535,7 @@ strictEqual(reexportWrapper.watchFiles.includes(reexportUrl), true) strictEqual(reexportWrapper.watchFiles.includes(packageUrl), true) strictEqual(reexportWrapper.watchFiles.includes(sourceWatchUrl), true) doesNotMatch(reexportWrapper.code, /from "file:/) +match(reexportWrapper.code, /export \{ val \} from "\.\/__iitm_module_1__\.js"/) /** * @param {string} specifier @@ -366,7 +559,7 @@ const commonJsReexportWrapper = await createWrapperModule({ load: loadModule }) -match(commonJsReexportWrapper.code, /export \{ \$foo as foo \}/) +match(commonJsReexportWrapper.code, /export \{ \$0 as foo \}/) doesNotMatch(commonJsReexportWrapper.code, /as default/) const quotedExportWrapper = await createWrapperModule({ @@ -380,7 +573,36 @@ const quotedExportWrapper = await createWrapperModule({ load: unexpectedIo }) -match(quotedExportWrapper.code, /export \{ \$quoted_name as "quoted name" \}/) +match(quotedExportWrapper.code, /export \{ \$0 as "quoted name" \}/) + +const quotedPassthroughWrapper = await createWrapperModule({ + module: { + url: 'file:///virtual/quoted-passthrough.mjs', + format: 'module', + source: 'const value = 42; export { value as "quoted name" }', + specifier: './quoted-passthrough.mjs', + passthroughExports: ['quoted name', 'missing'] + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +match(quotedPassthroughWrapper.code, /export \{ "quoted name" \} from "\.\/__iitm_module_0__\.js"/) +doesNotMatch(quotedPassthroughWrapper.code, /missing/) + +const defaultPassthroughWrapper = await createWrapperModule({ + module: { + url: 'file:///virtual/default-passthrough.mjs', + format: 'module', + source: 'export default 42', + specifier: './default-passthrough.mjs', + passthroughExports: ['default'] + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +match(defaultPassthroughWrapper.code, /export \{ default \} from "\.\/__iitm_module_0__\.js"/) /** * @param {string} specifier diff --git a/test/low-level/module-binder.mjs b/test/low-level/module-binder.mjs index bff0f3e5..303a604e 100644 --- a/test/low-level/module-binder.mjs +++ b/test/low-level/module-binder.mjs @@ -64,6 +64,25 @@ function interceptTimeouts () { strictEqual(slot.value, 99, 'flush does not clobber an overridden value') } +// Read-only bindings retain the source value and ignore hook writes. +{ + const source = { foo: 42 } + const binder = new ModuleBinder(source, undefined, undefined, undefined, ['foo']) + strictEqual(binder.namespace.foo, 42) + strictEqual(binder.write('foo', 99), true) + strictEqual(binder.namespace.foo, 42) + source.foo = 43 + strictEqual(binder.namespace.foo, 43) +} + +// Read-only bindings can use the defining namespace for star re-exports. +{ + const source = { foo: 1 } + const definingSource = { foo: 2 } + const binder = new ModuleBinder(source, undefined, undefined, undefined, ['foo'], [definingSource]) + strictEqual(binder.namespace.foo, 2) +} + // useFallback reads source.default when the named export is missing. { const source = { default: 7 } diff --git a/test/other/v18-bundlers.mjs b/test/other/v18-bundlers.mjs new file mode 100644 index 00000000..42f0a44e --- /dev/null +++ b/test/other/v18-bundlers.mjs @@ -0,0 +1,230 @@ +import { deepStrictEqual, strictEqual } from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import * as esbuild from 'esbuild' +import webpack from 'webpack' + +import { createWrapperModule } from '../../bundler.mjs' + +const packageRoot = fileURLToPath(new URL('../../', import.meta.url)) +const indexPath = join(packageRoot, 'index.js') +const temporaryDirectory = await realpath(await mkdtemp(join(tmpdir(), 'iitm-bundlers-'))) +const expectedResult = { + esmLive: 42, + esmStable: 43, + hookedLive: 42, + commonjs: 44 +} + +try { + const originalPath = join(temporaryDirectory, 'original.mjs') + const originalCommonJsPath = join(temporaryDirectory, 'original.cjs') + const dependencyPath = join(temporaryDirectory, 'dependency.cjs') + const originalSource = `export let live = 41 +export const stable = 42 +export function increment () { live++ } +` + await Promise.all([ + writeFile(originalPath, originalSource), + writeFile(dependencyPath, 'module.exports = 42\n') + ]) + + const wrappers = new Map([ + ['esm', await createWrapperModule({ + module: { + url: pathToFileURL(originalPath).href, + format: 'module', + source: originalSource, + specifier: 'iitm-virtual-esm', + passthroughExports: ['live'] + }, + resolve: unexpectedIo, + load: unexpectedIo + })], + ['commonjs', await createWrapperModule({ + module: { + url: pathToFileURL(originalCommonJsPath).href, + format: 'commonjs', + source: "module.exports = { value: require('./dependency.cjs') }\n", + specifier: 'iitm-virtual-commonjs' + }, + resolve: unexpectedIo, + load: unexpectedIo + })] + ]) + + const entrySource = ` +const Hook = require(${JSON.stringify(indexPath)}) + +let hookedEsm +/** @param {Record} exports The intercepted exports. */ +function hookExports (exports) { + if ('live' in exports) { + hookedEsm = exports + exports.live = 100 + exports.stable++ + return + } + exports.value += 2 +} + +new Hook(hookExports) + +async function main () { + const [esm, commonjs] = await Promise.all([ + import('iitm-virtual-esm'), + Promise.resolve(require('iitm-virtual-commonjs')) + ]) + esm.increment() + console.log(JSON.stringify({ + esmLive: esm.live, + esmStable: esm.stable, + hookedLive: hookedEsm.live, + commonjs: commonjs.value + })) +} + +main() +` + + await testEsbuild(entrySource, wrappers) + await testWebpack(entrySource, wrappers) +} finally { + await rm(temporaryDirectory, { recursive: true, force: true }) +} + +/** + * @returns {never} + */ +function unexpectedIo () { + throw new Error('Unexpected adapter I/O') +} + +/** + * @param {string} entrySource The application entry source. + * @param {Map>>} wrappers Generated wrappers by module format. + */ +async function testEsbuild (entrySource, wrappers) { + const outfile = join(temporaryDirectory, 'esbuild.cjs') + await esbuild.build({ + bundle: true, + format: 'cjs', + platform: 'node', + outfile, + stdin: { + contents: entrySource, + loader: 'js', + resolveDir: temporaryDirectory + }, + plugins: [{ + name: 'iitm-test-adapter', + setup (build) { + build.onResolve({ filter: /^iitm-virtual-/ }, args => ({ + path: args.path === 'iitm-virtual-esm' ? 'esm' : 'commonjs', + namespace: 'iitm-wrapper' + })) + build.onResolve({ filter: /^\.\/__iitm_/, namespace: 'iitm-wrapper' }, args => { + const wrapper = wrappers.get(args.importer) + const entry = wrapper.imports.find(entry => entry.specifier === args.path) + return { + path: fileURLToPath(entry.target.url), + external: entry.external + } + }) + build.onLoad({ filter: /.*/, namespace: 'iitm-wrapper' }, args => ({ + contents: wrappers.get(args.path).code, + loader: 'js', + resolveDir: temporaryDirectory + })) + } + }] + }) + + deepStrictEqual(runBundle(outfile), expectedResult) +} + +/** + * @param {string} entrySource The application entry source. + * @param {Map>>} wrappers Generated wrappers by module format. + */ +async function testWebpack (entrySource, wrappers) { + const webpackDirectory = join(temporaryDirectory, 'webpack') + const outputDirectory = join(webpackDirectory, 'dist') + const wrappersByContext = new Map() + await mkdir(outputDirectory, { recursive: true }) + + const entryPath = join(webpackDirectory, 'entry.cjs') + await writeFile(entryPath, entrySource) + for (const [name, wrapper] of wrappers) { + const directory = name === 'commonjs' ? temporaryDirectory : join(webpackDirectory, name) + const extension = name === 'commonjs' ? 'cjs' : 'mjs' + const filename = join(directory, `wrapper.${extension}`) + await mkdir(directory, { recursive: true }) + await writeFile(filename, wrapper.code) + wrappersByContext.set(directory, { filename, wrapper }) + } + + const replacement = new webpack.NormalModuleReplacementPlugin( + /^(?:iitm-virtual-|\.\/__iitm_)/, + resource => { + if (resource.request === 'iitm-virtual-esm') { + resource.request = wrappersByContext.get(join(webpackDirectory, 'esm')).filename + return + } + if (resource.request === 'iitm-virtual-commonjs') { + resource.request = wrappersByContext.get(temporaryDirectory).filename + return + } + + const { wrapper } = wrappersByContext.get(resource.context) + const entry = wrapper.imports.find(entry => entry.specifier === resource.request) + resource.request = fileURLToPath(entry.target.url) + } + ) + + const stats = await runWebpack({ + entry: entryPath, + mode: 'development', + target: 'node', + devtool: false, + output: { + path: outputDirectory, + filename: 'bundle.cjs', + chunkFilename: '[name].cjs' + }, + plugins: [replacement] + }) + const errors = stats.toJson({ all: false, errors: true }).errors + deepStrictEqual(errors, []) + deepStrictEqual(runBundle(join(outputDirectory, 'bundle.cjs')), expectedResult) +} + +/** + * @param {import('webpack').Configuration} configuration The webpack configuration. + * @returns {Promise} + */ +function runWebpack (configuration) { + return new Promise((resolve, reject) => { + webpack(configuration, (error, stats) => { + if (error) return reject(error) + resolve(stats) + }) + }) +} + +/** + * @param {string} filename The bundle entry file. + * @returns {{ esmLive: number, esmStable: number, hookedLive: number, commonjs: number }} + */ +function runBundle (filename) { + const result = spawnSync(process.execPath, [filename], { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: '' } + }) + strictEqual(result.status, 0, result.stderr) + return JSON.parse(result.stdout) +} diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts index ea3bc0ee..5093b6eb 100644 --- a/test/typescript/bundler.test.mts +++ b/test/typescript/bundler.test.mts @@ -10,7 +10,8 @@ const wrapper = await createWrapperModule({ format: 'module', source: 'export const value = 42', specifier: './something.mjs', - data: { version: '1.0.0' } + data: { version: '1.0.0' }, + passthroughExports: exports => exports.map(({ name }) => name) }, resolve () { throw new Error('Unexpected resolve') diff --git a/test/typescript/ts-node.test.mts b/test/typescript/ts-node.test.mts index 9789bcce..12e2c45d 100644 --- a/test/typescript/ts-node.test.mts +++ b/test/typescript/ts-node.test.mts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict' -import defaultHook, { Hook, addHook } from '../../index.js' +import defaultHook, { Hook, addHook, removeHook } from '../../index.js' import { sayHi } from '../fixtures/say-hi.mjs' addHook((url, exported) => { @@ -8,12 +8,18 @@ addHook((url, exported) => { } }) -new defaultHook((exported: any, name: string, baseDir: string|void) => { +new defaultHook(() => {}) +new Hook(() => {}) -}); +function checkHookExportTypes () { + const callableHook = new Hook((exported: (value: string) => string) => exported('test')) + callableHook.unhook() -new Hook((exported: any, name: string, baseDir: string|void) => { + const primitiveHook = (url: string, exported: number) => exported + url.length + addHook(primitiveHook) + removeHook(primitiveHook) +} -}); +void checkHookExportTypes assert.equal(sayHi('test'), 'Hooked') From 9065f19d0c877ad9b896b1e8e43222351620c452 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 8 Sep 2026 22:30:50 +0200 Subject: [PATCH 12/18] fix(bundler): preserve virtual and CommonJS module semantics An empty source string was treated as absent, so virtual modules fell through to a file URL read. The CommonJS wrapper also invoked its nested factory without forwarding the outer arguments. --- lib/get-exports.mjs | 2 +- lib/wrapper.mjs | 6 ++++-- test/low-level/bundler.mjs | 29 ++++++++++++++++++++++++++--- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/lib/get-exports.mjs b/lib/get-exports.mjs index 2460acd0..55b1ed65 100644 --- a/lib/get-exports.mjs +++ b/lib/get-exports.mjs @@ -287,7 +287,7 @@ export function * getExports (url, context, includeExportDeclarations = false) { } } - if (!source) { + if (source == null) { if (format === 'builtin') { // Builtins don't give us the source property, so we're stuck // just requiring it to get the exports. diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index 4ac0da10..5a1472fa 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -586,8 +586,10 @@ export function buildCommonJSWrapperSource ({ source = prepareCommonJSSource(source) const parameters = preserveOuterBindings ? '' : 'exports, require, module, __filename, __dirname' - const argumentsList = preserveOuterBindings ? '' : ', module.exports, require, module, __filename, __dirname' - return `(function (${parameters}) {${source}\n}).call(module.exports${argumentsList})\n` + + const invocation = preserveOuterBindings + ? 'apply(module.exports, arguments)' + : 'call(module.exports, module.exports, require, module, __filename, __dirname)' + return `(function (${parameters}) {${source}\n}).${invocation}\n` + `require(${JSON.stringify(runtimeSpecifier)}).registerCommonJS(` + `${JSON.stringify(realUrl)}, module, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)})\n` } diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index e83a7f81..fd321c4c 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -105,6 +105,21 @@ const emptyPassthroughWrapper = await createWrapperModule({ }) deepStrictEqual(emptyPassthroughWrapper, wrapper) +const emptySourceUrl = 'virtual:iitm-empty' +const emptySourceWrapper = await createWrapperModule({ + module: { + url: emptySourceUrl, + format: 'module', + source: '', + specifier: emptySourceUrl + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +strictEqual(emptySourceWrapper.imports[1].target.url, emptySourceUrl) +doesNotMatch(emptySourceWrapper.code, /^export /m) + const formatDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-format-')) try { const packageJsonUrl = pathToFileURL(join(formatDirectory, 'package.json')).href @@ -336,7 +351,10 @@ const commonJsWrapper = await createWrapperModule({ module: { url: commonJsUrl, format: 'commonjs', - source: '#!/usr/bin/env node\nmodule.exports = { value: 42 }\nreturn\nmodule.exports.unreachable = true', + source: '#!/usr/bin/env node\n' + + 'module.exports = { value: 42, argumentsLength: arguments.length }\n' + + 'return\n' + + 'module.exports.unreachable = true', specifier: './something.js', data: { version: '1.0.0' }, passthroughExports: unexpectedPassthroughSelection @@ -428,7 +446,7 @@ const commonJsDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-commonjs-') try { const commonJsFilename = join(commonJsDirectory, 'wrapper.cjs') await writeFile(commonJsFilename, commonJsCode) - deepStrictEqual(require(commonJsFilename), { value: 42, hooked: true }) + deepStrictEqual(require(commonJsFilename), { value: 42, argumentsLength: 5, hooked: true }) strictEqual(unfilteredCalls, 2) } finally { unfilteredHook.unhook() @@ -559,7 +577,12 @@ const commonJsReexportWrapper = await createWrapperModule({ load: loadModule }) -match(commonJsReexportWrapper.code, /export \{ \$0 as foo \}/) +match(commonJsReexportWrapper.code, /export \{ \$0 as foo(?:,| \})/) +if (parseInt(process.versions.node, 10) >= 23) { + match(commonJsReexportWrapper.code, /as "module\.exports"/) +} else { + doesNotMatch(commonJsReexportWrapper.code, /as "module\.exports"/) +} doesNotMatch(commonJsReexportWrapper.code, /as default/) const quotedExportWrapper = await createWrapperModule({ From c21a3702aefac22ec73f5cf93c0e568a57698441 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 8 Sep 2026 22:31:59 +0200 Subject: [PATCH 13/18] docs(bundler): mark integration API experimental --- README.md | 2 ++ bundler.d.mts | 4 ++++ bundler.d.ts | 4 ++++ bundler.js | 8 ++++++++ bundler.mjs | 3 +++ index.d.ts | 2 ++ 6 files changed, 23 insertions(+) diff --git a/README.md b/README.md index de9da28a..5e2da30e 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ node --import=./instrument.mjs ./my-app.mjs ## Bundler integrations +> **Note:** The bundler integration API is experimental. It may change in minor versions. + Bundlers can generate ESM and CommonJS wrappers with `createWrapperModule`: diff --git a/bundler.d.mts b/bundler.d.mts index 4c934036..372ed035 100644 --- a/bundler.d.mts +++ b/bundler.d.mts @@ -73,6 +73,10 @@ export type CreateWrapperModuleOptions = { ) => LoadResult | Promise } +/** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + */ export declare function createWrapperModule( options: CreateWrapperModuleOptions ): Promise diff --git a/bundler.d.ts b/bundler.d.ts index 9ff71c04..61e42f00 100644 --- a/bundler.d.ts +++ b/bundler.d.ts @@ -1,5 +1,9 @@ export * from './bundler.mjs' +/** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + */ export declare function getNodeModuleFormat( url: string, packageJsonUrl?: string, diff --git a/bundler.js b/bundler.js index d4d9cbe4..2126cacb 100644 --- a/bundler.js +++ b/bundler.js @@ -10,6 +10,9 @@ const getNodeModuleFormat = createGetNodeModuleFormat(readFileSync) let createWrapperModuleImplementation /** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + * * @param {Parameters[0]} options */ async function createWrapperModule (options) { @@ -18,4 +21,9 @@ async function createWrapperModule (options) { } exports.createWrapperModule = createWrapperModule + +/** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + */ exports.getNodeModuleFormat = getNodeModuleFormat diff --git a/bundler.mjs b/bundler.mjs index c8074b59..afec7d06 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -57,6 +57,9 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href */ /** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + * * Creates an ESM wrapper without embedding bundler-specific module identifiers. * * @param {object} options diff --git a/index.d.ts b/index.d.ts index 4d04b4d8..5fb1dc8a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -18,6 +18,7 @@ export type Namespace = { [key: string]: any } * starting from the package name. * @param {baseDir} string The absolute path of the module, if not provided in * `name`. + * The `data` and `format` arguments are experimental and may change in minor versions. * @param {data} Data Optional metadata embedded by a bundler. * @param {format} string The intercepted module format, when available. * @return any A value that can will be assigned to `exports.default`. This is @@ -70,6 +71,7 @@ export default Hook * @param {exported} { [string]: any } An object representing the exported * items of a module. * @param {specifier} string The original import or require specifier. + * The `data` and `format` arguments are experimental and may change in minor versions. * @param {data} Data Optional metadata embedded by a bundler. * @param {format} string The intercepted module format, when available. */ From aaaf5116ff1094297050c53d86b5462974a313b9 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 8 Sep 2026 22:32:19 +0200 Subject: [PATCH 14/18] ci: run builds on declared platforms --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e81ced9..26081cc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - run: npm run lint build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: From ca16c48b2db68e6c43e5f8eb73b0cc94164cafcd Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 9 Sep 2026 00:30:01 +0200 Subject: [PATCH 15/18] fix: preserve hooks across loader variants Windows package paths, load-time module formats, and repeated evaluations could bypass hooks, import a wrapper recursively, or replay stale registrations. --- .github/workflows/ci.yml | 35 +++-- bundler.d.mts | 17 ++- create-hook.mjs | 70 ++++++++-- index.js | 16 ++- lib/register.js | 6 +- lib/wrapper.mjs | 36 +++-- .../test-nextjs-app/iitm-turbopack.mjs | 2 +- test/fixtures/test-nextjs-app/next.config.ts | 7 +- test/fixtures/test-nextjs-app/package.json | 1 + test/integration-tests/turbopack-wrapper.mjs | 7 +- test/low-level/bundler.mjs | 78 +++++++++++ .../v22.15-sync-register-hooks-commonjs.mjs | 126 +++++++++++++++++- test/typescript/bundler.test.mts | 9 +- 13 files changed, 357 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26081cc7..8f7c3af7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - run: npm run lint build: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: matrix: @@ -35,11 +35,6 @@ jobs: - 21.x - 22.x - 24.x - os: - - ubuntu-latest - - macos-latest - - windows-latest - steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Use Node.js ${{ matrix.node-version }} @@ -50,19 +45,41 @@ jobs: - run: npm test - name: Rename coverage file run: > - mv coverage/lcov.info coverage/${{ matrix.node-version }}_${{ matrix.os }}_lcov.info + mv coverage/lcov.info coverage/${{ matrix.node-version }}_lcov.info - name: Archive code coverage results if: success() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: coverage_${{ matrix.os }}_${{ matrix.node-version}} + name: coverage_${{ matrix.node-version}} if-no-files-found: ignore - path: coverage/${{ matrix.node-version }}_${{ matrix.os }}_lcov.info + path: coverage/${{ matrix.node-version }}_lcov.info # This will clobber any coverage generated by the previous `npm test`. # We are opting to omit TS coverage and stick to pass or fail only for TS. - run: npm run test:ts + bundlers: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + node-version: + - 18.5.0 + - 24.x + os: + - ubuntu-latest + - macos-latest + - windows-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ matrix.node-version }} + - run: npm install + - run: npx imhotap --files test/low-level/bundler.mjs test/other/v18-bundlers.mjs + coverage: runs-on: ubuntu-latest diff --git a/bundler.d.mts b/bundler.d.mts index 372ed035..6bf73450 100644 --- a/bundler.d.mts +++ b/bundler.d.mts @@ -8,6 +8,15 @@ export type JsonValue = | JsonValue[] | { [key: string]: JsonValue } +export type JsonCompatible = + Value extends boolean | null | number | string + ? Value + : Value extends readonly unknown[] + ? { [Key in keyof Value]: JsonCompatible } + : Value extends object + ? { [Key in keyof Value]: JsonCompatible } + : never + export type WrapperExport = { name: string url: string @@ -18,12 +27,12 @@ export type PassthroughExports = | Iterable | ((exports: readonly WrapperExport[]) => Iterable) -export type BundlerModule = { +export type BundlerModule = { url: string format: string specifier: string source?: WrapperSource - data?: Data + data?: JsonCompatible passthroughExports?: PassthroughExports } @@ -61,7 +70,7 @@ export type WrapperModule = { sideEffects: true } -export type CreateWrapperModuleOptions = { +export type CreateWrapperModuleOptions = { module: BundlerModule resolve: ( specifier: string, @@ -77,6 +86,6 @@ export type CreateWrapperModuleOptions = { * EXPERIMENTAL * This API is experimental and may change in minor versions. */ -export declare function createWrapperModule( +export declare function createWrapperModule( options: CreateWrapperModuleOptions ): Promise diff --git a/create-hook.mjs b/create-hook.mjs index 3dfcae6b..987a0d1b 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -8,7 +8,7 @@ import { builtinModules } from 'module' import { readFileSync } from 'fs' import createGetNodeModuleFormat from './lib/get-node-module-format.js' import { driveSync, driveAsync } from './lib/io.mjs' -import { buildCommonJSWrapperSource, buildWrapperSource, processModule } from './lib/wrapper.mjs' +import { buildCommonJSWrapperSource, buildWrapperSource, processModule, sourceToString } from './lib/wrapper.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' // Re-exported for backwards compatibility: `supportsSyncHooks` now lives in its @@ -22,6 +22,7 @@ const getNodeModuleFormat = createGetNodeModuleFormat(readFileSync) // FIXME: Typescript extensions are added temporarily until we find a better // way of supporting arbitrary extensions const EXTENSION_RE = /\.(js|mjs|cjs|ts|mts|cts)$/ +const DATA_JAVASCRIPT_RE = /^(?:application|text)\/javascript(?:[;,])/i // The `-typescript` formats are listed unconditionally; getExports strips the // types when the runtime supports it and otherwise falls back to onWrapFailure. const HANDLED_FORMATS = new Set([ @@ -32,6 +33,7 @@ const TRACE_WARNINGS = process.execArgv.includes('--trace-warnings') /** @typedef {import('node:module').LoadHookContext} LoadContext */ /** @typedef {import('node:module').LoadFnOutput} LoadResult */ /** @typedef {string | { specifier: string, format: 'module-typescript' | 'commonjs-typescript' }} SpecifierData */ +/** @typedef {{ specifier: string, format?: string, originalUrl: string }} RequireSpecifierData */ function hasIitm (url) { // Fast path: avoid URL parsing on the hot path when there's clearly no iitm. @@ -150,6 +152,16 @@ function addIitm (url) { return urlObj.href } +/** + * @param {string} url + * @returns {boolean} + */ +function isJavaScriptUrl (url) { + const urlObj = new URL(url) + return urlObj.protocol === 'node:' || EXTENSION_RE.test(urlObj.pathname) || + (urlObj.protocol === 'data:' && DATA_JAVASCRIPT_RE.test(urlObj.pathname)) +} + /** * @param {{ url: string }} meta * @param {boolean} [commonjs] Whether to create CommonJS-specific synchronous hooks. @@ -323,6 +335,10 @@ export function createHook (meta, commonjs) { return result } + if (result.format === undefined && !isJavaScriptUrl(result.url)) { + return result + } + // If the file is referencing itself, we need to skip adding the iitm search params if (result.url === parentURL) { return { @@ -396,8 +412,9 @@ export function createHook (meta, commonjs) { } commonJsSpecifiers ??= new Map() - commonJsSpecifiers.set(result.url, { specifier, format }) - return result + const wrapperUrl = format === undefined && isJavaScriptUrl(result.url) ? addIitm(result.url) : result.url + commonJsSpecifiers.set(wrapperUrl, { specifier, format, originalUrl: result.url }) + return wrapperUrl === result.url ? result : { ...result, url: wrapperUrl } } } @@ -509,17 +526,45 @@ export function createHook (meta, commonjs) { /** * @param {string} url + * @param {LoadContext} context * @param {LoadResult} result - * @param {{ specifier: string, format?: string }} specifierData + * @param {RequireSpecifierData} specifierData + * @param {(url: string, context?: Partial) => LoadResult} nextLoad * @returns {LoadResult} */ - let wrapCommonJS + let wrapRequireLoad if (commonjs === true) { - wrapCommonJS = (url, result, specifierData) => { - commonJsSpecifiers.delete(url) + wrapRequireLoad = (url, context, result, specifierData, nextLoad) => { const format = result.format ?? specifierData.format let source = result.source + if (format === 'module' || format === 'module-typescript') { + const processContext = { ...context, format } + /** + * @param {string} loadUrl + * @param {Partial} loadContext + * @returns {LoadResult} + */ + const loadModule = (loadUrl, loadContext) => { + return loadUrl === url ? result : nextLoad(loadUrl, loadContext) + } + try { + const { bindings } = driveSync( + processModule({ srcUrl: url, context: processContext }), + { resolve: cachedResolve, load: loadModule } + ) + return { + ...result, + format: 'module', + source: onWrapSuccess(url, processContext, specifierData.specifier, bindings), + shortCircuit: true + } + } catch (cause) { + onWrapFailure(url, cause) + return result + } + } + if (url.startsWith('node:')) { source = `module.exports = process.getBuiltinModule(${JSON.stringify(url)})\n` } else if ((format === 'commonjs' || format === 'commonjs-typescript') && source == null && @@ -536,7 +581,7 @@ export function createHook (meta, commonjs) { if (format === 'commonjs-typescript') { const stripTypeScriptTypes = process.getBuiltinModule('module').stripTypeScriptTypes if (stripTypeScriptTypes !== undefined) { - source = stripTypeScriptTypes(Buffer.isBuffer(source) ? source.toString('utf8') : source, { mode: 'strip' }) + source = stripTypeScriptTypes(sourceToString(source), { mode: 'strip' }) } } return { @@ -714,20 +759,21 @@ export function createHook (meta, commonjs) { let loadSyncCommonJS if (commonjs === true) { loadSyncCommonJS = (url, context, nextLoad) => { - if (hasIitm(url)) return loadSync(url, context, nextLoad) - const specifierData = commonJsSpecifiers?.get(url) if (specifierData !== undefined) { let result try { - result = nextLoad(url, context) + result = nextLoad(specifierData.originalUrl, context) } catch (error) { commonJsSpecifiers.delete(url) throw error } - return wrapCommonJS(url, result, specifierData) + commonJsSpecifiers.delete(url) + return wrapRequireLoad(specifierData.originalUrl, context, result, specifierData, nextLoad) } + if (hasIitm(url)) return loadSync(url, context, nextLoad) + return loadSync(url, context, nextLoad) } } diff --git a/index.js b/index.js index 3db89eee..ad4ee89c 100644 --- a/index.js +++ b/index.js @@ -37,14 +37,22 @@ function isTurbopackSpecifier (specifier, baseDir) { if (!usingTurbopack) return false const specifierWithoutTurbopackHash = specifier.slice(0, specifier.lastIndexOf('-')) - return baseDir.endsWith(specifierWithoutTurbopackHash) + return baseDir.endsWith(specifierWithoutTurbopackHash.replaceAll('/', path.sep)) +} + +/** + * @param {string} specifier + * @param {string} baseDir + */ +function matchesPackageDirectory (specifier, baseDir) { + return baseDir.endsWith(specifier.replaceAll('/', path.sep)) || isTurbopackSpecifier(specifier, baseDir) } function addHook (hook, extendedHook = hook) { importHooks.push(hook) toHook.forEach(([name, namespace, specifier]) => hook(name, namespace, specifier)) extendedHooks.set(hook, extendedHook) - for (const entry of toHookExtended) { + for (const entry of toHookExtended.values()) { const namespace = entry.module === undefined ? entry.namespace : entry.module.exports const replacement = extendedHook(entry.name, namespace, entry.specifier, entry.data, entry.format) if (entry.module !== undefined && replacement !== undefined) entry.module.exports = replacement @@ -136,7 +144,7 @@ function callExtendedHook (hookFn, modules, internals, name, namespace, specifie } else if (matchArg === name) { if (!baseDir) { result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) - } else if (baseDir.endsWith(specifiers.get(loadUrl)) || isTurbopackSpecifier(specifiers.get(loadUrl), baseDir)) { + } else if (matchesPackageDirectory(specifiers.get(loadUrl), baseDir)) { result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) } else if (internals) { const internalPath = name + path.sep + path.relative(baseDir, filePath) @@ -281,7 +289,7 @@ function Hook (modules, options, hookFn) { if (!baseDir) { // built-in module (or unexpected non file:// name?) callHookFn(hookFn, namespace, name, baseDir) - } else if (baseDir.endsWith(specifiers.get(loadUrl)) || isTurbopackSpecifier(specifiers.get(loadUrl), baseDir)) { + } else if (matchesPackageDirectory(specifiers.get(loadUrl), baseDir)) { // An import of the top-level module (e.g. `import 'ioredis'`). // Note: Slight behaviour difference from RITM. RITM uses // `require.resolve(name)` to see if filename is the module diff --git a/lib/register.js b/lib/register.js index 4fbb81fb..a5ed0de4 100644 --- a/lib/register.js +++ b/lib/register.js @@ -7,7 +7,7 @@ const binders = new WeakMap() const specifiers = new Map() const toHook = [] const extendedHooks = new Map() -const toHookExtended = [] +const toHookExtended = new Map() /** * @typedef {object} HookEntry @@ -84,7 +84,7 @@ function registerWithData (name, binder, specifier, data) { for (const hook of extendedHooks.values()) { hook(name, proxy, specifier, data, 'module') } - toHookExtended.push({ name, namespace: proxy, specifier, data, format: 'module' }) + toHookExtended.set(`module\0${name}`, { name, namespace: proxy, specifier, data, format: 'module' }) } /** @@ -108,7 +108,7 @@ function registerCommonJS (name, module, specifier, data) { const replacement = hook(name, module.exports, specifier, data, 'commonjs') if (replacement !== undefined) module.exports = replacement } - toHookExtended.push(entry) + toHookExtended.set(`commonjs\0${name}`, entry) } // Delays (ms) for re-reading exports that were still in their temporal dead zone diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index 5a1472fa..4fceebfc 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -254,6 +254,7 @@ export function * processModule ({ export function * resolveExportBindings ({ srcUrl, context, exportNames, moduleExportsCache }) { const memo = new Map() const pending = new Set() + const resolutionCache = new Map() const bindings = [] for (const name of exportNames) { const binding = yield * resolveExportBinding({ @@ -262,7 +263,8 @@ export function * resolveExportBindings ({ srcUrl, context, exportNames, moduleE context, moduleExportsCache, memo, - pending + pending, + resolutionCache }) bindings.push(binding ?? { name, url: srcUrl }) } @@ -277,9 +279,10 @@ export function * resolveExportBindings ({ srcUrl, context, exportNames, moduleE * @param {Map} params.moduleExportsCache Parsed module exports. * @param {Map} params.memo Resolved bindings by module and export name. * @param {Set} params.pending Bindings on the active resolution path. + * @param {Map} params.resolutionCache Resolved re-export targets. * @returns {Generator} */ -function * resolveExportBinding ({ srcUrl, name, context, moduleExportsCache, memo, pending }) { +function * resolveExportBinding ({ srcUrl, name, context, moduleExportsCache, memo, pending, resolutionCache }) { const key = `${srcUrl}\0${name}` if (memo.has(key)) { const binding = memo.get(key) @@ -296,7 +299,7 @@ function * resolveExportBinding ({ srcUrl, name, context, moduleExportsCache, me binding = { name, url: srcUrl } if (declaration.localName !== undefined) binding.localName = declaration.localName } else if (declaration?.type === 'reexport' && declaration.specifier !== undefined) { - const target = yield * resolveBindingTarget(declaration.specifier, srcUrl) + const target = yield * resolveBindingTarget(declaration.specifier, srcUrl, resolutionCache) if (declaration.importName === undefined) { binding = { name, url: target.url } } else { @@ -306,7 +309,8 @@ function * resolveExportBinding ({ srcUrl, name, context, moduleExportsCache, me context: { ...context, format: target.format }, moduleExportsCache, memo, - pending + pending, + resolutionCache }) if (imported !== undefined) binding = { ...imported, name } } @@ -317,7 +321,8 @@ function * resolveExportBinding ({ srcUrl, name, context, moduleExportsCache, me starReexports: moduleExports.starReexports, moduleExportsCache, memo, - pending + pending, + resolutionCache }) } else if (hasExportName(moduleExports.exportNames, name)) { binding = { name, url: srcUrl } @@ -338,19 +343,21 @@ function * resolveExportBinding ({ srcUrl, name, context, moduleExportsCache, me * @param {Map} params.moduleExportsCache Parsed module exports. * @param {Map} params.memo Resolved bindings by module and export name. * @param {Set} params.pending Bindings on the active resolution path. + * @param {Map} params.resolutionCache Resolved re-export targets. * @returns {Generator} */ -function * resolveStarExport ({ name, context, starReexports, moduleExportsCache, memo, pending }) { +function * resolveStarExport ({ name, context, starReexports, moduleExportsCache, memo, pending, resolutionCache }) { let binding for (const { specifier, parentURL } of starReexports) { - const target = yield * resolveBindingTarget(specifier, parentURL) + const target = yield * resolveBindingTarget(specifier, parentURL, resolutionCache) const candidate = yield * resolveExportBinding({ srcUrl: target.url, name, context: { ...context, format: target.format }, moduleExportsCache, memo, - pending + pending, + resolutionCache }) if (candidate === undefined) continue if (binding === undefined) { @@ -365,11 +372,18 @@ function * resolveStarExport ({ name, context, starReexports, moduleExportsCache /** * @param {string} specifier The re-exported module specifier. * @param {string} parentURL The declaring module URL. + * @param {Map} resolutionCache Resolved re-export targets. * @returns {Generator} */ -function * resolveBindingTarget (specifier, parentURL) { +function * resolveBindingTarget (specifier, parentURL, resolutionCache) { + const key = `${parentURL}\0${specifier}` + const cached = resolutionCache.get(key) + if (cached !== undefined) return cached + const request = isBareSpecifier(specifier) ? specifier : new URL(specifier, parentURL).href - return yield [RESOLVE, request, { parentURL }] + const target = yield [RESOLVE, request, { parentURL }] + resolutionCache.set(key, target) + return target } /** @@ -547,7 +561,7 @@ export function buildWrapperSourceWithData (options) { * @param {string | ArrayBuffer | ArrayBufferView} source * @returns {string} */ -function sourceToString (source) { +export function sourceToString (source) { if (typeof source === 'string') return source if (Buffer.isBuffer(source)) return source.toString('utf8') if (ArrayBuffer.isView(source)) { diff --git a/test/fixtures/test-nextjs-app/iitm-turbopack.mjs b/test/fixtures/test-nextjs-app/iitm-turbopack.mjs index 84031e20..e976f124 100644 --- a/test/fixtures/test-nextjs-app/iitm-turbopack.mjs +++ b/test/fixtures/test-nextjs-app/iitm-turbopack.mjs @@ -1,4 +1,4 @@ -import Hook from '../../../index.js' +import Hook from 'import-in-the-middle' import * as wrapped from './iitm-wrapper.mjs' /** diff --git a/test/fixtures/test-nextjs-app/next.config.ts b/test/fixtures/test-nextjs-app/next.config.ts index ec446e8f..f5f0fa32 100644 --- a/test/fixtures/test-nextjs-app/next.config.ts +++ b/test/fixtures/test-nextjs-app/next.config.ts @@ -1,7 +1,12 @@ +import path from "node:path"; + import type { NextConfig } from "next"; const nextConfig: NextConfig = { - serverExternalPackages: ['camelcase'] + serverExternalPackages: ['camelcase'], + turbopack: { + root: path.resolve(process.cwd(), '../../..') + } }; export default nextConfig; diff --git a/test/fixtures/test-nextjs-app/package.json b/test/fixtures/test-nextjs-app/package.json index 7cb2acfc..3e014518 100644 --- a/test/fixtures/test-nextjs-app/package.json +++ b/test/fixtures/test-nextjs-app/package.json @@ -9,6 +9,7 @@ }, "dependencies": { "camelcase": "8.0.0", + "import-in-the-middle": "file:../../..", "next": "16.2.1", "react": "19.2.4", "react-dom": "19.2.4" diff --git a/test/integration-tests/turbopack-wrapper.mjs b/test/integration-tests/turbopack-wrapper.mjs index c512fa00..f3596f57 100644 --- a/test/integration-tests/turbopack-wrapper.mjs +++ b/test/integration-tests/turbopack-wrapper.mjs @@ -1,4 +1,5 @@ import { readFile, rm, writeFile } from 'node:fs/promises' +import { createRequire } from 'node:module' import { dirname, relative, sep } from 'node:path' import { pathToFileURL, fileURLToPath } from 'node:url' @@ -16,6 +17,7 @@ const generatedNames = [ * @returns {Promise>>} */ export async function prepareTurbopackWrapper (appDirectory, initialValue) { + const require = createRequire(`${appDirectory}/package.json`) const originalPath = `${appDirectory}/iitm-original.mjs` const wrapperPath = `${appDirectory}/iitm-wrapper.mjs` const originalSource = `export { live, increment } from './iitm-dependency.mjs' @@ -41,7 +43,10 @@ export const stable = 42 let code = wrapper.code for (const entry of wrapper.imports) { - const specifier = relativeImport(wrapperPath, entry.target.url) + const targetUrl = entry.kind === 'runtime' + ? pathToFileURL(require.resolve('import-in-the-middle/lib/bundler-runtime.js')).href + : entry.target.url + const specifier = relativeImport(wrapperPath, targetUrl) code = code.replaceAll(JSON.stringify(entry.specifier), JSON.stringify(specifier)) } await writeFile(wrapperPath, code) diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index fd321c4c..74dbbbb1 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -479,6 +479,18 @@ registerModuleWithData(hookedPackageUrl, 'some-external-module', { version: '2.0 strictEqual(packageBaseDirectory, fileURLToPath(new URL('.', hookedPackageUrl)).slice(0, -1)) packageHook.unhook() +const scopedPackageUrl = new URL( + '../fixtures/node_modules/@scope/some-scoped-module/index.mjs', + import.meta.url +).href +let scopedPackageCalls = 0 +const scopedPackageHook = new Hook(['@scope/some-scoped-module'], () => { + scopedPackageCalls++ +}) +registerModuleWithData(scopedPackageUrl, '@scope/some-scoped-module', undefined) +strictEqual(scopedPackageCalls, 1) +scopedPackageHook.unhook() + const packageInternalUrl = new URL('../fixtures/node_modules/some-external-module/sub.mjs', import.meta.url).href let packageInternalName const packageInternalHook = new Hook(['some-external-module'], { internals: true }, (exports, name) => { @@ -512,6 +524,30 @@ const commonJsInternalHook = new Hook(['some-external-module'], { internals: tru strictEqual(commonJsInternalName, join('some-external-module', 'sub.js')) commonJsInternalHook.unhook() +const reloadFilename = join(fileURLToPath(new URL('../fixtures/', import.meta.url)), 'reload.cjs') +const reloadUrl = pathToFileURL(reloadFilename).href +registerCommonJS(reloadUrl, { exports: { value: 1 } }, 'reload', undefined) +registerCommonJS(reloadUrl, { exports: { value: 2 } }, 'reload', undefined) +registerCommonJS(reloadUrl, { exports: { value: 3 } }, 'reload', undefined) +registerModuleWithData(reloadUrl, 'reload', undefined) +const reloadCalls = [] +/** + * @param {{ value?: number }} exports The latest module exports. + * @param {string} name The canonical module URL. + * @param {string|undefined} baseDir The package directory. + * @param {unknown} data Consumer data associated with the module. + * @param {'module'|'commonjs'} format The module format. + */ +function captureReloadedValue (exports, name, baseDir, data, format) { + reloadCalls.push({ format, value: exports.value }) +} +const reloadHook = new Hook([reloadFilename], captureReloadedValue) +deepStrictEqual(reloadCalls, [ + { format: 'commonjs', value: 3 }, + { format: 'module', value: undefined } +]) +reloadHook.unhook() + let invalidFileUrlName const invalidFileUrlHook = new Hook((exports, name) => { invalidFileUrlName = name @@ -555,6 +591,48 @@ strictEqual(reexportWrapper.watchFiles.includes(sourceWatchUrl), true) doesNotMatch(reexportWrapper.code, /from "file:/) match(reexportWrapper.code, /export \{ val \} from "\.\/__iitm_module_1__\.js"/) +const repeatedReexportUrl = 'file:///virtual/repeated-reexport.mjs' +const repeatedLeafUrl = 'file:///virtual/repeated-reexport-leaf.mjs' +let repeatedResolveCalls = 0 +/** + * @param {string} specifier + * @param {{ parentURL: string }} context + */ +function resolveRepeatedReexport (specifier, context) { + strictEqual(specifier, repeatedLeafUrl) + strictEqual(context.parentURL, repeatedReexportUrl) + repeatedResolveCalls++ + return { url: repeatedLeafUrl, format: 'module' } +} + +/** + * @param {string} url + */ +function loadRepeatedReexport (url) { + strictEqual(url, repeatedLeafUrl) + return { format: 'module', source: 'export const first = 1, second = 2, third = 3' } +} + +/** + * @param {ReadonlyArray<{ name: string }>} exports The resolved exports. + */ +function selectRepeatedExports (exports) { + return exports.map(({ name }) => name) +} + +await createWrapperModule({ + module: { + url: repeatedReexportUrl, + format: 'module', + source: "export { first, second, third } from './repeated-reexport-leaf.mjs'", + specifier: 'repeated-reexport', + passthroughExports: selectRepeatedExports + }, + resolve: resolveRepeatedReexport, + load: loadRepeatedReexport +}) +strictEqual(repeatedResolveCalls, 1) + /** * @param {string} specifier */ diff --git a/test/register/v22.15-sync-register-hooks-commonjs.mjs b/test/register/v22.15-sync-register-hooks-commonjs.mjs index 84ab1ae5..857ff3f4 100644 --- a/test/register/v22.15-sync-register-hooks-commonjs.mjs +++ b/test/register/v22.15-sync-register-hooks-commonjs.mjs @@ -1,6 +1,6 @@ import { deepStrictEqual, match, strictEqual, throws } from 'node:assert/strict' import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { createRequire } from 'node:module' +import * as nodeModule from 'node:module' import { tmpdir } from 'node:os' import { join } from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' @@ -49,7 +49,8 @@ strictEqual(resolveAsRequire(import.meta.url, 'module', { parentURL: import.meta.url }).shortCircuit, true) strictEqual(resolveAsRequire('https://example.com/module').url, 'https://example.com/module') -strictEqual(resolveAsRequire('file:///unknown.wasm').url, 'file:///unknown.wasm') +const unknownWasmUrl = 'file:///unknown.wasm' +strictEqual(resolveAsRequire(unknownWasmUrl).url, unknownWasmUrl) const filteredHook = createHook(hookMeta, true) filteredHook.applyOptions({ include: ['included'] }) @@ -74,7 +75,7 @@ try { const invalidPackageDirectory = fileURLToPath(new URL('.', invalidPackageUrl)) process.getBuiltinModule('fs').mkdirSync(invalidPackageDirectory) writeFileSync(join(invalidPackageDirectory, 'package.json'), '{') - strictEqual(resolveAsRequire(invalidPackageUrl).url, invalidPackageUrl) + strictEqual(resolveAsRequire(invalidPackageUrl).url, `${invalidPackageUrl}?iitm=true`) } finally { rmSync(formatDirectory, { recursive: true, force: true }) } @@ -98,10 +99,25 @@ try { resolveAsRequire(typeScriptUrl, 'commonjs-typescript') const typeScript = lowLevelHook.loadSyncCommonJS(typeScriptUrl, {}, () => ({ format: 'commonjs-typescript', - source: Buffer.from('const value: number = 43; module.exports = value') + source: new TextEncoder().encode('const value: number = 43; module.exports = value') })) match(typeScript.source, /module\.exports = value/) + const loadTimeModuleUrl = 'data:text/javascript,export%20const%20value%20%3D%2042' + const loadTimeResolution = resolveAsRequire(loadTimeModuleUrl) + let loadedUrl + const loadTimeModule = lowLevelHook.loadSyncCommonJS(loadTimeResolution.url, {}, url => { + loadedUrl = url + return { + format: 'module', + source: 'export const value = 42' + } + }) + strictEqual(loadedUrl, loadTimeModuleUrl) + strictEqual(loadTimeModule.format, 'module') + match(loadTimeModule.source, /export \{ \$0 as value \}/) + match(loadTimeModule.source, /\nregister\(/) + const skippedUrl = pathToFileURL(join(fallbackDirectory, 'skipped.cjs')).href resolveAsRequire(skippedUrl, 'builtin') const skipped = { format: 'builtin', source: 'module.exports = 44' } @@ -151,12 +167,110 @@ try { const commonJsUrl = new URL('../fixtures/something.js', import.meta.url) const commonJsTypeScriptUrl = new URL('../fixtures/typescript-cjs-hook.cts', import.meta.url) const esmUrl = new URL('../fixtures/something.mjs', import.meta.url) +const loadTimeSpecifier = 'iitm-load-time-module' +const loadTimeModuleUrl = 'data:text/javascript,export%20const%20value%20%3D%2042' +const applicationJavaScriptUrl = 'data:application/javascript,export%20const%20value%20%3D%2042' +const nativeFormatDirectory = mkdtempSync(join(tmpdir(), 'iitm-native-formats-')) +const loadTimeJsonFilename = join(nativeFormatDirectory, 'load-time.json') +const loadTimeJsonUrl = pathToFileURL(loadTimeJsonFilename).href +const loadTimeWasmSpecifier = 'iitm-load-time-wasm' +const loadTimeWasmFilename = join(nativeFormatDirectory, 'load-time.wasm') +const loadTimeWasmUrl = pathToFileURL(loadTimeWasmFilename).href +writeFileSync(loadTimeJsonFilename, '{"value":42}') +writeFileSync(loadTimeWasmFilename, new Uint8Array([ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7f, + 0x03, 0x02, 0x01, 0x00, + 0x07, 0x0a, 0x01, 0x06, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x00, 0x00, + 0x0a, 0x06, 0x01, 0x04, 0x00, 0x41, 0x2a, 0x0b +])) + +/** + * @param {string} specifier + * @param {object} context + * @param {Function} nextResolve + * @returns {object} + */ +function resolveLoadTimeModule (specifier, context, nextResolve) { + if (specifier === loadTimeSpecifier || specifier === loadTimeModuleUrl) { + return { url: loadTimeModuleUrl, shortCircuit: true } + } + if (specifier === loadTimeWasmSpecifier) { + return { url: loadTimeWasmUrl, shortCircuit: true } + } + return nextResolve(specifier, context) +} + +/** + * @param {string} url + * @param {object} context + * @param {Function} nextLoad + * @returns {object} + */ +function loadLoadTimeModule (url, context, nextLoad) { + if (url === loadTimeModuleUrl) { + return { + format: 'module', + source: 'export const value = 42', + shortCircuit: true + } + } + return nextLoad(url, context) +} + +nodeModule.registerHooks({ resolve: resolveLoadTimeModule, load: loadLoadTimeModule }) register({ commonjs: true, - include: [commonJsUrl.href, commonJsTypeScriptUrl.href, esmUrl.href, 'fs', 'node:test'] + include: [ + commonJsUrl.href, + commonJsTypeScriptUrl.href, + esmUrl.href, + loadTimeSpecifier, + /^data:application\/javascript,/, + loadTimeJsonUrl, + loadTimeWasmSpecifier, + 'fs', + 'node:test' + ] }) -const require = createRequire(import.meta.url) +const require = nodeModule.createRequire(import.meta.url) +try { + deepStrictEqual(require(loadTimeJsonFilename), { value: 42 }) + const wasmNamespace = await import(loadTimeWasmSpecifier) + strictEqual(wasmNamespace.answer(), 42) +} finally { + rmSync(nativeFormatDirectory, { recursive: true, force: true }) +} +let applicationJavaScriptHookCalls = 0 +/** @param {object} exports The wrapped ESM namespace. */ +function patchApplicationJavaScript (exports) { + applicationJavaScriptHookCalls++ + exports.value = 43 +} +const applicationJavaScriptHook = new Hook([applicationJavaScriptUrl], patchApplicationJavaScript) +try { + const namespace = await import(applicationJavaScriptUrl) + strictEqual(namespace.value, 43) + strictEqual(applicationJavaScriptHookCalls, 1) +} finally { + applicationJavaScriptHook.unhook() +} +let loadTimeHookCalls = 0 +/** @param {object} exports The wrapped ESM namespace. */ +function patchLoadTimeModule (exports) { + loadTimeHookCalls++ + exports.value = 43 +} +const loadTimeHook = new Hook([loadTimeSpecifier], patchLoadTimeModule) +try { + const namespace = require(loadTimeSpecifier) + strictEqual(namespace.value, 43) + strictEqual(loadTimeHookCalls, 1) +} finally { + loadTimeHook.unhook() +} + const commonJsFilename = fileURLToPath(commonJsUrl) const commonJsHook = new Hook([commonJsFilename], exports => ({ value: exports(), diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts index 5093b6eb..efa1b526 100644 --- a/test/typescript/bundler.test.mts +++ b/test/typescript/bundler.test.mts @@ -4,13 +4,20 @@ import { getNodeModuleFormat } from '../../bundler.js' import { createWrapperModule } from '../../bundler.mjs' const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href + +interface WrapperData { + version: string + values: readonly ['value'] +} + +const data: WrapperData = { version: '1.0.0', values: ['value'] as const } const wrapper = await createWrapperModule({ module: { url: moduleUrl, format: 'module', source: 'export const value = 42', specifier: './something.mjs', - data: { version: '1.0.0' }, + data, passthroughExports: exports => exports.map(({ name }) => name) }, resolve () { From 6e802f7befa2d39ca95a2497936257ee02cab799 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 9 Sep 2026 09:47:17 +0200 Subject: [PATCH 16/18] fix: preserve bundler hooks across module boundaries Bundler-generated code can execute in a separate module graph without CommonJS filename globals. Some loaders also expose the module format only during load. These paths could miss active hooks, retain failed registrations, or misread valid adapter values. --- bundler.mjs | 14 ++- create-hook.mjs | 62 ++++++++++-- index.js | 2 +- lib/register.js | 10 +- lib/wrapper.mjs | 6 +- test/low-level/bundler.mjs | 85 +++++++++++++++- test/low-level/unknown-extension-format.mjs | 103 ++++++++++++++++++++ test/other/v18-bundlers.mjs | 28 +++++- 8 files changed, 286 insertions(+), 24 deletions(-) create mode 100644 test/low-level/unknown-extension-format.mjs diff --git a/bundler.mjs b/bundler.mjs index afec7d06..3db01439 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -104,7 +104,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } watchFiles.add(url) } if (result.watchFiles !== undefined) { - for (const watchFile of result.watchFiles) { + for (const watchFile of normalizeStringIterable(result.watchFiles)) { watchFiles.add(watchFile) } } @@ -122,7 +122,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } formats.set(result.url, result.format) } if (result.watchFiles !== undefined) { - for (const watchFile of result.watchFiles) { + for (const watchFile of normalizeStringIterable(result.watchFiles)) { watchFiles.add(watchFile) } } @@ -185,7 +185,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } } const passthroughExports = selectedPassthroughExports === undefined ? undefined - : new Set(selectedPassthroughExports) + : new Set(normalizeStringIterable(selectedPassthroughExports)) /** @type {WrapperImport[]} */ const imports = [{ @@ -238,3 +238,11 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } sideEffects: true } } + +/** + * @param {Iterable} values + * @returns {Iterable} + */ +function normalizeStringIterable (values) { + return typeof values === 'string' ? [values] : values +} diff --git a/create-hook.mjs b/create-hook.mjs index 987a0d1b..3ba57e80 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -335,10 +335,6 @@ export function createHook (meta, commonjs) { return result } - if (result.format === undefined && !isJavaScriptUrl(result.url)) { - return result - } - // If the file is referencing itself, we need to skip adding the iitm search params if (result.url === parentURL) { return { @@ -623,10 +619,36 @@ export function createHook (meta, commonjs) { processContext = { ...context, format: specifierData.format } } + let loadModule = parentGetSource + if (processContext.format === undefined && !isJavaScriptUrl(realUrl)) { + let result + try { + result = await parentGetSource(realUrl, processContext) + } catch (cause) { + specifiers.delete(realUrl) + throw cause + } + if (result.format !== undefined && !HANDLED_FORMATS.has(result.format)) { + specifiers.delete(realUrl) + return result + } + if (result.format !== undefined) processContext = { ...processContext, format: result.format } + + /** + * @param {string} loadUrl + * @param {Partial} loadContext + * @returns {LoadResult|Promise} + */ + const loadPreloadedModule = (loadUrl, loadContext) => { + return loadUrl === realUrl ? result : parentGetSource(loadUrl, loadContext) + } + loadModule = loadPreloadedModule + } + try { const { bindings } = await driveAsync( processModule({ srcUrl: realUrl, context: processContext }), - { resolve: cachedResolve, load: parentGetSource } + { resolve: cachedResolve, load: loadModule } ) return { source: onWrapSuccess(realUrl, processContext, originalSpecifier, bindings) } } catch (cause) { @@ -663,10 +685,36 @@ export function createHook (meta, commonjs) { processContext = { ...context, format: specifierData.format } } + let loadModule = nextLoad + if (processContext.format === undefined && !isJavaScriptUrl(realUrl)) { + let result + try { + result = nextLoad(realUrl, processContext) + } catch (cause) { + specifiers.delete(realUrl) + throw cause + } + if (result.format !== undefined && !HANDLED_FORMATS.has(result.format)) { + specifiers.delete(realUrl) + return result + } + if (result.format !== undefined) processContext = { ...processContext, format: result.format } + + /** + * @param {string} loadUrl + * @param {Partial} loadContext + * @returns {LoadResult} + */ + const loadPreloadedModule = (loadUrl, loadContext) => { + return loadUrl === realUrl ? result : nextLoad(loadUrl, loadContext) + } + loadModule = loadPreloadedModule + } + try { const { bindings } = driveSync( processModule({ srcUrl: realUrl, context: processContext }), - { resolve: cachedResolve, load: nextLoad } + { resolve: cachedResolve, load: loadModule } ) return { source: onWrapSuccess(realUrl, processContext, originalSpecifier, bindings) } } catch (cause) { @@ -681,6 +729,7 @@ export function createHook (meta, commonjs) { async function load (url, context, parentLoad) { if (hasIitm(url)) { const result = await getSource(url, context, parentLoad) + if (result?.format && !HANDLED_FORMATS.has(result.format)) return result // If wrapping failed, `getSource()` may have fallen back to `parentLoad`, // which can legally return `source: null` (e.g. for non-JS formats). if (result && typeof result === 'object' && result.source != null) { @@ -722,6 +771,7 @@ export function createHook (meta, commonjs) { function loadSync (url, context, nextLoad) { if (hasIitm(url)) { const result = getSourceSync(url, context, nextLoad) + if (result?.format && !HANDLED_FORMATS.has(result.format)) return result // If wrapping failed, `getSourceSync()` may have fallen back to `nextLoad`, // which can legally return `source: null` (e.g. for non-JS formats). if (result && typeof result === 'object' && result.source != null) { diff --git a/index.js b/index.js index ad4ee89c..088ce765 100644 --- a/index.js +++ b/index.js @@ -144,7 +144,7 @@ function callExtendedHook (hookFn, modules, internals, name, namespace, specifie } else if (matchArg === name) { if (!baseDir) { result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) - } else if (matchesPackageDirectory(specifiers.get(loadUrl), baseDir)) { + } else if (matchesPackageDirectory(specifier, baseDir)) { result = callExtendedHookFn(hookFn, namespace, name, baseDir, data, format) } else if (internals) { const internalPath = name + path.sep + path.relative(baseDir, filePath) diff --git a/lib/register.js b/lib/register.js index a5ed0de4..cde3c3b8 100644 --- a/lib/register.js +++ b/lib/register.js @@ -6,8 +6,12 @@ const importHooks = [] // TODO should this be a Set? const binders = new WeakMap() const specifiers = new Map() const toHook = [] -const extendedHooks = new Map() -const toHookExtended = new Map() +const registrySymbol = Symbol.for('import-in-the-middle:bundler-registry:v1') +const registry = globalThis[registrySymbol] ??= { + extendedHooks: new Map(), + toHookExtended: new Map() +} +const { extendedHooks, toHookExtended } = registry /** * @typedef {object} HookEntry @@ -78,7 +82,6 @@ function register (name, binder, specifier) { */ function registerWithData (name, binder, specifier, data) { const { namespace } = binder - specifiers.set(name, specifier) binders.set(namespace, binder) const proxy = new Proxy(namespace, proxyHandler) for (const hook of extendedHooks.values()) { @@ -95,7 +98,6 @@ function registerWithData (name, binder, specifier, data) { * @returns {void} */ function registerCommonJS (name, module, specifier, data) { - specifiers.set(name, specifier) const entry = { name, namespace: module.exports, diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index 4fceebfc..b2f6ff40 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -600,9 +600,9 @@ export function buildCommonJSWrapperSource ({ source = prepareCommonJSSource(source) const parameters = preserveOuterBindings ? '' : 'exports, require, module, __filename, __dirname' - const invocation = preserveOuterBindings - ? 'apply(module.exports, arguments)' - : 'call(module.exports, module.exports, require, module, __filename, __dirname)' + const invocation = 'call(module.exports, module.exports, require, module, ' + + 'typeof __filename === "undefined" ? undefined : __filename, ' + + 'typeof __dirname === "undefined" ? undefined : __dirname)' return `(function (${parameters}) {${source}\n}).${invocation}\n` + `require(${JSON.stringify(runtimeSpecifier)}).registerCommonJS(` + `${JSON.stringify(realUrl)}, module, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)})\n` diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 74dbbbb1..44b4ddad 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -35,7 +35,7 @@ function unexpectedPassthroughSelection () { /** * @param {ReadonlyArray<{ name: string, url: string, localName?: string }>} exports The resolved exports. - * @returns {string[]} + * @returns {string} */ function selectLiveExport (exports) { deepStrictEqual(exports, [ @@ -43,7 +43,7 @@ function selectLiveExport (exports) { { name: 'stable', url: liveModuleUrl, localName: 'stable' }, { name: 'increment', url: liveModuleUrl, localName: 'increment' } ]) - return ['live'] + return 'live' } /** @@ -258,7 +258,7 @@ const staticPassthroughWrapper = await createWrapperModule({ format: 'module', source: "export { live } from './unresolved.mjs'", specifier: './static-passthrough.mjs', - passthroughExports: ['live'] + passthroughExports: 'live' }, resolve: unexpectedIo, load: unexpectedIo @@ -465,7 +465,7 @@ function resolveModule (specifier, context) { return { url: new URL(specifier, context.parentURL).href, format: 'module', - watchFiles: [packageUrl] + watchFiles: packageUrl } } @@ -517,6 +517,81 @@ registerCommonJS( strictEqual(commonJsPackageName, undefined) commonJsPackageHook.unhook() +registerCommonJS(hookedPackageUrl, { exports: {} }, './sub', undefined) +const sameUrlFormats = [] + +/** + * @param {object} exports The registered exports. + * @param {string} name The package name. + * @param {string|undefined} baseDir The package directory. + * @param {unknown} data Consumer data associated with the module. + * @param {'module'|'commonjs'} format The module format. + */ +function captureSameUrlFormat (exports, name, baseDir, data, format) { + sameUrlFormats.push(format) +} + +const sameUrlHook = new Hook(['some-external-module'], captureSameUrlFormat) +deepStrictEqual(sameUrlFormats, ['module']) +sameUrlHook.unhook() + +const registerPath = require.resolve('../../lib/register.js') +delete require.cache[registerPath] +const bundledRegister = require(registerPath) +let crossCopyCalls = 0 + +/** + * @param {object} exports The module namespace. + * @param {string} name The package name. + * @param {string|undefined} baseDir The package directory. + * @param {unknown} data Consumer data associated with the module. + * @param {'module'|'commonjs'} format The module format. + */ +function captureCrossCopyRegistration (exports, name, baseDir, data, format) { + crossCopyCalls++ + strictEqual(name, 'cross-copy-package') + deepStrictEqual(data, { version: '1.0.0' }) + strictEqual(format, 'module') +} + +const crossCopyHook = new Hook(['cross-copy-package'], captureCrossCopyRegistration) +bundledRegister.registerWithData( + 'file:///tmp/node_modules/cross-copy-package/index.mjs', + new bundledRegister.ModuleBinder({}), + 'cross-copy-package', + { version: '1.0.0' } +) +strictEqual(crossCopyCalls, 1) +crossCopyHook.unhook() + +delete require.cache[registerPath] +const earlyBundledRegister = require(registerPath) +earlyBundledRegister.registerCommonJS( + 'file:///tmp/node_modules/late-cross-copy-package/index.js', + { exports: {} }, + 'late-cross-copy-package', + { version: '2.0.0' } +) +let lateCrossCopyCalls = 0 + +/** + * @param {object} exports The CommonJS exports. + * @param {string} name The package name. + * @param {string|undefined} baseDir The package directory. + * @param {unknown} data Consumer data associated with the module. + * @param {'module'|'commonjs'} format The module format. + */ +function captureLateCrossCopyRegistration (exports, name, baseDir, data, format) { + lateCrossCopyCalls++ + strictEqual(name, 'late-cross-copy-package') + deepStrictEqual(data, { version: '2.0.0' }) + strictEqual(format, 'commonjs') +} + +const lateCrossCopyHook = new Hook(['late-cross-copy-package'], captureLateCrossCopyRegistration) +strictEqual(lateCrossCopyCalls, 1) +lateCrossCopyHook.unhook() + let commonJsInternalName const commonJsInternalHook = new Hook(['some-external-module'], { internals: true }, (exports, name) => { commonJsInternalName = name @@ -565,7 +640,7 @@ async function loadModule (url, context) { return { source: await readFile(new URL(url), 'utf8'), format: context.format, - watchFiles: [sourceWatchUrl] + watchFiles: sourceWatchUrl } } diff --git a/test/low-level/unknown-extension-format.mjs b/test/low-level/unknown-extension-format.mjs new file mode 100644 index 00000000..9c53ea27 --- /dev/null +++ b/test/low-level/unknown-extension-format.mjs @@ -0,0 +1,103 @@ +import { deepStrictEqual, match, rejects, strictEqual, throws } from 'node:assert/strict' + +import { createHook } from '../fixtures/inspectable-create-hook.mjs' + +const parentURL = 'file:///app/entry.mjs' +const moduleURL = 'file:///app/node_modules/example/module.custom' +const moduleSource = 'export const value = 42' + +/** + * @returns {{ url: string }} + */ +function resolveUnknownFormat () { + return { url: moduleURL } +} + +let asyncLoads = 0 + +/** + * @param {string} url + * @returns {Promise<{ format: string, source: string }>} + */ +async function loadUnknownFormat (url) { + strictEqual(url, moduleURL) + asyncLoads++ + return { format: 'module', source: moduleSource } +} + +const asyncHook = createHook(import.meta) +const asyncResolution = await asyncHook.resolve('example', { parentURL }, resolveUnknownFormat) +match(asyncResolution.url, /[?&]iitm=true/) +const asyncResult = await asyncHook.load(asyncResolution.url, {}, loadUnknownFormat) +match(asyncResult.source, /register/) +strictEqual(asyncLoads, 1) +strictEqual(asyncHook.specifiers.size, 0) + +let syncLoads = 0 + +/** + * @param {string} url + * @returns {{ format: string, source: string }} + */ +function loadUnknownFormatSync (url) { + strictEqual(url, moduleURL) + syncLoads++ + return { format: 'module', source: moduleSource } +} + +const syncHook = createHook(import.meta) +const syncResolution = syncHook.resolveSync('example', { parentURL }, resolveUnknownFormat) +match(syncResolution.url, /[?&]iitm=true/) +const syncResult = syncHook.loadSync(syncResolution.url, {}, loadUnknownFormatSync) +match(syncResult.source, /register/) +strictEqual(syncLoads, 1) +strictEqual(syncHook.specifiers.size, 0) + +const unsupportedURL = 'file:///app/node_modules/example/data.custom' +const unsupportedResult = { format: 'json', source: '{}' } + +/** + * @returns {{ url: string }} + */ +function resolveUnsupportedFormat () { + return { url: unsupportedURL } +} + +/** + * @param {string} url + * @returns {Promise<{ format: string, source: string }>} + */ +async function loadUnsupportedFormat (url) { + strictEqual(url, unsupportedURL) + return unsupportedResult +} + +const unsupportedHook = createHook(import.meta) +const unsupportedResolution = await unsupportedHook.resolve('example/data', { parentURL }, resolveUnsupportedFormat) +const unsupported = await unsupportedHook.load(unsupportedResolution.url, {}, loadUnsupportedFormat) +deepStrictEqual(unsupported, unsupportedResult) +strictEqual(unsupportedHook.specifiers.size, 0) + +const asyncFailureHook = createHook(import.meta) +const asyncFailureResolution = await asyncFailureHook.resolve('example', { parentURL }, resolveUnknownFormat) + +async function failAsyncLoad () { + throw new Error('async load failed') +} + +await rejects(asyncFailureHook.load(asyncFailureResolution.url, {}, failAsyncLoad), { + message: 'async load failed' +}) +strictEqual(asyncFailureHook.specifiers.size, 0) + +const syncFailureHook = createHook(import.meta) +const syncFailureResolution = syncFailureHook.resolveSync('example', { parentURL }, resolveUnknownFormat) + +function failSyncLoad () { + throw new Error('sync load failed') +} + +throws(() => syncFailureHook.loadSync(syncFailureResolution.url, {}, failSyncLoad), { + message: 'sync load failed' +}) +strictEqual(syncFailureHook.specifiers.size, 0) diff --git a/test/other/v18-bundlers.mjs b/test/other/v18-bundlers.mjs index 42f0a44e..13a7ccb5 100644 --- a/test/other/v18-bundlers.mjs +++ b/test/other/v18-bundlers.mjs @@ -109,10 +109,34 @@ function unexpectedIo () { * @param {Map>>} wrappers Generated wrappers by module format. */ async function testEsbuild (entrySource, wrappers) { - const outfile = join(temporaryDirectory, 'esbuild.cjs') + await Promise.all([ + buildEsbuildBundle(entrySource, wrappers, 'cjs', false), + buildEsbuildBundle(entrySource, wrappers, 'esm', true) + ]) +} + +/** + * @param {string} entrySource The application entry source. + * @param {Map>>} wrappers Generated wrappers by module format. + * @param {'cjs'|'esm'} format The output module format. + * @param {boolean} minify Whether esbuild minifies the output. + * @returns {Promise} + */ +async function buildEsbuildBundle (entrySource, wrappers, format, minify) { + const extension = format === 'esm' + ? 'mjs' + : 'cjs' + const outfile = join(temporaryDirectory, `esbuild.${extension}`) await esbuild.build({ + banner: format === 'esm' + ? { + js: `import { createRequire } from 'node:module' +const require = createRequire(import.meta.url)` + } + : undefined, bundle: true, - format: 'cjs', + format, + minify, platform: 'node', outfile, stdin: { From fd5e5ff612ff3f562758598667427c9461719fc6 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 9 Sep 2026 15:50:16 +0200 Subject: [PATCH 17/18] feat: expose CommonJS source offsets to bundlers Bundler adapters need the original source location to compose source maps without parsing generated wrapper code. --- README.md | 4 ++++ bundler.d.mts | 1 + bundler.mjs | 6 ++++-- create-hook.mjs | 10 ++++++++-- lib/wrapper.mjs | 2 +- test/low-level/bundler.mjs | 4 +++- .../v22.15-sync-register-hooks-commonjs.mjs | 19 +++++++++++++++---- test/typescript/bundler.test.mts | 1 + 8 files changed, 37 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5e2da30e..231dd418 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,10 @@ the manifest, so filesystem paths, virtual IDs, external modules, and cache invalidation remain owned by the bundler. `watchFiles` are file URLs that the adapter converts to its native watch-dependency format. +CommonJS results also contain `sourceLineOffset`. It specifies the number of +generated lines before the original source. The source starts at column zero, +so an adapter can shift an existing source map without parsing the wrapper. + The runtime import in the manifest must be bundled with the wrapper. Keeping it external can create a second hook registry at runtime. It is CommonJS and must go through the bundler's normal CommonJS transform. diff --git a/bundler.d.mts b/bundler.d.mts index 6bf73450..179cbee6 100644 --- a/bundler.d.mts +++ b/bundler.d.mts @@ -68,6 +68,7 @@ export type WrapperModule = { imports: WrapperImport[] watchFiles: string[] sideEffects: true + sourceLineOffset?: number } export type CreateWrapperModuleOptions = { diff --git a/bundler.mjs b/bundler.mjs index 3db01439..7e229454 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -71,7 +71,8 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * code: string, * imports: WrapperImport[], * watchFiles: string[], - * sideEffects: true + * sideEffects: true, + * sourceLineOffset?: number * }>} */ export async function createWrapperModule ({ module: moduleData, resolve, load }) { @@ -158,7 +159,8 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } external: false }], watchFiles: Array.from(watchFiles), - sideEffects: true + sideEffects: true, + sourceLineOffset: 1 } } diff --git a/create-hook.mjs b/create-hook.mjs index 3ba57e80..fefab8f0 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -8,7 +8,12 @@ import { builtinModules } from 'module' import { readFileSync } from 'fs' import createGetNodeModuleFormat from './lib/get-node-module-format.js' import { driveSync, driveAsync } from './lib/io.mjs' -import { buildCommonJSWrapperSource, buildWrapperSource, processModule, sourceToString } from './lib/wrapper.mjs' +import { + buildCommonJSWrapperSource, + buildWrapperSourceWithData, + processModule, + sourceToString +} from './lib/wrapper.mjs' import { supportsSyncHooks } from './supports-sync-hooks.mjs' // Re-exported for backwards compatibility: `supportsSyncHooks` now lives in its @@ -500,10 +505,11 @@ export function createHook (meta, commonjs) { if (context.format === 'commonjs') { cjsInIitmChain.add(realUrl) } - return buildWrapperSource({ + return buildWrapperSourceWithData({ realUrl, bindings, originalSpecifier, + data: undefined, runtimeSpecifier: iitmURL }) } diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index b2f6ff40..ef64fb6f 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -603,7 +603,7 @@ export function buildCommonJSWrapperSource ({ const invocation = 'call(module.exports, module.exports, require, module, ' + 'typeof __filename === "undefined" ? undefined : __filename, ' + 'typeof __dirname === "undefined" ? undefined : __dirname)' - return `(function (${parameters}) {${source}\n}).${invocation}\n` + + return `(function (${parameters}) {\n${source}\n}).${invocation}\n` + `require(${JSON.stringify(runtimeSpecifier)}).registerCommonJS(` + `${JSON.stringify(realUrl)}, module, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)})\n` } diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 44b4ddad..6e2cfd3c 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -77,6 +77,7 @@ const wrapper = await createCommonJSWrapperModule({ }) strictEqual(wrapper.sideEffects, true) +strictEqual(wrapper.sourceLineOffset, undefined) deepStrictEqual(wrapper.watchFiles, [moduleUrl]) strictEqual(wrapper.imports.length, 2) strictEqual(wrapper.imports[0].specifier, './__iitm_runtime__.js') @@ -365,7 +366,8 @@ const commonJsWrapper = await createWrapperModule({ strictEqual(commonJsWrapper.imports.length, 1) strictEqual(commonJsWrapper.imports[0].kind, 'runtime') -match(commonJsWrapper.code, /^\(function \(\) \{/) +strictEqual(commonJsWrapper.sourceLineOffset, 1) +match(commonJsWrapper.code, /^\(function \(\) \{\n\/\/\/usr\/bin\/env node\n/) match(commonJsWrapper.code, /registerCommonJS/) doesNotMatch(commonJsWrapper.code, /^(?:import|export) /m) doesNotMatch(commonJsWrapper.code, /function \(exports, require, module/) diff --git a/test/register/v22.15-sync-register-hooks-commonjs.mjs b/test/register/v22.15-sync-register-hooks-commonjs.mjs index 857ff3f4..5c9d290e 100644 --- a/test/register/v22.15-sync-register-hooks-commonjs.mjs +++ b/test/register/v22.15-sync-register-hooks-commonjs.mjs @@ -116,7 +116,7 @@ try { strictEqual(loadedUrl, loadTimeModuleUrl) strictEqual(loadTimeModule.format, 'module') match(loadTimeModule.source, /export \{ \$0 as value \}/) - match(loadTimeModule.source, /\nregister\(/) + match(loadTimeModule.source, /\nregisterWithData\(/) const skippedUrl = pathToFileURL(join(fallbackDirectory, 'skipped.cjs')).href resolveAsRequire(skippedUrl, 'builtin') @@ -132,7 +132,7 @@ try { format: 'commonjs', source: 'module.exports = 45' })) - match(chainLoad.source, /\nregister\(/) + match(chainLoad.source, /\nregisterWithData\(/) const child = { url: pathToFileURL(join(fallbackDirectory, 'child.cjs')).href, format: 'commonjs' } strictEqual(chainHook.resolveSyncCommonJS('child', { conditions: ['require'], @@ -294,10 +294,21 @@ strictEqual(require(commonJsTypeScriptFilename).epsilon, 6) commonJsTypeScriptHook.unhook() const esmFilename = fileURLToPath(esmUrl) -const esmHook = new Hook([esmFilename], exports => { +let esmFormat +/** + * @param {object} exports The wrapped ESM namespace. + * @param {string} name The module name. + * @param {string|undefined} baseDir The package directory. + * @param {unknown} data Consumer data associated with the module. + * @param {'module'|'commonjs'|undefined} format The module format. + */ +function patchEsm (exports, name, baseDir, data, format) { exports.foo = 43 -}) + esmFormat = format +} +const esmHook = new Hook([esmFilename], patchEsm) strictEqual(require(esmFilename).foo, 43) +strictEqual(esmFormat, 'module') esmHook.unhook() const marker = Symbol('iitm-commonjs') diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts index efa1b526..57b8d932 100644 --- a/test/typescript/bundler.test.mts +++ b/test/typescript/bundler.test.mts @@ -29,5 +29,6 @@ const wrapper = await createWrapperModule({ }) assert.equal(wrapper.sideEffects, true) +assert.equal(wrapper.sourceLineOffset, undefined) assert.equal(wrapper.imports[0].kind, 'runtime') assert.equal(getNodeModuleFormat(moduleUrl), 'module') From 7d81ae2ffd35f5ba79784178aa525ba8503970fe Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Wed, 9 Sep 2026 19:36:42 +0200 Subject: [PATCH 18/18] feat: add experimental bundler integration API Bundlers need IITM package identity, format detection, and wrapper semantics without copying loader internals. Typeless CommonJS detection preserves module.exports identity before wrapper selection. Bundler metadata lookups stay fresh across watch rebuilds. Loader format detection keeps its process cache to avoid repeated synchronous package reads. --- README.md | 35 ++++-- bundler.d.mts | 68 ++++++++++-- bundler.js | 10 +- bundler.mjs | 98 +++++++++++------ create-hook.mjs | 27 +++-- lib/bundler-runtime.js | 3 +- lib/get-node-module-format.js | 26 +++-- lib/get-package-details.js | 74 +++++++++++++ lib/wrapper.mjs | 22 +--- test/low-level/bundler.mjs | 102 +++++++++++++++++- .../v22.15-sync-register-hooks-commonjs.mjs | 19 +++- test/typescript/bundler.test.mts | 58 +++++++++- 12 files changed, 448 insertions(+), 94 deletions(-) create mode 100644 lib/get-package-details.js diff --git a/README.md b/README.md index 231dd418..c84af101 100644 --- a/README.md +++ b/README.md @@ -131,18 +131,41 @@ const wrapper = await createWrapperModule({ }) ``` -CommonJS integrations can use the lazy-loading facade. It also exposes the -module format detection used by the Node loader: +`format` is optional. When omitted, IITM detects ESM or CommonJS from the source before it creates the wrapper. -```js -const { createWrapperModule, getNodeModuleFormat } = require('import-in-the-middle/bundler') +CommonJS integrations can use the lazy-loading facade. Both entry points expose +`getPackageDetails`, which finds the nearest named package for a resolved file. +The CommonJS facade also exposes the module format detection used by the Node +loader: -const format = getNodeModuleFormat(url, packageJsonUrl, packageJson.type) +```js +const { + createWrapperModule, + getNodeModuleFormat, + getPackageDetails +} = require('import-in-the-middle/bundler') + +const packageDetails = getPackageDetails(url) +const format = getNodeModuleFormat(url, packageDetails?.packageJsonUrl, packageDetails?.type) ``` +`getPackageDetails` accepts a resolved `file:` URL. It returns the package +`name`, optional `version` and `type`, package and `package.json` URLs, and the +slash-separated module `path`. It walks past unnamed package scopes, which lets +bundlers identify linked workspace packages without relying on a `node_modules` +path. It returns `undefined` when no named package owns the URL. + `url` is the canonical `file:` or `node:` URL reported to hooks. `resolve` and `load` adapt the bundler's resolver and source loader to the same URL-based -module graph. +module graph. `resolve` always receives the declaring module's `parentURL`. +Both callbacks can be omitted when an explicitly formatted CommonJS module +supplies its source. Only `load` is required when that source is omitted. + +`getNodeModuleFormat` returns `undefined` for typeless `.js` and `.ts` files. +The bundler must determine their format from the source. + +The package helpers read current metadata on each call. A bundler can cache +their results for one build and discard that cache before a watch rebuild. The optional `data` value must be JSON-serializable. It is embedded in the wrapper and passed as the fourth argument to `Hook` callbacks, allowing package diff --git a/bundler.d.mts b/bundler.d.mts index 179cbee6..99036e30 100644 --- a/bundler.d.mts +++ b/bundler.d.mts @@ -1,5 +1,14 @@ export type WrapperSource = string | ArrayBuffer | ArrayBufferView +export type PackageDetails = { + name: string + packageJsonUrl: string + packageUrl: string + path: string + type?: string + version?: string +} + export type JsonValue = | boolean | null @@ -29,7 +38,7 @@ export type PassthroughExports = export type BundlerModule = { url: string - format: string + format?: string specifier: string source?: WrapperSource data?: JsonCompatible @@ -41,6 +50,10 @@ export type ModuleContext = { parentURL?: string } +export type ResolveContext = ModuleContext & { + parentURL: string +} + export type ModuleTarget = { url: string format?: string @@ -71,18 +84,47 @@ export type WrapperModule = { sourceLineOffset?: number } -export type CreateWrapperModuleOptions = { +type CommonJSFormat = 'commonjs' | 'commonjs-typescript' + +type Resolve = ( + specifier: string, + context: ResolveContext +) => ResolveResult | Promise + +type Load = ( + url: string, + context: ModuleContext +) => LoadResult | Promise + +type InlineCommonJSOptions = { + module: BundlerModule & { + format: CommonJSFormat + source: WrapperSource + } + resolve?: Resolve + load?: Load +} + +type LoadedCommonJSOptions = { + module: BundlerModule & { + format: CommonJSFormat + source?: WrapperSource + } + resolve?: Resolve + load: Load +} + +type AdapterBackedOptions = { module: BundlerModule - resolve: ( - specifier: string, - context: ModuleContext - ) => ResolveResult | Promise - load: ( - url: string, - context: ModuleContext - ) => LoadResult | Promise + resolve: Resolve + load: Load } +export type CreateWrapperModuleOptions = + | InlineCommonJSOptions + | LoadedCommonJSOptions + | AdapterBackedOptions + /** * EXPERIMENTAL * This API is experimental and may change in minor versions. @@ -90,3 +132,9 @@ export type CreateWrapperModuleOptions = { export declare function createWrapperModule( options: CreateWrapperModuleOptions ): Promise + +/** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + */ +export declare function getPackageDetails(url: string): PackageDetails | undefined diff --git a/bundler.js b/bundler.js index 2126cacb..19929bd1 100644 --- a/bundler.js +++ b/bundler.js @@ -3,8 +3,10 @@ const { readFileSync } = require('node:fs') const createGetNodeModuleFormat = require('./lib/get-node-module-format.js') +const createGetPackageDetails = require('./lib/get-package-details.js') -const getNodeModuleFormat = createGetNodeModuleFormat(readFileSync) +const getNodeModuleFormat = createGetNodeModuleFormat(readFileSync, false) +const getPackageDetails = createGetPackageDetails(readFileSync) /** @type {typeof import('./bundler.mjs').createWrapperModule|undefined} */ let createWrapperModuleImplementation @@ -22,6 +24,12 @@ async function createWrapperModule (options) { exports.createWrapperModule = createWrapperModule +/** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + */ +exports.getPackageDetails = getPackageDetails + /** * EXPERIMENTAL * This API is experimental and may change in minor versions. diff --git a/bundler.mjs b/bundler.mjs index 7e229454..68a2db82 100644 --- a/bundler.mjs +++ b/bundler.mjs @@ -1,7 +1,9 @@ 'use strict' +import { readFileSync } from 'fs' import { builtinModules } from 'module' +import createGetPackageDetails from './lib/get-package-details.js' import { driveAsync } from './lib/io.mjs' import { buildCommonJSWrapperSource, @@ -14,10 +16,16 @@ const RUNTIME_SPECIFIER = './__iitm_runtime__.js' const MODULE_SPECIFIER_PREFIX = './__iitm_module_' const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href +/** + * EXPERIMENTAL + * This API is experimental and may change in minor versions. + */ +export const getPackageDetails = createGetPackageDetails(readFileSync) + /** * @typedef {object} BundlerModule * @property {string} url - * @property {string} format + * @property {string} [format] * @property {string} specifier * @property {string | ArrayBuffer | ArrayBufferView} [source] * @property {unknown} [data] @@ -34,6 +42,12 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * @property {string} [parentURL] */ +/** + * @typedef {object} ResolveContext + * @property {string} [format] + * @property {string} parentURL + */ + /** * @typedef {object} ResolveResult * @property {string} url @@ -64,9 +78,11 @@ const runtimeUrl = new URL('./lib/bundler-runtime.js', import.meta.url).href * * @param {object} options * @param {BundlerModule} options.module - * @param {(specifier: string, context: ModuleContext) => - * (ResolveResult | Promise)} options.resolve - * @param {(url: string, context: ModuleContext) => (LoadResult | Promise)} options.load + * @param {(specifier: string, context: ResolveContext) => + * (ResolveResult | Promise)} [options.resolve] Required unless the module has an explicit CommonJS + * format. + * @param {(url: string, context: ModuleContext) => (LoadResult | Promise)} [options.load] Required unless + * an explicitly formatted CommonJS module supplies its source. * @returns {Promise<{ * code: string, * imports: WrapperImport[], @@ -79,6 +95,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } const context = { format: moduleData.format, cache: false } const watchFiles = new Set() const formats = new Map([[moduleData.url, moduleData.format]]) + let source = moduleData.source if (moduleData.url.startsWith('file:')) { watchFiles.add(moduleData.url) @@ -90,14 +107,17 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } * @returns {Promise} */ const loadModule = async (url, loadContext) => { - if (url === moduleData.url && moduleData.source !== undefined) { + if (url === moduleData.url && source !== undefined) { return { - source: moduleData.source, + source, format: moduleData.format } } const result = await load(url, loadContext) + if (url === moduleData.url && result.source !== undefined) { + source = result.source + } if (result.format !== undefined) { formats.set(url, result.format) } @@ -114,7 +134,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } /** * @param {string} specifier - * @param {ModuleContext} resolveContext + * @param {ResolveContext} resolveContext * @returns {Promise} */ const resolveModule = async (specifier, resolveContext) => { @@ -131,7 +151,6 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } } if (moduleData.format === 'commonjs' || moduleData.format === 'commonjs-typescript') { - let source = moduleData.source if (source === undefined) { const result = await loadModule(moduleData.url, context) source = result.source @@ -140,28 +159,7 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } throw new TypeError(`The bundler load adapter returned no source for '${moduleData.url}'`) } - return { - code: buildCommonJSWrapperSource({ - realUrl: moduleData.url, - source, - originalSpecifier: moduleData.specifier, - data: moduleData.data, - runtimeSpecifier: RUNTIME_SPECIFIER, - preserveOuterBindings: true - }), - imports: [{ - specifier: RUNTIME_SPECIFIER, - kind: 'runtime', - target: { - url: runtimeUrl, - format: 'commonjs' - }, - external: false - }], - watchFiles: Array.from(watchFiles), - sideEffects: true, - sourceLineOffset: 1 - } + return createCommonJSWrapper(moduleData, source, watchFiles) } const io = { resolve: resolveModule, load: loadModule } @@ -174,6 +172,9 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } context, moduleExportsCache }), io) + if (context.format === 'commonjs' || context.format === 'commonjs-typescript') { + return createCommonJSWrapper(moduleData, source, watchFiles) + } let selectedPassthroughExports = moduleData.passthroughExports if (selectPassthroughExports !== undefined) { const exportNames = Array.isArray(bindings) ? bindings.slice() : Array.from(bindings.keys()) @@ -241,6 +242,43 @@ export async function createWrapperModule ({ module: moduleData, resolve, load } } } +/** + * @param {BundlerModule} moduleData + * @param {string | ArrayBuffer | ArrayBufferView} source + * @param {Set} watchFiles + * @returns {{ + * code: string, + * imports: WrapperImport[], + * watchFiles: string[], + * sideEffects: true, + * sourceLineOffset: number + * }} + */ +function createCommonJSWrapper (moduleData, source, watchFiles) { + return { + code: buildCommonJSWrapperSource({ + realUrl: moduleData.url, + source, + originalSpecifier: moduleData.specifier, + data: moduleData.data, + runtimeSpecifier: RUNTIME_SPECIFIER, + preserveOuterBindings: true + }), + imports: [{ + specifier: RUNTIME_SPECIFIER, + kind: 'runtime', + target: { + url: runtimeUrl, + format: 'commonjs' + }, + external: false + }], + watchFiles: Array.from(watchFiles), + sideEffects: true, + sourceLineOffset: 1 + } +} + /** * @param {Iterable} values * @returns {Iterable} diff --git a/create-hook.mjs b/create-hook.mjs index fefab8f0..cd9144a8 100644 --- a/create-hook.mjs +++ b/create-hook.mjs @@ -537,29 +537,40 @@ export function createHook (meta, commonjs) { let wrapRequireLoad if (commonjs === true) { wrapRequireLoad = (url, context, result, specifierData, nextLoad) => { - const format = result.format ?? specifierData.format + let format = result.format ?? specifierData.format let source = result.source - if (format === 'module' || format === 'module-typescript') { + if (format === undefined && source == null && url.startsWith('file:')) { + source = process.getBuiltinModule('fs').readFileSync(fileURLToPath(url)) + } + + if (format === 'module' || format === 'module-typescript' || + (format === undefined && isJavaScriptUrl(url))) { const processContext = { ...context, format } + const loaded = source === result.source ? result : { ...result, source } /** * @param {string} loadUrl * @param {Partial} loadContext * @returns {LoadResult} */ const loadModule = (loadUrl, loadContext) => { - return loadUrl === url ? result : nextLoad(loadUrl, loadContext) + return loadUrl === url ? loaded : nextLoad(loadUrl, loadContext) } try { const { bindings } = driveSync( processModule({ srcUrl: url, context: processContext }), { resolve: cachedResolve, load: loadModule } ) - return { - ...result, - format: 'module', - source: onWrapSuccess(url, processContext, specifierData.specifier, bindings), - shortCircuit: true + if (processContext.format === 'commonjs') { + format = 'commonjs' + cjsInIitmChain.add(url) + } else { + return { + ...result, + format: 'module', + source: onWrapSuccess(url, processContext, specifierData.specifier, bindings), + shortCircuit: true + } } } catch (cause) { onWrapFailure(url, cause) diff --git a/lib/bundler-runtime.js b/lib/bundler-runtime.js index 3ea401aa..25009bff 100644 --- a/lib/bundler-runtime.js +++ b/lib/bundler-runtime.js @@ -1,8 +1,7 @@ 'use strict' -const { ModuleBinder, register, registerCommonJS, registerWithData } = require('./register.js') +const { ModuleBinder, registerCommonJS, registerWithData } = require('./register.js') exports.ModuleBinder = ModuleBinder -exports.register = register exports.registerCommonJS = registerCommonJS exports.registerWithData = registerWithData diff --git a/lib/get-node-module-format.js b/lib/get-node-module-format.js index 65c57b1c..289974a5 100644 --- a/lib/get-node-module-format.js +++ b/lib/get-node-module-format.js @@ -5,19 +5,21 @@ /** * @param {'.js'|'.ts'} extension * @param {string|undefined} type - * @returns {NodeModuleFormat} + * @returns {NodeModuleFormat|undefined} */ function getPackageFormat (extension, type) { + if (type !== 'module' && type !== 'commonjs') return undefined if (type === 'module') return extension === '.ts' ? 'module-typescript' : 'module' return extension === '.ts' ? 'commonjs-typescript' : 'commonjs' } /** * @param {typeof import('node:fs').readFileSync} readFileSync + * @param {boolean} [cachePackageTypes=true] * @returns {(url: string, packageJsonUrl?: string, packageType?: string) => NodeModuleFormat|undefined} */ -module.exports = function createGetNodeModuleFormat (readFileSync) { - let packageTypes +module.exports = function createGetNodeModuleFormat (readFileSync, cachePackageTypes = true) { + const packageTypes = cachePackageTypes ? new Map() : undefined /** * @param {string} url @@ -43,35 +45,37 @@ module.exports = function createGetNodeModuleFormat (readFileSync) { if (extension === '.mts') return 'module-typescript' if (extension === '.cts') return 'commonjs-typescript' - packageTypes ??= new Map() const packageDirectory = packageJsonUrl === undefined ? undefined : new URL('.', packageJsonUrl).href - const visited = [] + const visited = packageTypes === undefined ? undefined : [] let directory = new URL('.', url) while (true) { if (directory.href === packageDirectory) { return getPackageFormat(extension, packageType) } - if (packageDirectory === undefined && packageTypes.has(directory.href)) { + if (packageTypes?.has(directory.href)) { const type = packageTypes.get(directory.href) for (const href of visited) packageTypes.set(href, type) return getPackageFormat(extension, type) } - visited.push(directory.href) + visited?.push(directory.href) try { const source = readFileSync(new URL('package.json', directory), 'utf8') const type = JSON.parse(source).type - packageTypes.set(directory.href, type) - if (packageDirectory !== undefined) return getPackageFormat(extension, type) - continue + if (packageTypes !== undefined) { + for (const href of visited) packageTypes.set(href, type) + } + return getPackageFormat(extension, type) } catch (error) { if (error.code !== 'ENOENT') return undefined } const parent = new URL('../', directory) if (parent.href === directory.href) { - for (const href of visited) packageTypes.set(href, undefined) + if (packageTypes !== undefined) { + for (const href of visited) packageTypes.set(href, undefined) + } return getPackageFormat(extension, undefined) } directory = parent diff --git a/lib/get-package-details.js b/lib/get-package-details.js new file mode 100644 index 00000000..07386f27 --- /dev/null +++ b/lib/get-package-details.js @@ -0,0 +1,74 @@ +'use strict' + +const path = require('node:path') +const { fileURLToPath } = require('node:url') + +/** + * @typedef {object} Package + * @property {string} name + * @property {string} packageJsonUrl + * @property {string} packageUrl + * @property {string} [type] + * @property {string} [version] + */ + +/** + * @typedef {Package & { path: string }} PackageDetails + */ + +/** + * @param {typeof import('node:fs').readFileSync} readFileSync + * @returns {(url: string) => PackageDetails|undefined} + */ +module.exports = function createGetPackageDetails (readFileSync) { + /** + * @param {string} url + * @returns {PackageDetails|undefined} + */ + return function getPackageDetails (url) { + if (!url.startsWith('file:')) return + + const moduleUrl = new URL(url) + moduleUrl.hash = '' + moduleUrl.search = '' + let directory = new URL('.', moduleUrl) + + while (true) { + const packageJsonUrl = new URL('package.json', directory) + let packageJson + try { + packageJson = JSON.parse(readFileSync(packageJsonUrl, 'utf8')) + } catch (error) { + if (error?.code !== 'ENOENT') throw error + } + + if (typeof packageJson?.name === 'string' && packageJson.name !== '') { + const packageData = Object.freeze({ + name: packageJson.name, + packageJsonUrl: packageJsonUrl.href, + packageUrl: directory.href, + type: typeof packageJson.type === 'string' ? packageJson.type : undefined, + version: typeof packageJson.version === 'string' ? packageJson.version : undefined + }) + return createPackageDetails(moduleUrl, packageData) + } + + const parent = new URL('../', directory) + if (parent.href === directory.href) return + directory = parent + } + } +} + +/** + * @param {URL} moduleUrl + * @param {Package} packageData + * @returns {PackageDetails} + */ +function createPackageDetails (moduleUrl, packageData) { + const modulePath = path.relative(fileURLToPath(packageData.packageUrl), fileURLToPath(moduleUrl)) + return { + ...packageData, + path: modulePath.replaceAll(path.sep, '/') + } +} diff --git a/lib/wrapper.mjs b/lib/wrapper.mjs index ef64fb6f..c44e297d 100644 --- a/lib/wrapper.mjs +++ b/lib/wrapper.mjs @@ -424,8 +424,7 @@ function hasExportName (exportNames, name) { */ /** - * @param {WrapperOptions & { data?: unknown }} options - * @param {boolean} withData Whether to use the extended bundler registry. + * @param {WrapperOptions & { data: unknown }} options * @returns {string} */ function buildESMWrapperSource ({ @@ -436,7 +435,7 @@ function buildESMWrapperSource ({ runtimeSpecifier, mapImport, passthroughExports -}, withData) { +}) { const moduleSpecifier = mapImport?.(realUrl) ?? realUrl // The wrapped module imports its namespace as `namespace`, which serves // every export but the ones a same-origin `export *` collision forced onto @@ -524,11 +523,8 @@ const __binder = new ModuleBinder(namespace, [${bindingNames}], __write${binding : `${bindingSources === undefined ? ', undefined' : ''}, [${passthroughNames}]${passthroughSourceArguments}`}) ` const reexports = exportSpecifiers === '' ? '' : `export { ${exportSpecifiers} }\n` - const registerName = withData ? 'registerWithData' : 'register' - const registrationData = withData ? `, ${JSON.stringify(data)}` : '' - return ` -import { ${registerName}, ModuleBinder } from ${JSON.stringify(runtimeSpecifier)} +import { registerWithData, ModuleBinder } from ${JSON.stringify(runtimeSpecifier)} import * as namespace from ${JSON.stringify(moduleSpecifier)} ${originImports} ${binder} @@ -537,24 +533,16 @@ ${passthroughReexports} __binder.flush() -${registerName}(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}${registrationData}) +registerWithData(${JSON.stringify(realUrl)}, __binder, ${JSON.stringify(originalSpecifier)}, ${JSON.stringify(data)}) ` } -/** - * @param {WrapperOptions} options - * @returns {string} - */ -export function buildWrapperSource (options) { - return buildESMWrapperSource(options, false) -} - /** * @param {WrapperOptions & { data: unknown }} options * @returns {string} */ export function buildWrapperSourceWithData (options) { - return buildESMWrapperSource(options, true) + return buildESMWrapperSource(options) } /** diff --git a/test/low-level/bundler.mjs b/test/low-level/bundler.mjs index 6e2cfd3c..1949ea9e 100644 --- a/test/low-level/bundler.mjs +++ b/test/low-level/bundler.mjs @@ -1,4 +1,4 @@ -import { strictEqual, deepStrictEqual, match, doesNotMatch, rejects } from 'assert' +import { strictEqual, deepStrictEqual, match, doesNotMatch, rejects, throws } from 'assert' import { spawnSync } from 'child_process' import { readFile, mkdir, mkdtemp, writeFile, rm } from 'fs/promises' import { createRequire } from 'module' @@ -7,14 +7,16 @@ import { join } from 'path' import { fileURLToPath, pathToFileURL } from 'url' import Hook from '../../index.js' -import { createWrapperModule } from '../../bundler.mjs' +import { createWrapperModule, getPackageDetails } from '../../bundler.mjs' const require = createRequire(import.meta.url) const { createWrapperModule: createCommonJSWrapperModule, + getPackageDetails: getCommonJSPackageDetails, getNodeModuleFormat } = require('../../bundler.js') const { ModuleBinder, registerCommonJS, registerWithData } = require('../../lib/bundler-runtime.js') +const createGetNodeModuleFormat = require('../../lib/get-node-module-format.js') const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href const reexportLeafUrl = new URL('../fixtures/reexport-same-source-leaf.mjs', import.meta.url).href const source = await readFile(new URL(moduleUrl), 'utf8') @@ -92,6 +94,23 @@ match(wrapper.code, /\nregisterWithData\(/) match(wrapper.code, /\{"version":"1\.0\.0"\}\)/) doesNotMatch(wrapper.code, /from "file:/) +const inferredCommonJsWrapper = await createWrapperModule({ + module: { + url: new URL('../fixtures/typeless-commonjs.js', import.meta.url).href, + source: 'module.exports = class Example {}', + specifier: 'typeless-commonjs', + data: { version: '1.0.0' } + }, + resolve: unexpectedIo, + load: unexpectedIo +}) + +strictEqual(inferredCommonJsWrapper.imports.length, 1) +strictEqual(inferredCommonJsWrapper.imports[0].kind, 'runtime') +strictEqual(inferredCommonJsWrapper.sourceLineOffset, 1) +match(inferredCommonJsWrapper.code, /registerCommonJS\(/) +doesNotMatch(inferredCommonJsWrapper.code, /registerWithData\(/) + const emptyPassthroughWrapper = await createWrapperModule({ module: { url: moduleUrl, @@ -132,6 +151,9 @@ try { const nestedDirectory = join(formatDirectory, 'nested') await mkdir(nestedDirectory) await writeFile(join(nestedDirectory, 'package.json'), '{"type":"commonjs"}') + const typelessDirectory = join(formatDirectory, 'typeless') + await mkdir(typelessDirectory) + await writeFile(join(typelessDirectory, 'package.json'), '{}') strictEqual( getNodeModuleFormat(pathToFileURL(join(nestedDirectory, 'module.js')).href, packageJsonUrl, 'module'), 'commonjs' @@ -142,6 +164,10 @@ try { strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.cjs')).href), 'commonjs') strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.mts')).href), 'module-typescript') strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.cts')).href), 'commonjs-typescript') + strictEqual(getNodeModuleFormat(pathToFileURL(join(typelessDirectory, 'module.js')).href), undefined) + strictEqual(getNodeModuleFormat(pathToFileURL(join(typelessDirectory, 'module.ts')).href), undefined) + await writeFile(join(formatDirectory, 'package.json'), '{"type":"commonjs"}') + strictEqual(getNodeModuleFormat(pathToFileURL(join(formatDirectory, 'module.js')).href), 'commonjs') } finally { await rm(formatDirectory, { recursive: true, force: true }) } @@ -149,6 +175,74 @@ try { strictEqual(getNodeModuleFormat('node:fs'), 'builtin') strictEqual(getNodeModuleFormat(moduleUrl.replace(/\.mjs$/, '.json')), undefined) +let packageJsonReads = 0 +const getCachedNodeModuleFormat = createGetNodeModuleFormat(() => { + packageJsonReads++ + return '{"type":"module"}' +}) +strictEqual(getCachedNodeModuleFormat('file:///cached/one.js'), 'module') +strictEqual(getCachedNodeModuleFormat('file:///cached/two.js'), 'module') +strictEqual(packageJsonReads, 1) + +const packageDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-package-')) +const packageJsonUrl = pathToFileURL(join(packageDirectory, 'package.json')).href +await writeFile(join(packageDirectory, 'package.json'), JSON.stringify({ + name: '@scope/example', + type: 'module', + version: '1.2.3' +})) +await mkdir(join(packageDirectory, 'nested'), { recursive: true }) +await writeFile(join(packageDirectory, 'nested/package.json'), '{"type":"commonjs"}') +const packageModuleUrl = pathToFileURL(join(packageDirectory, 'nested/module.js')).href +const expectedPackageDetails = { + name: '@scope/example', + packageJsonUrl, + packageUrl: new URL('.', packageJsonUrl).href, + path: 'nested/module.js', + type: 'module', + version: '1.2.3' +} + +deepStrictEqual(getPackageDetails(packageModuleUrl), expectedPackageDetails) +deepStrictEqual(getPackageDetails(`${packageModuleUrl}?loader#fragment`), expectedPackageDetails) +deepStrictEqual(getCommonJSPackageDetails(packageModuleUrl), expectedPackageDetails) +await writeFile(join(packageDirectory, 'package.json'), JSON.stringify({ + name: '@scope/example-renamed', + type: 'commonjs', + version: '2.0.0' +})) +const updatedPackageDetails = { + ...expectedPackageDetails, + name: '@scope/example-renamed', + type: 'commonjs', + version: '2.0.0' +} +deepStrictEqual(getPackageDetails(packageModuleUrl), updatedPackageDetails) +deepStrictEqual(getCommonJSPackageDetails(packageModuleUrl), updatedPackageDetails) +strictEqual(getPackageDetails('node:fs'), undefined) +const noPackageUrl = pathToFileURL(join(tmpdir(), 'iitm-no-package/module.js')).href +strictEqual(getPackageDetails(noPackageUrl), undefined) +strictEqual(getPackageDetails(noPackageUrl), undefined) + +const minimalPackageDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-minimal-package-')) +const minimalPackageJsonUrl = pathToFileURL(join(minimalPackageDirectory, 'package.json')).href +await writeFile(join(minimalPackageDirectory, 'package.json'), '{"name":"minimal"}') +deepStrictEqual(getPackageDetails(pathToFileURL(join(minimalPackageDirectory, 'module.js')).href), { + name: 'minimal', + packageJsonUrl: minimalPackageJsonUrl, + packageUrl: new URL('.', minimalPackageJsonUrl).href, + path: 'module.js', + type: undefined, + version: undefined +}) + +const invalidPackageDirectory = await mkdtemp(join(tmpdir(), 'iitm-bundler-invalid-package-')) +await writeFile(join(invalidPackageDirectory, 'package.json'), '{') +throws( + () => getPackageDetails(pathToFileURL(join(invalidPackageDirectory, 'module.js')).href), + SyntaxError +) + /** * @param {object} exported * @param {string} name @@ -359,9 +453,7 @@ const commonJsWrapper = await createWrapperModule({ specifier: './something.js', data: { version: '1.0.0' }, passthroughExports: unexpectedPassthroughSelection - }, - resolve: unexpectedIo, - load: unexpectedIo + } }) strictEqual(commonJsWrapper.imports.length, 1) diff --git a/test/register/v22.15-sync-register-hooks-commonjs.mjs b/test/register/v22.15-sync-register-hooks-commonjs.mjs index 5c9d290e..e251773c 100644 --- a/test/register/v22.15-sync-register-hooks-commonjs.mjs +++ b/test/register/v22.15-sync-register-hooks-commonjs.mjs @@ -80,8 +80,8 @@ try { rmSync(formatDirectory, { recursive: true, force: true }) } -strictEqual(resolveAsRequire('file:///iitm-default/module.ts').url, 'file:///iitm-default/module.ts') -strictEqual(resolveAsRequire('file:///iitm-default/module.js').url, 'file:///iitm-default/module.js') +strictEqual(resolveAsRequire('file:///iitm-default/module.ts').url, 'file:///iitm-default/module.ts?iitm=true') +strictEqual(resolveAsRequire('file:///iitm-default/module.js').url, 'file:///iitm-default/module.js?iitm=true') const fallbackDirectory = mkdtempSync(join(tmpdir(), 'iitm-commonjs-source-')) const fallbackFilename = join(fallbackDirectory, 'module.cjs') @@ -173,10 +173,13 @@ const applicationJavaScriptUrl = 'data:application/javascript,export%20const%20v const nativeFormatDirectory = mkdtempSync(join(tmpdir(), 'iitm-native-formats-')) const loadTimeJsonFilename = join(nativeFormatDirectory, 'load-time.json') const loadTimeJsonUrl = pathToFileURL(loadTimeJsonFilename).href +const transformedCommonJsFilename = join(nativeFormatDirectory, 'transformed.js') +const transformedCommonJsUrl = pathToFileURL(transformedCommonJsFilename).href const loadTimeWasmSpecifier = 'iitm-load-time-wasm' const loadTimeWasmFilename = join(nativeFormatDirectory, 'load-time.wasm') const loadTimeWasmUrl = pathToFileURL(loadTimeWasmFilename).href writeFileSync(loadTimeJsonFilename, '{"value":42}') +writeFileSync(transformedCommonJsFilename, "module.exports = 'disk'\n") writeFileSync(loadTimeWasmFilename, new Uint8Array([ 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x05, 0x01, 0x60, 0x00, 0x01, 0x7f, @@ -195,6 +198,9 @@ function resolveLoadTimeModule (specifier, context, nextResolve) { if (specifier === loadTimeSpecifier || specifier === loadTimeModuleUrl) { return { url: loadTimeModuleUrl, shortCircuit: true } } + if (specifier === transformedCommonJsFilename || specifier === transformedCommonJsUrl) { + return { url: transformedCommonJsUrl, format: undefined, shortCircuit: true } + } if (specifier === loadTimeWasmSpecifier) { return { url: loadTimeWasmUrl, shortCircuit: true } } @@ -215,6 +221,13 @@ function loadLoadTimeModule (url, context, nextLoad) { shortCircuit: true } } + if (url === transformedCommonJsUrl) { + return { + format: undefined, + source: "module.exports = 'loader'\n", + shortCircuit: true + } + } return nextLoad(url, context) } @@ -226,6 +239,7 @@ register({ commonJsTypeScriptUrl.href, esmUrl.href, loadTimeSpecifier, + transformedCommonJsUrl, /^data:application\/javascript,/, loadTimeJsonUrl, loadTimeWasmSpecifier, @@ -237,6 +251,7 @@ register({ const require = nodeModule.createRequire(import.meta.url) try { deepStrictEqual(require(loadTimeJsonFilename), { value: 42 }) + strictEqual(require(transformedCommonJsFilename), 'loader') const wasmNamespace = await import(loadTimeWasmSpecifier) strictEqual(wasmNamespace.answer(), 42) } finally { diff --git a/test/typescript/bundler.test.mts b/test/typescript/bundler.test.mts index 57b8d932..584b531c 100644 --- a/test/typescript/bundler.test.mts +++ b/test/typescript/bundler.test.mts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' -import { getNodeModuleFormat } from '../../bundler.js' -import { createWrapperModule } from '../../bundler.mjs' +import { getNodeModuleFormat, getPackageDetails as getCommonJSPackageDetails } from '../../bundler.js' +import { createWrapperModule, getPackageDetails } from '../../bundler.mjs' const moduleUrl = new URL('../fixtures/something.mjs', import.meta.url).href @@ -32,3 +32,57 @@ assert.equal(wrapper.sideEffects, true) assert.equal(wrapper.sourceLineOffset, undefined) assert.equal(wrapper.imports[0].kind, 'runtime') assert.equal(getNodeModuleFormat(moduleUrl), 'module') +assert.equal(getPackageDetails(moduleUrl)?.path, 'something.mjs') +assert.equal(getCommonJSPackageDetails(moduleUrl)?.name, 'test-fixtures') + +await createWrapperModule({ + module: { + url: moduleUrl, + format: 'commonjs', + source: 'module.exports = 42', + specifier: './something.js' + } +}) + +if (import.meta.url === '') { + await createWrapperModule({ + module: { + url: moduleUrl, + format: 'commonjs', + source: '' as string | undefined, + specifier: './something.js' + }, + load: () => ({ source: 'module.exports = 42' }) + }) + + // @ts-expect-error ESM wrappers require a load adapter. + await createWrapperModule({ + module: { + url: moduleUrl, + format: 'module', + source: 'export const value = 42', + specifier: './something.mjs' + }, + resolve: specifier => ({ url: specifier }) + }) + + // @ts-expect-error ESM wrappers require a resolve adapter. + await createWrapperModule({ + module: { + url: moduleUrl, + format: 'module', + source: 'export const value = 42', + specifier: './something.mjs' + }, + load: () => ({ source: 'export const value = 42' }) + }) + + // @ts-expect-error CommonJS wrappers without inline source require a load adapter. + await createWrapperModule({ + module: { + url: moduleUrl, + format: 'commonjs', + specifier: './something.js' + } + }) +}