Skip to content

Commit 79b01b8

Browse files
james-elicxjames-elicx
authored
fix: dynamically resolve Turbopack external module mappings (#1139)
* fix: dynamically resolve Turbopack external module mappings for workerd * add regression test and fix extra scenario * dynamically extract turbopack property name * move shiki test from app-router to playground16 * fix 'on workerd' wording in changeset * add JSDoc for discoverExternalModuleMappings params * use readdirSync withFileTypes instead of lstatSync * track case keys in a Set and separate indentation from content * fix wording in discoverBareExternalImports comments * expand comment on regex escaping in discoverExternalSubpaths * update lockfile * address feedback * revert back to filePath * use cross platform regex util
1 parent 69b808b commit 79b01b8

7 files changed

Lines changed: 504 additions & 422 deletions

File tree

‎.changeset/five-gifts-hope.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
"@opennextjs/cloudflare": patch
3+
---
4+
5+
Fix Turbopack external module resolution by dynamically discovering external imports at build time.
6+
7+
When packages are listed in `serverExternalPackages`, Turbopack externalizes them via `externalImport()` which uses dynamic `await import(id)`. The bundler (ESBuild) can't statically analyze `import(id)` with a variable, so these modules aren't included in the worker bundle.
8+
9+
This patch:
10+
11+
- Discovers hashed Turbopack external module mappings from `.next/node_modules/` symlinks (e.g. `shiki-43d062b67f27bbdc` → `shiki`)
12+
- Scans traced chunk files for bare external imports (e.g. `externalImport("shiki")`) and subpath imports (e.g. `shiki/engine/javascript`)
13+
- Generates explicit `switch/case` entries so the bundler can statically resolve and include these modules

‎examples/playground16/app/api/shiki/route.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { createHighlighter } from "shiki";
2+
import { createJavaScriptRegexEngine } from "shiki/engine/javascript";
3+
4+
export async function GET() {
5+
const highlighter = await createHighlighter({
6+
themes: ["vitesse-dark"],
7+
langs: ["javascript"],
8+
engine: createJavaScriptRegexEngine(),
9+
});
10+
11+
const html = highlighter.codeToHtml('console.log("hello")', {
12+
lang: "javascript",
13+
theme: "vitesse-dark",
14+
});
15+
16+
return new Response(JSON.stringify({ html }), {
17+
headers: { "content-type": "application/json" },
18+
});
19+
}

‎examples/playground16/e2e/shiki.spec.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { expect, test } from "@playwright/test";
2+
3+
// Regression test for Turbopack external module resolution.
4+
// When shiki is in serverExternalPackages, Turbopack externalizes it via `externalImport()`,
5+
// which does `await import("shiki")` with a dynamic variable. The bundler can't statically
6+
// analyze `import(id)`, so the module isn't included. The patch adds explicit switch cases
7+
// (e.g. `case "shiki": await import("shiki")`) so the bundler can trace them.
8+
// This also covers subpath imports like "shiki/engine/javascript".
9+
test("shiki syntax highlighting via API route", async ({ request }) => {
10+
const response = await request.get("/api/shiki");
11+
expect(response.status()).toEqual(200);
12+
13+
const json = await response.json();
14+
expect(json).toMatchObject({
15+
html: '<pre class="shiki vitesse-dark" style="background-color:#121212;color:#dbd7caee" tabindex="0"><code><span class="line"><span style="color:#BD976A">console</span><span style="color:#666666">.</span><span style="color:#80A665">log</span><span style="color:#666666">(</span><span style="color:#C98A7D77">"</span><span style="color:#C98A7D">hello</span><span style="color:#C98A7D77">"</span><span style="color:#666666">)</span></span></code></pre>',
16+
});
17+
});

‎examples/playground16/next.config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ initOpenNextCloudflareForDev();
55

66
const nextConfig: NextConfig = {
77
typescript: { ignoreBuildErrors: true },
8+
serverExternalPackages: ["shiki"],
89
};
910

1011
export default nextConfig;

‎examples/playground16/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
"dependencies": {
2020
"next": "16.2.3",
2121
"react-dom": "19.2.4",
22-
"react": "19.2.4"
22+
"react": "19.2.4",
23+
"shiki": "^3.22.0"
2324
},
2425
"devDependencies": {
2526
"@opennextjs/cloudflare": "workspace:*",

‎packages/cloudflare/src/cli/build/patches/plugins/turbopack.ts‎

Lines changed: 205 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import fs from "node:fs";
2+
import path from "node:path";
3+
14
import { patchCode } from "@opennextjs/aws/build/patch/astCodePatcher.js";
25
import type { CodePatcher } from "@opennextjs/aws/build/patch/codePatcher.js";
36
import { getCrossPlatformPathRegex } from "@opennextjs/aws/utils/regex.js";
@@ -10,6 +13,204 @@ fix:
1013
requireChunk(chunkPath)
1114
`;
1215

16+
/**
17+
* Discover Turbopack external module mappings by reading symlinks in .next/node_modules/.
18+
*
19+
* Turbopack externalizes packages listed in serverExternalPackages and creates hashed
20+
* identifiers (e.g. "shiki-43d062b67f27bbdc") with symlinks in .next/node_modules/ pointing
21+
* to the real packages (e.g. ../../node_modules/shiki). At runtime, externalImport() does
22+
* `await import("shiki-43d062b67f27bbdc/wasm")` which fails because the bundler can't
23+
* statically analyze those hashed names. This function discovers the mappings so we can
24+
* generate explicit switch cases for the bundler.
25+
*
26+
* @param filePath Absolute path to the Turbopack runtime file being patched
27+
* (e.g. `/abs/path/to/.open-next/server-functions/default/.../.next/server/chunks/ssr/[turbopack]_runtime.js`).
28+
* This must be the actual file in `.open-next/` (not `buildOptions.appBuildOutputPath`)
29+
* because the `.next/node_modules/` symlinks are in the traced copy, not the original.
30+
* @returns A map from hashed identifiers to real package names (e.g. "shiki-43d062b67f27bbdc" -> "shiki").
31+
*/
32+
function discoverExternalModuleMappings(filePath: string): Map<string, string> {
33+
// filePath is like: .../.next/server/chunks/ssr/[turbopack]_runtime.js
34+
// We need: .../.next/node_modules/
35+
const dotNextDir = filePath.replace(/\/server\/chunks\/.*$/, "");
36+
const nodeModulesDir = path.join(dotNextDir, "node_modules");
37+
38+
const mappings = new Map<string, string>();
39+
40+
if (!fs.existsSync(nodeModulesDir)) {
41+
return mappings;
42+
}
43+
44+
for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) {
45+
try {
46+
if (entry.isSymbolicLink()) {
47+
const entryPath = path.join(nodeModulesDir, entry.name);
48+
const target = fs.readlinkSync(entryPath);
49+
// target is like "../../node_modules/shiki" — extract package name
50+
const match = target.match(/node_modules\/(.+)$/);
51+
if (match?.[1]) {
52+
mappings.set(entry.name, match[1]);
53+
}
54+
}
55+
} catch {
56+
// skip entries we can't read
57+
}
58+
}
59+
60+
return mappings;
61+
}
62+
63+
/**
64+
* Build a dynamic inlineExternalImportRule that includes cases for all discovered
65+
* Turbopack external module hashes, mapping them back to their real package names.
66+
*
67+
* We use a switch for exact matches (including bare + subpath cases) and a fallback
68+
* for the default case. Since switch/case can only match exact strings, we enumerate
69+
* known subpaths from the traced files to cover cases like "shiki-hash/wasm".
70+
*/
71+
function buildExternalImportRule(
72+
mappings: Map<string, string>,
73+
tracedFiles: string[],
74+
runtimeCode: string
75+
): string {
76+
// Tracks module names that already have a switch case, to avoid duplicates.
77+
const casedModules = new Set<string>();
78+
const cases: string[] = [];
79+
80+
function addCase(moduleName: string, importPath: string) {
81+
if (!casedModules.has(moduleName)) {
82+
casedModules.add(moduleName);
83+
cases.push(`case "${moduleName}":\n $RAW = await import("${importPath}");\n break;`);
84+
}
85+
}
86+
87+
// Always include the @vercel/og rewrite
88+
addCase("next/dist/compiled/@vercel/og/index.node.js", "next/dist/compiled/@vercel/og/index.edge.js");
89+
90+
// Add case for each discovered external module mapping (bare import)
91+
for (const [hashedName, realName] of mappings) {
92+
addCase(hashedName, realName);
93+
}
94+
95+
// Discover subpath imports from the traced chunk files.
96+
// Chunks reference external modules like "shiki-hash/wasm" — scan for these patterns.
97+
const subpathCases = discoverExternalSubpaths(mappings, tracedFiles);
98+
for (const [hashedSubpath, realSubpath] of subpathCases) {
99+
addCase(hashedSubpath, realSubpath);
100+
}
101+
102+
// Discover bare external imports from chunk files (e.g. externalImport("shiki")).
103+
// These need explicit switch cases so the bundler can statically resolve them.
104+
const bareImports = discoverBareExternalImports(tracedFiles, runtimeCode);
105+
for (const [moduleName, realName] of bareImports) {
106+
addCase(moduleName, realName);
107+
}
108+
109+
// Indent each case line by 4 spaces to align with the switch body in the YAML fix block.
110+
const indentedCases = cases
111+
.flatMap((c) => c.split("\n"))
112+
.map((line) => ` ${line}`)
113+
.join("\n");
114+
115+
return `
116+
rule:
117+
pattern: "$RAW = await import($ID)"
118+
inside:
119+
regex: "externalImport"
120+
kind: function_declaration
121+
stopBy: end
122+
fix: |-
123+
switch ($ID) {
124+
${indentedCases}
125+
default:
126+
$RAW = await import($ID);
127+
}
128+
`;
129+
}
130+
131+
/**
132+
* Scan traced chunk files for bare external module imports (e.g. `externalImport("shiki")`).
133+
*
134+
* In some Turbopack versions, externalized packages are referenced by their real names
135+
* (not hashed). The default `await import(id)` with a variable `id` can't be statically
136+
* analyzed by the bundler (ESBuild). By adding explicit switch cases with string literals,
137+
* we make these imports statically discoverable so they get bundled into the worker.
138+
*/
139+
function discoverBareExternalImports(tracedFiles: string[], runtimeCode: string): Map<string, string> {
140+
const bareImports = new Map<string, string>();
141+
142+
// Turbopack assigns `externalImport` to a property on the context prototype,
143+
// e.g. `contextPrototype.y = externalImport`. The property name could change between versions,
144+
// so we extract it dynamically from the runtime code rather than hardcoding it.
145+
const propMatch = runtimeCode.match(/contextPrototype\.(\w+)\s*=\s*externalImport/);
146+
if (!propMatch?.[1]) {
147+
return bareImports;
148+
}
149+
const externalImportAlias = propMatch[1];
150+
151+
// Chunks call externalImport as e.g. `.y("shiki")` — build a regex using the discovered property name.
152+
const externalImportRegexp = new RegExp(`\\.${externalImportAlias}\\("([^"]+)"\\)`, "g");
153+
154+
const chunkFiles = tracedFiles.filter((f) => f.includes(".next/server/chunks/"));
155+
156+
for (const filePath of chunkFiles) {
157+
try {
158+
const content = fs.readFileSync(filePath, "utf-8");
159+
for (const externalImportMatch of content.matchAll(externalImportRegexp)) {
160+
const moduleName = externalImportMatch[1];
161+
if (moduleName) {
162+
// Identity mapping — the module name is already the real name
163+
bareImports.set(moduleName, moduleName);
164+
}
165+
}
166+
} catch {
167+
// skip files we can't read
168+
}
169+
}
170+
171+
return bareImports;
172+
}
173+
174+
/**
175+
* Scan traced chunk files for external module subpath imports.
176+
* E.g. find "shiki-43d062b67f27bbdc/wasm" in chunk code and map it to "shiki/wasm".
177+
*
178+
* Only scans files with "[externals]" in the name since those are the chunks that
179+
* contain externalImport calls.
180+
*/
181+
function discoverExternalSubpaths(mappings: Map<string, string>, tracedFiles: string[]): Map<string, string> {
182+
const subpaths = new Map<string, string>();
183+
184+
const externalChunks = tracedFiles.filter((f) => f.includes("[externals]"));
185+
186+
for (const [hashedName, realName] of mappings) {
187+
// Build a regex to find quoted subpath imports of this hashed module in chunk source code.
188+
// E.g. for hashedName "shiki-43d062b67f27bbdc", this matches strings like
189+
// "shiki-43d062b67f27bbdc/wasm" or "shiki-43d062b67f27bbdc/engine/javascript".
190+
// The hashedName is escaped to safely use it as a literal in the regex pattern.
191+
const escaped = getCrossPlatformPathRegex(hashedName, { escape: true });
192+
const pattern = new RegExp(`"(${escaped}/[^"]*)"`, "g");
193+
194+
for (const filePath of externalChunks) {
195+
try {
196+
const content = fs.readFileSync(filePath, "utf-8");
197+
for (const match of content.matchAll(pattern)) {
198+
const fullHashedPath = match[1];
199+
if (fullHashedPath) {
200+
const subpath = fullHashedPath.slice(hashedName.length);
201+
const realSubpath = realName + subpath;
202+
subpaths.set(fullHashedPath, realSubpath);
203+
}
204+
}
205+
} catch {
206+
// skip files we can't read
207+
}
208+
}
209+
}
210+
211+
return subpaths;
212+
}
213+
13214
export const patchTurbopackRuntime: CodePatcher = {
14215
name: "inline-turbopack-chunks",
15216
patches: [
@@ -19,8 +220,10 @@ export const patchTurbopackRuntime: CodePatcher = {
19220
escape: false,
20221
}),
21222
contentFilter: /loadRuntimeChunkPath/,
22-
patchCode: async ({ code, tracedFiles }) => {
23-
let patched = patchCode(code, inlineExternalImportRule);
223+
patchCode: async ({ code, tracedFiles, filePath }) => {
224+
const mappings = discoverExternalModuleMappings(filePath);
225+
const externalImportRule = buildExternalImportRule(mappings, tracedFiles, code);
226+
let patched = patchCode(code, externalImportRule);
24227
patched = patchCode(patched, inlineChunksRule);
25228

26229
return `${patched}\n${inlineChunksFn(tracedFiles)}`;
@@ -63,27 +266,3 @@ ${chunks
63266
}
64267
`;
65268
}
66-
67-
// Turbopack imports `og` via `externalImport`.
68-
// We patch it to:
69-
// - add the explicit path so that the file is inlined by wrangler
70-
// - use the edge version of the module instead of the node version.
71-
//
72-
// Modules that are not inlined (no added to the switch), would generate an error similar to:
73-
// Failed to load external module path/to/module: Error: No such module "path/to/module"
74-
const inlineExternalImportRule = `
75-
rule:
76-
pattern: "$RAW = await import($ID)"
77-
inside:
78-
regex: "externalImport"
79-
kind: function_declaration
80-
stopBy: end
81-
fix: |-
82-
switch ($ID) {
83-
case "next/dist/compiled/@vercel/og/index.node.js":
84-
$RAW = await import("next/dist/compiled/@vercel/og/index.edge.js");
85-
break;
86-
default:
87-
$RAW = await import($ID);
88-
}
89-
`;

0 commit comments

Comments
 (0)