From b605366cb42ed3668c4312ed85f6e3dbc497b1d4 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 17:55:06 +0800 Subject: [PATCH 01/14] =?UTF-8?q?feat(compiler):=20=E9=9A=8F=E5=8C=85?= =?UTF-8?q?=E5=8F=91=E7=B1=BB=E5=9E=8B=E5=A3=B0=E6=98=8E=EF=BC=8C=E4=B8=8B?= =?UTF-8?q?=E6=B8=B8=E4=B8=8D=E7=94=A8=E5=86=8D=E8=87=AA=E5=B7=B1=E5=86=99?= =?UTF-8?q?=20ambient=20=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @dimina-kit/compiler 之前一个 .d.ts 都不带，TypeScript 下游只能自己手写一份 declare module。手写的那份会过期：devkit 的 shims.d.ts 把 pool-node 默认导出写成 "编译出错时 resolve undefined"，而它实际是 reject。 - 用 tsc 从源码 JSDoc 生成声明到 dist/types/，六个导出子路径都加 types 条件； build 里带上这一步。 - check-types 校验每个子路径都有 types 条件且文件存在，再用 types-fixture/consumer.ts 按下游视角类型检查一遍：声明缺失或退化成 any 时，fixture 里故意写错的调用不再报错， tsc 会因为"未使用的 @ts-expect-error"失败。 - 补三处 JSDoc：pool 的 onLog 少了运行时一直在传的 stage；pool-node 的 createNodeCompilerPool 那段 JSDoc 中间隔着注释和 let chain，实际挂到了 chain 上， 从没生效过；默认导出的 build 没有参数类型。 - 删掉 devkit 的 shims.d.ts，改用包自带声明。 --- packages/compiler/README.md | 13 ++++- packages/compiler/package.json | 37 +++++++++--- packages/compiler/scripts/test-types.js | 47 +++++++++++++++ packages/compiler/src/pool-node.js | 29 +++++++--- packages/compiler/src/pool.js | 2 +- packages/compiler/tsconfig.types.json | 25 ++++++++ packages/compiler/types-fixture/consumer.ts | 57 +++++++++++++++++++ packages/compiler/types-fixture/tsconfig.json | 13 +++++ packages/devkit/src/shims.d.ts | 11 ---- pnpm-lock.yaml | 3 + 10 files changed, 208 insertions(+), 29 deletions(-) create mode 100644 packages/compiler/scripts/test-types.js create mode 100644 packages/compiler/tsconfig.types.json create mode 100644 packages/compiler/types-fixture/consumer.ts create mode 100644 packages/compiler/types-fixture/tsconfig.json delete mode 100644 packages/devkit/src/shims.d.ts diff --git a/packages/compiler/README.md b/packages/compiler/README.md index b494c84f..363b530a 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -10,6 +10,16 @@ pnpm add @dimina-kit/compiler ``` +## 类型声明 + +包自带 `.d.ts`（从源码 JSDoc 生成），六个导出子路径都带 `types` 条件，TypeScript 下游直接 import 就有补全和参数校验，不用自己维护一份 ambient 声明： + +```ts +import { createCompilerPool } from '@dimina-kit/compiler/pool' +``` + +声明由 `build:types` 生成到 `dist/types/`（`build` 已经包含这一步）。`check-types` 会校验每个子路径都有 `types` 条件、指向的文件确实存在，再用 `types-fixture/consumer.ts` 按下游视角完整类型检查一遍：声明缺失或退化成 `any` 时，fixture 里那些故意写错的调用不再报错，tsc 就会因为「未使用的 `@ts-expect-error`」失败。 + ## 三种接入，按需选择 | 导出 | 用途 | 编排（并行/复用/合并）由谁做 | @@ -362,9 +372,10 @@ pnpm install ## 构建 ```bash -pnpm --filter @dimina-kit/compiler build # node + 三个 browser bundle +pnpm --filter @dimina-kit/compiler build # node + 三个 browser bundle + 类型声明 pnpm --filter @dimina-kit/compiler build:browser # 仅 browser(core + stage-worker + pool) pnpm --filter @dimina-kit/compiler build:node # 仅 node +pnpm --filter @dimina-kit/compiler build:types # 仅 dist/types/*.d.ts ``` ## 测试 diff --git a/packages/compiler/package.json b/packages/compiler/package.json index be7797a6..e81a0684 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -18,12 +18,30 @@ }, "main": "./dist/compile-core.node.js", "exports": { - ".": "./dist/compile-core.node.js", - "./browser": "./dist/compile-core.browser.js", - "./pool-node": "./dist/pool.node.js", - "./pool": "./dist/pool.browser.js", - "./stage-worker": "./dist/stage-worker.browser.js", - "./toolchain": "./dist/toolchain.browser.js", + ".": { + "types": "./dist/types/compile-core.d.ts", + "default": "./dist/compile-core.node.js" + }, + "./browser": { + "types": "./dist/types/browser-entry.d.ts", + "default": "./dist/compile-core.browser.js" + }, + "./pool-node": { + "types": "./dist/types/pool-node.d.ts", + "default": "./dist/pool.node.js" + }, + "./pool": { + "types": "./dist/types/pool.d.ts", + "default": "./dist/pool.browser.js" + }, + "./stage-worker": { + "types": "./dist/types/stage-worker.d.ts", + "default": "./dist/stage-worker.browser.js" + }, + "./toolchain": { + "types": "./dist/types/toolchain.d.ts", + "default": "./dist/toolchain.browser.js" + }, "./package.json": "./package.json" }, "files": [ @@ -36,7 +54,9 @@ "snapshot:upstream-versions": "node scripts/snapshot-upstream-versions.js", "build:node": "node scripts/build-compiler.js node", "build:browser": "node scripts/build-compiler.js browser", - "build": "pnpm run build:node && pnpm run build:browser", + "build:types": "tsc -p tsconfig.types.json", + "build": "pnpm run build:node && pnpm run build:browser && pnpm run build:types", + "check-types": "pnpm run build:types && node scripts/test-types.js", "test:node": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-node.js", "test:appid": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-appid-fallback.js", "test:hardening": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-hardening.js", @@ -97,6 +117,7 @@ } }, "devDependencies": { - "esbuild-wasm": "^0.28.1" + "esbuild-wasm": "^0.28.1", + "typescript": "5.9.2" } } diff --git a/packages/compiler/scripts/test-types.js b/packages/compiler/scripts/test-types.js new file mode 100644 index 00000000..844d47fc --- /dev/null +++ b/packages/compiler/scripts/test-types.js @@ -0,0 +1,47 @@ +// Guards the declarations shipped alongside the bundles: every published subpath +// must carry a `types` condition that resolves to a real file, and the consumer +// fixture must still type-check against them. Run after `build:types`. +import { spawnSync } from 'node:child_process' +import { createRequire } from 'node:module' +import { existsSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const require = createRequire(import.meta.url) +const pkgRoot = path.dirname(fileURLToPath(new URL('../package.json', import.meta.url))) +const pkg = require('../package.json') + +const failures = [] + +for (const [subpath, target] of Object.entries(pkg.exports)) { + if (subpath === './package.json') continue + if (typeof target === 'string' || !target.types) { + failures.push(`exports["${subpath}"] ships no "types" condition — consumers get an untyped import`) + continue + } + const typesFile = path.join(pkgRoot, target.types) + if (!existsSync(typesFile)) { + failures.push(`exports["${subpath}"].types points at ${target.types}, which does not exist`) + } +} + +if (failures.length > 0) { + console.error(failures.map(f => ` - ${f}`).join('\n')) + process.exit(1) +} + +const tscBin = path.join(path.dirname(require.resolve('typescript/package.json')), 'bin', 'tsc') +const fixture = path.join(pkgRoot, 'types-fixture', 'tsconfig.json') +const tsc = spawnSync(process.execPath, [tscBin, '-p', fixture, '--pretty', 'false'], { + cwd: pkgRoot, + encoding: 'utf8', +}) + +if (tsc.status !== 0) { + console.error(tsc.stdout || '') + console.error(tsc.stderr || '') + console.error(`consumer fixture failed to type-check against the published declarations (tsc exit ${tsc.status})`) + process.exit(1) +} + +console.log(`types ok: ${Object.keys(pkg.exports).length - 1} typed subpaths, consumer fixture type-checks`) diff --git a/packages/compiler/src/pool-node.js b/packages/compiler/src/pool-node.js index 020958c1..60cf61d4 100644 --- a/packages/compiler/src/pool-node.js +++ b/packages/compiler/src/pool-node.js @@ -58,6 +58,19 @@ function isDeadToolchainServiceError(message) { // rapid edit-compile loops warm while an IDE left idle overnight stops holding the memory. const DEFAULT_IDLE_SHRINK_MS = 300000 +// ALL Node disk-pool builds serialize through this single module-level chain, not a +// per-instance one: setupCompile/publishToDist go through dmcc's process-global env +// singletons (storeInfo / getTargetPath / getAppId), so builds from two DIFFERENT +// pool instances would corrupt each other just as surely as two builds in one pool +// (one pool publishing the other's staging dir under the other's appId). +let chain = Promise.resolve() + +/** + * @typedef {{ template?: string[], style?: string[], viewScript?: string[] }} FileTypes + * @typedef {{ sourcemap?: boolean, fileTypes?: FileTypes }} BuildOptions + * @typedef {{ appId: string, name: string, path: string }} BuildResult + */ + /** * Create a resident Node stage-worker pool. * @param {{ @@ -66,15 +79,8 @@ const DEFAULT_IDLE_SHRINK_MS = 300000 * retryOnWorkerDeath?: boolean, // default true — one transparent whole-build retry after a worker death * idleShrinkMs?: number|false, // default 300000 — idle ms before workers are shrunk; 0/false/Infinity disables * }} [opts] - * @returns {{ build: (outputDir:string, workPath:string, useAppIdDir?:boolean, options?:object)=>Promise<{appId:string,name:string,path:string}>, dispose: ()=>Promise, stages: string[] }} + * @returns {{ build: (outputDir: string, workPath: string, useAppIdDir?: boolean, options?: BuildOptions) => Promise, dispose: () => Promise, stages: string[] }} */ -// ALL Node disk-pool builds serialize through this single module-level chain, not a -// per-instance one: setupCompile/publishToDist go through dmcc's process-global env -// singletons (storeInfo / getTargetPath / getAppId), so builds from two DIFFERENT -// pool instances would corrupt each other just as surely as two builds in one pool -// (one pool publishing the other's staging dir under the other's appId). -let chain = Promise.resolve() - export function createNodeCompilerPool({ stages = STAGE_NAMES, sendTimeoutMs = DEFAULT_SEND_TIMEOUT_MS, @@ -313,6 +319,13 @@ export function oxcNativeBindingHint(message) { // Callers that want structured errors (`.stage`/`.code`) + explicit teardown should // use createNodeCompilerPool() directly instead. let singleton = null +/** + * @param {string} outputDir + * @param {string} workPath + * @param {boolean} [useAppIdDir] + * @param {BuildOptions} [options] + * @returns {Promise} + */ export default async function build(outputDir, workPath, useAppIdDir = true, options = {}) { if (!singleton) singleton = createNodeCompilerPool() try { diff --git a/packages/compiler/src/pool.js b/packages/compiler/src/pool.js index be6a609d..a76b78c1 100644 --- a/packages/compiler/src/pool.js +++ b/packages/compiler/src/pool.js @@ -44,7 +44,7 @@ const WORKER_DEATH_CODES = new Set(['compiler-worker-timeout', 'compiler-worker- * toolchainSetupURL: string, // required: ESM URL that installs __esbuildTransform/__oxcParseSync in the worker * stages?: string[], // default ['logic','view','style'] * workPath?: string, // default '/work' - * onLog?: (entry: { level: string, message: string }) => void, // worker console diagnostics + * onLog?: (entry: { level: string, message: string, stage: string }) => void, // worker console diagnostics, tagged with the stage worker that emitted them * sendTimeoutMs?: number, // default 30000 — inactivity window per setup/compile-subset round trip * warmupTimeoutMs?: number, // default 120000 — inactivity window for the warmup round trip * retryOnWorkerDeath?: boolean, // default true — one transparent whole-attempt retry after a worker death diff --git a/packages/compiler/tsconfig.types.json b/packages/compiler/tsconfig.types.json new file mode 100644 index 00000000..0eb9fc46 --- /dev/null +++ b/packages/compiler/tsconfig.types.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": false, + "noCheck": true, + "declaration": true, + "emitDeclarationOnly": true, + "skipLibCheck": true, + "module": "esnext", + "moduleResolution": "bundler", + "target": "es2022", + "lib": ["es2022", "dom"], + "types": [], + "rootDir": "src", + "outDir": "dist/types" + }, + "include": [ + "src/compile-core.js", + "src/browser-entry.js", + "src/pool.js", + "src/pool-node.js", + "src/stage-worker.js", + "src/toolchain.js" + ] +} diff --git a/packages/compiler/types-fixture/consumer.ts b/packages/compiler/types-fixture/consumer.ts new file mode 100644 index 00000000..5725273f --- /dev/null +++ b/packages/compiler/types-fixture/consumer.ts @@ -0,0 +1,57 @@ +// Type-checks every published entry point the way a downstream TypeScript app +// imports it. The `@ts-expect-error` lines are the real guard: if an entry +// silently loses its declarations (missing `types` condition, an emit that +// degrades to `any`), the deliberately wrong call stops erroring and tsc fails +// this file with "unused '@ts-expect-error' directive". + +import { collectOutputs, STAGE_NAMES } from '@dimina-kit/compiler' +import { initToolchain } from '@dimina-kit/compiler/browser' +import { createCompilerPool } from '@dimina-kit/compiler/pool' +import { createNodeCompilerPool } from '@dimina-kit/compiler/pool-node' +import '@dimina-kit/compiler/stage-worker' +import { installOxc } from '@dimina-kit/compiler/toolchain' + +const stages: string[] = STAGE_NAMES +void stages + +const outputs: Record = collectOutputs({ fs: {}, targetPath: '/dist' }) +void outputs +// @ts-expect-error targetPath is required +collectOutputs({ fs: {} }) + +const ready: Promise = initToolchain() +void ready +// @ts-expect-error initToolchain takes no arguments +initToolchain('oxc') + +installOxc({ parseSync: () => undefined }) +// @ts-expect-error the oxc module bag is an object, not its specifier +installOxc('oxc-parser') + +const pool = createCompilerPool({ + createWorker: () => new Worker('/stage-worker.browser.js', { type: 'module' }), + toolchainSetupURL: '/toolchain-setup.mjs', + onLog: (entry) => { + const line: string = `[${entry.stage}] ${entry.level}: ${entry.message}` + void line + }, +}) +// @ts-expect-error createWorker is required +createCompilerPool({ toolchainSetupURL: '/toolchain-setup.mjs' }) +// @ts-expect-error toolchainSetupURL is a URL string +createCompilerPool({ createWorker: () => new Worker('/w.js'), toolchainSetupURL: 42 }) + +export async function compileOnce(): Promise { + await pool.warmup() + const result = await pool.compile({ files: { 'app.json': '{}' } }) + const appId: string = result.appId + // @ts-expect-error the compile result carries appId/name/files only + void result.bundle + await pool.dispose() + return appId +} + +const nodePool = createNodeCompilerPool({ stages: ['logic'] }) +void nodePool +// @ts-expect-error stages is a list of stage names +createNodeCompilerPool({ stages: 'logic' }) diff --git a/packages/compiler/types-fixture/tsconfig.json b/packages/compiler/types-fixture/tsconfig.json new file mode 100644 index 00000000..ecf746f9 --- /dev/null +++ b/packages/compiler/types-fixture/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "noEmit": true, + "strict": true, + "skipLibCheck": true, + "module": "esnext", + "moduleResolution": "bundler", + "target": "es2022", + "lib": ["es2022", "dom"], + "types": [] + }, + "include": ["consumer.ts"] +} diff --git a/packages/devkit/src/shims.d.ts b/packages/devkit/src/shims.d.ts deleted file mode 100644 index 2ca92633..00000000 --- a/packages/devkit/src/shims.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -// @dimina-kit/compiler ships no type declarations. The Node disk pool's default export -// is a drop-in for dmcc's build(): same 4-arg signature, resolves undefined on compile -// error (never rethrows), keeps its 3 stage workers warm across rebuilds. -declare module '@dimina-kit/compiler/pool-node' { - export default function build( - targetPath: string, - workPath: string, - useAppIdDir?: boolean, - options?: { sourcemap?: boolean, fileTypes?: { template?: string[], style?: string[], viewScript?: string[] } }, - ): Promise<{ appId: string, name: string, path: string } | undefined> -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aad5a419..00cad895 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -116,6 +116,9 @@ importers: esbuild-wasm: specifier: ^0.28.1 version: 0.28.2 + typescript: + specifier: 5.9.2 + version: 5.9.2 packages/design: dependencies: From 60797969c824790a3e7f2dd43fd6f594ff27d9b7 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 21:51:23 +0800 Subject: [PATCH 02/14] =?UTF-8?q?fix(compiler):=20=E8=AE=A9=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=20fixture=20=E7=9C=9F=E7=9A=84=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E5=8F=91=E5=87=BA=E5=8E=BB=E7=9A=84=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixture 开着 skipLibCheck，等于把要检查的东西本身跳过了：compile-core.d.ts 里 compileStage 解构了 sourcemap，而 JSDoc 的参数类型里没有这个字段（TS2339）， 一直没人看见。关掉之后这个错误立刻现形，补上 JSDoc 才重新变绿。 顺带修掉三处声明与实际行为对不上的地方： - createCompilerPool 原来写 `options = {}`，生成的声明里参数就是可选的。 TypeScript 下游写 createCompilerPool() 能过类型检查，一运行就抛"createWorker is required"。现在参数必填，函数体里补 `|| {}` 保证这种调用仍然得到那句人话报错。 - warmup() 声明成 Promise，README 写的是 Promise。那个数组是 settleAll 的内部记账，暴露出去会被当成每个 stage 的结果，改成 await 后不返回。 - onLog 的 level 被推断成 string。stage-worker 只包了 console 的 log/warn/error 三个方法，写成这三者的联合类型，下游 switch 才会在漏分支时报错。 --- packages/compiler/src/compile-core.js | 2 +- packages/compiler/src/pool.js | 15 ++++++++++----- packages/compiler/types-fixture/consumer.ts | 10 ++++++++-- packages/compiler/types-fixture/tsconfig.json | 5 ++++- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/compiler/src/compile-core.js b/packages/compiler/src/compile-core.js index 99ea1bec..ee397b3c 100644 --- a/packages/compiler/src/compile-core.js +++ b/packages/compiler/src/compile-core.js @@ -416,7 +416,7 @@ export async function setupCompile({ fs, workPath = '/work', options = {}, npmSc * from `setupCompile`. Self-contained: it points the fs shim at `fs` and restores * the compiler env from the bundle, so it can run in a fresh worker realm. * Products are written into `fs`. - * @param {{ stage: 'logic'|'view'|'style', pages: object, storeInfo: object, fs: object }} opts + * @param {{ stage: 'logic'|'view'|'style', pages: object, storeInfo: object, fs: object, sourcemap?: boolean }} opts */ export async function compileStage({ stage, pages, storeInfo: bundle, fs, sourcemap = false } = {}) { assertFs(fs) diff --git a/packages/compiler/src/pool.js b/packages/compiler/src/pool.js index a76b78c1..b64fa35f 100644 --- a/packages/compiler/src/pool.js +++ b/packages/compiler/src/pool.js @@ -44,13 +44,16 @@ const WORKER_DEATH_CODES = new Set(['compiler-worker-timeout', 'compiler-worker- * toolchainSetupURL: string, // required: ESM URL that installs __esbuildTransform/__oxcParseSync in the worker * stages?: string[], // default ['logic','view','style'] * workPath?: string, // default '/work' - * onLog?: (entry: { level: string, message: string, stage: string }) => void, // worker console diagnostics, tagged with the stage worker that emitted them + * onLog?: (entry: { level: 'log'|'warn'|'error', message: string, stage: string }) => void, // worker console diagnostics, tagged with the stage worker that emitted them; level is whichever console.* the compiler called (stage-worker.js patches exactly these three) * sendTimeoutMs?: number, // default 30000 — inactivity window per setup/compile-subset round trip * warmupTimeoutMs?: number, // default 120000 — inactivity window for the warmup round trip * retryOnWorkerDeath?: boolean, // default true — one transparent whole-attempt retry after a worker death * }} options */ -export function createCompilerPool(options = {}) { +// options 没有默认值是有意的：createWorker 和 toolchainSetupURL 都必填，参数整体 +// 也就必填，生成的 .d.ts 才不会让 TypeScript 下游写出能过类型检查、一运行就抛的 +// createCompilerPool()。函数体里的 `|| {}` 只是让那种调用仍然拿到下面这句人话报错。 +export function createCompilerPool(options) { const { createWorker, toolchainSetupURL, @@ -60,7 +63,7 @@ export function createCompilerPool(options = {}) { sendTimeoutMs = DEFAULT_SEND_TIMEOUT_MS, warmupTimeoutMs = DEFAULT_WARMUP_TIMEOUT_MS, retryOnWorkerDeath = true, - } = options + } = options || {} if (typeof createWorker !== 'function') { throw new Error('[compiler] createCompilerPool: options.createWorker (() => Worker) is required') } @@ -153,8 +156,10 @@ export function createCompilerPool(options = {}) { return entry.warmed } - function warmup() { - return settleAll(workers.map(ensureWarm)) + // 返回值刻意丢掉：settleAll 的数组是内部记账，暴露出去下游会以为那是每个 stage + // 的结果。await 之后什么都拿不到，正好对应 README 里写的 Promise。 + async function warmup() { + await settleAll(workers.map(ensureWarm)) } // One full compile attempt against the resident realms. Ends quiescent: settleAll diff --git a/packages/compiler/types-fixture/consumer.ts b/packages/compiler/types-fixture/consumer.ts index 5725273f..b6c50887 100644 --- a/packages/compiler/types-fixture/consumer.ts +++ b/packages/compiler/types-fixture/consumer.ts @@ -32,17 +32,23 @@ const pool = createCompilerPool({ createWorker: () => new Worker('/stage-worker.browser.js', { type: 'module' }), toolchainSetupURL: '/toolchain-setup.mjs', onLog: (entry) => { - const line: string = `[${entry.stage}] ${entry.level}: ${entry.message}` + // The level is one of the three console methods the stage worker patches — + // a widened `string` here would let a downstream switch miss cases silently. + const level: 'log' | 'warn' | 'error' = entry.level + const line: string = `[${entry.stage}] ${level}: ${entry.message}` void line }, }) // @ts-expect-error createWorker is required createCompilerPool({ toolchainSetupURL: '/toolchain-setup.mjs' }) +// @ts-expect-error both required options live in the (required) options object +createCompilerPool() // @ts-expect-error toolchainSetupURL is a URL string createCompilerPool({ createWorker: () => new Worker('/w.js'), toolchainSetupURL: 42 }) export async function compileOnce(): Promise { - await pool.warmup() + const warm: Promise = pool.warmup() + await warm const result = await pool.compile({ files: { 'app.json': '{}' } }) const appId: string = result.appId // @ts-expect-error the compile result carries appId/name/files only diff --git a/packages/compiler/types-fixture/tsconfig.json b/packages/compiler/types-fixture/tsconfig.json index ecf746f9..16528988 100644 --- a/packages/compiler/types-fixture/tsconfig.json +++ b/packages/compiler/types-fixture/tsconfig.json @@ -2,7 +2,10 @@ "compilerOptions": { "noEmit": true, "strict": true, - "skipLibCheck": true, + // 本包发出去的 .d.ts 正是这个 fixture 要检查的东西，skipLibCheck 会把它们内部 + // 的错误全部跳过——JSDoc 少写一个字段、解构出一个类型上不存在的属性，都会被 + // 咽掉，下游装上包才报出来。所以这里必须是 false。 + "skipLibCheck": false, "module": "esnext", "moduleResolution": "bundler", "target": "es2022", From a03c4eb6292e1f937be53c5d4e676220d2299475 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 19:25:23 +0800 Subject: [PATCH 03/14] =?UTF-8?q?feat(compiler):=20=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E9=9D=99=E6=80=81=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E6=B8=85=E5=8D=95=EF=BC=8C=E6=9E=84=E5=BB=BA=E6=9C=9F=E6=8C=A1?= =?UTF-8?q?=E4=BD=8F=E4=BA=A7=E7=89=A9=E6=BC=82=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dist 里三个 .browser.js 必须由宿主原样托管：stage worker 只被 new Worker(url) 引用，另外两个只被 fetch 后从 Blob URL import。没有 一处是打包器看得见的静态 import，所以文件名一旦变了，宿主那边不会 有任何编译期报错，只在运行时 404。 新增 @dimina-kit/compiler/browser-assets：清单 COMPILER_BROWSER_ASSETS、 从 ./browser 入口推出资源路径的 resolveBrowserAssets，宿主不用再手抄 文件名。同时发 ESM 与 CJS，只做字符串拼接，不依赖 node:path。 browser 构建现在开 metafile，拿 esbuild 自己的产物记录对着清单自检： 清单里的产物没被生成、生成了清单外的新产物（比如拆出 chunk）、或某个 静态资源开始带静态 import（不再自包含），构建直接失败。 scripts/test-browser-assets.js 把上述失败模式直接驱动一遍，并接进 package.json 的 test（turbo run test 会跑到）。 --- packages/compiler/README.md | 23 ++- packages/compiler/package.json | 7 + packages/compiler/scripts/build-compiler.js | 37 ++++- .../compiler/scripts/test-browser-assets.js | 109 +++++++++++++ packages/compiler/src/browser-assets.js | 149 ++++++++++++++++++ packages/compiler/tsconfig.types.json | 3 +- packages/compiler/types-fixture/consumer.ts | 8 + 7 files changed, 333 insertions(+), 3 deletions(-) create mode 100644 packages/compiler/scripts/test-browser-assets.js create mode 100644 packages/compiler/src/browser-assets.js diff --git a/packages/compiler/README.md b/packages/compiler/README.md index 363b530a..a56cab13 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -39,6 +39,23 @@ import { createCompilerPool } from '@dimina-kit/compiler/pool' | `dist/stage-worker.browser.js` | 浏览器 Worker | 常驻 stage worker（已内联 core + memfs），pool 用它做并行 | | `dist/pool.browser.js` | 浏览器主线程 | 编排池 `createCompilerPool` | +### 浏览器静态资源：哪三个文件必须原样托管 + +上表里带 `.browser.js` 的三个产物，宿主要**原样拷贝、原样托管**，不能再过一遍自己的打包器。stage worker 只被 `new Worker(url)` 引用，另外两个只被 fetch 下来从 Blob URL import——没有一处是打包器能看见的静态 import，所以打包器要么整个漏掉这些文件，要么把它们改写坏，两种情况都不报错，只在运行时 404 或行为异常。 + +文件名由本包给出，不要在宿主里手抄： + +```js +const { resolveBrowserAssets } = require('@dimina-kit/compiler/browser-assets') +const { files } = resolveBrowserAssets(require.resolve('@dimina-kit/compiler/browser')) +// files:dist 里那三个 .browser.js 的绝对路径，按清单顺序 +for (const file of files) fs.copyFileSync(file, path.join(publicDir, path.basename(file))) +``` + +`./browser-assets` 同时发 ESM 和 CJS（走 `require` 条件），只做字符串拼接，不依赖 `node:path`。 + +每次浏览器构建都会拿 esbuild 的 metafile 对着这份清单自检：产物改了名、被拆出新 chunk、或某个静态资源开始带静态 import（不再是自包含的单文件），构建当场失败，而不是几个月后在某个宿主那里 404。`toolchain.browser.js` 不在清单里——它由宿主用自己的打包器 import（`@dimina-kit/compiler/toolchain`），拷过去也没人 fetch。 + ## 架构 本包是**编译器与文件系统之间的一层适配**，再往上叠一层**编排**。真正的编译逻辑在 `dimina` 子模块的 `@dimina/compiler`，本包用一个**无后端的 fs 转发 shim** 把它每一次 `fs.xxx` 指向下游注入的 fs；`pool` 则在上面替下游管好 worker 池与并行——下游不再手写任何 worker/合并逻辑。 @@ -380,6 +397,8 @@ pnpm --filter @dimina-kit/compiler build:types # 仅 dist/types/*.d.ts ## 测试 +`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）只包含不需要构建的静态资源契约测试；下面这些各自要先构建，按需单跑。 + 测试里用 memfs 扮演「下游 fs」： ```bash @@ -393,6 +412,7 @@ pnpm --filter @dimina-kit/compiler test:lazy-toolchain # 按 stage 懒加载边 pnpm --filter @dimina-kit/compiler test:idle-shrink # idle 收缩:超时终止 worker、下次 build 透明复活、活动取消、进程可自然退出 pnpm --filter @dimina-kit/compiler test:stage-worker-message-order # stage worker 回复严格按请求到达序串行(build 在途时 introspect 不抢 FIFO 配对) pnpm --filter @dimina-kit/compiler test:stage-load-retry # stage 工具链加载失败不缓存,chunk 恢复后 preloadStage 可重试成功 +pnpm --filter @dimina-kit/compiler test:browser-assets # 静态资源清单:改名/新 chunk/出现静态 import 都会被构建期检查拦下 ``` `pool` 的浏览器端到端验证在 `dimina-web-client`（`npm run test:pool`，Playwright 驱动，产物与单线程逐结构一致）。`pool-node` 的宿主端验证在 `@dimina-kit/devkit` 测试套件（真实 fork + `openProject`）。 @@ -406,9 +426,10 @@ pnpm --filter @dimina-kit/compiler test:stage-load-retry # stage 工 - `src/pool-node.js` — **Node 编排池** `createNodeCompilerPool` + dmcc drop-in 默认导出 `build()`：常驻 worker_threads、真实磁盘、全局 build 串行、死 worker 懒复活、idle 自动收缩（`idleShrinkMs`）。 - `src/stage-worker-node.js` — Node 常驻 stage worker：spawn 时按 workerData 里的 stage 身份预热本 stage 工具链，恢复 storeInfo → `runStage(stage, { sourcemap })` 写共享 staging 目录；应答 `{ type: 'introspect' }` 报告本 realm 已加载的重依赖。 - `src/toolchain.js` — 写 `toolchainSetupURL` 模块的可选助手（`installOxc` / `installEsbuildFromURL`，后者内置 esbuild-wasm 静态资源的 Blob-URL 兜底）。导出为 `@dimina-kit/compiler/toolchain`。 +- `src/browser-assets.js` — 浏览器静态资源清单与契约（`COMPILER_BROWSER_ASSETS` / `resolveBrowserAssets`，见上文），构建期检查也用它。导出为 `@dimina-kit/compiler/browser-assets`。 - `src/shims/fs.js` — **无后端的 fs 转发层**（`setFs`/`resetFs`/`getFs`）；compiler 所有 `fs.xxx` 走它，未注入即抛错。 - `src/shims/*` — 其余 node 内置与原生依赖的浏览器替身（oxc/esbuild/less/`os.homedir`/…）。 -- `scripts/build-compiler.js` — esbuild 打包。onLoad 给 logic/view/style-compiler 与 utils 追加 `__reset*` 导出（喂 `resetCompilerState`，不改子模块源码）；浏览器分支内联真实 `cssnano`+`autoprefixer`（autoprefixer pin 到 node 运行时解析的同一份，避免 esbuild 解析到多加 `-ms-` 前缀的另一版本）；browser 模式产出 core / stage-worker / pool 三个单文件 bundle；node 模式开 `splitting`（stage 编译器成为运行时 chunk——单文件会把 chunk 的 external `import 'sass'` 提升回入口顶层，懒加载会静默失效）。 +- `scripts/build-compiler.js` — esbuild 打包。onLoad 给 logic/view/style-compiler 与 utils 追加 `__reset*` 导出（喂 `resetCompilerState`，不改子模块源码）；浏览器分支内联真实 `cssnano`+`autoprefixer`（autoprefixer pin 到 node 运行时解析的同一份，避免 esbuild 解析到多加 `-ms-` 前缀的另一版本）；browser 模式产出 core / stage-worker / pool 三个单文件 bundle，并按 metafile 对 `src/browser-assets.js` 的清单自检（漏产物、多产物、静态资源出现静态 import 都直接失败）；node 模式开 `splitting`（stage 编译器成为运行时 chunk——单文件会把 chunk 的 external `import 'sass'` 提升回入口顶层，懒加载会静默失效）。 - `scripts/{register-kit,kit-resolve-hook}.js` — node 用的 ESM resolve hook（默认解析优先、从 dimina-kit workspace 根兜底解析 bare 依赖）。 ## License diff --git a/packages/compiler/package.json b/packages/compiler/package.json index e81a0684..e7d93901 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -42,6 +42,11 @@ "types": "./dist/types/toolchain.d.ts", "default": "./dist/toolchain.browser.js" }, + "./browser-assets": { + "types": "./dist/types/browser-assets.d.ts", + "require": "./dist/browser-assets.cjs", + "default": "./dist/browser-assets.js" + }, "./package.json": "./package.json" }, "files": [ @@ -64,6 +69,8 @@ "test:realm-reuse": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-realm-reuse.js", "test:pool-node": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-node.js", "test:pool-scopehash": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-scopehash.js", + "test": "pnpm run test:browser-assets", + "test:browser-assets": "node scripts/test-browser-assets.js", "test:crypto-shim": "node scripts/test-crypto-shim.js", "test:pool-hardening": "node scripts/build-compiler.js node && node scripts/test-pool-hardening.js", "test:pool-toolchain-death": "node scripts/build-compiler.js node && node scripts/test-pool-toolchain-death.js", diff --git a/packages/compiler/scripts/build-compiler.js b/packages/compiler/scripts/build-compiler.js index 4a649372..addd371f 100644 --- a/packages/compiler/scripts/build-compiler.js +++ b/packages/compiler/scripts/build-compiler.js @@ -3,6 +3,7 @@ import { readFile } from 'node:fs/promises' import { fileURLToPath } from 'node:url' import { createRequire } from 'node:module' import path from 'node:path' +import { COMPILER_BROWSER_ASSETS, browserOutputsFromMetafile, checkBrowserAssetContract } from '../src/browser-assets.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const root = path.resolve(__dirname, '..') @@ -219,6 +220,7 @@ const builds = MODE === 'node' ] const { rm } = await import('node:fs/promises') +const browserMetafileOutputs = {} for (const b of builds) { const built = { ...opts, @@ -228,10 +230,43 @@ for (const b of builds) { ...(b.chunkDir ? { splitting: true, chunkNames: `${b.chunkDir}/[name]-[hash]` } : {}), + // Browser mode: the metafile is what the static-asset contract is checked + // against below — esbuild's own record of what it emitted and what each + // output still imports, rather than a re-scan of the bytes. + ...(MODE === 'browser' ? { metafile: true } : {}), } // Chunk names embed content hashes; stale ones from earlier builds would pile up in // dist (and ship in the npm package), so each build owns and clears its chunk dir. if (b.chunkDir) await rm(path.join(root, 'dist', b.chunkDir), { recursive: true, force: true }) - await esbuild.build(built) + const result = await esbuild.build(built) + if (result.metafile) Object.assign(browserMetafileOutputs, result.metafile.outputs) console.log(`✅ built MODE=${MODE} USE_WASM=${USE_WASM ? 1 : 0} -> ${b.entries.map((e) => `dist/${e.out}.js`).join(', ')}`) } + +// The static-asset manifest itself: dependency-free string code, emitted in both +// modes (either build alone leaves a usable dist) and in both formats, because the +// hosts that copy these files are as often CommonJS build scripts as ESM ones. +for (const [format, outfile] of [['esm', 'browser-assets.js'], ['cjs', 'browser-assets.cjs']]) { + await esbuild.build({ + entryPoints: [path.join(root, 'src/browser-assets.js')], + outfile: path.join(root, 'dist', outfile), + bundle: true, + format, + target: ['es2022'], + logLevel: 'warning', + }) +} +console.log('✅ built dist/browser-assets.js + dist/browser-assets.cjs') + +// The browser bundles double as static assets a host copies and serves. Their names +// and the "no static imports" rule are stated once in src/browser-assets.js, and +// enforced here so a rename or a newly split chunk fails the build instead of +// 404-ing (or half-loading) inside a host months later. +if (MODE === 'browser') { + const problems = checkBrowserAssetContract(browserOutputsFromMetafile({ outputs: browserMetafileOutputs })) + if (problems.length > 0) { + console.error(problems.map((line) => ` ✗ ${line}`).join('\n')) + process.exit(1) + } + console.log(`✅ browser static-asset contract holds (${COMPILER_BROWSER_ASSETS.length} assets, no static imports)`) +} diff --git a/packages/compiler/scripts/test-browser-assets.js b/packages/compiler/scripts/test-browser-assets.js new file mode 100644 index 00000000..be2db3ac --- /dev/null +++ b/packages/compiler/scripts/test-browser-assets.js @@ -0,0 +1,109 @@ +// Contract tests for src/browser-assets.js — the static-asset manifest hosts copy +// from, and the check build-compiler.js runs against every browser build. +// +// The failure modes below are exactly what the build-time check exists to catch, so +// they are driven here as synthetic output lists (no build step): a renamed output, +// an unclassified new output, and an asset that stopped being self-contained. +import { existsSync } from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { + BUNDLER_ONLY_BROWSER_OUTPUTS, + COMPILER_BROWSER_ASSETS, + browserOutputsFromMetafile, + checkBrowserAssetContract, + resolveBrowserAssets, +} from '../src/browser-assets.js' + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') + +let failed = false +const chk = (cond, msg) => { if (!cond) { failed = true; console.error(`❌ ${msg}`) } else console.log(`✅ ${msg}`) } + +const assetNames = COMPILER_BROWSER_ASSETS.map((asset) => asset.name) +const healthy = [ + ...assetNames.map((name) => ({ name, staticImports: [] })), + ...BUNDLER_ONLY_BROWSER_OUTPUTS.map((name) => ({ name, staticImports: [] })), +] + +chk(checkBrowserAssetContract(healthy).length === 0, 'a build that emits exactly the listed outputs passes') + +// Every asset is reachable through the exports map, so a host never has to guess +// where dist/ is — and the names in the manifest are the names on disk. +const pkg = JSON.parse(await (await import('node:fs/promises')).readFile(path.join(root, 'package.json'), 'utf8')) +chk( + typeof pkg.exports['./browser-assets'] === 'object' && pkg.exports['./browser-assets'].require === './dist/browser-assets.cjs', + 'the manifest is published for CommonJS hosts too (exports["./browser-assets"].require)', +) + +const renamed = healthy.map((output) => (output.name === 'pool.browser.js' ? { ...output, name: 'pool.js' } : output)) +const renamedProblems = checkBrowserAssetContract(renamed) +chk( + renamedProblems.some((line) => line.includes('did not emit it')), + 'renaming an output without updating the manifest is reported', +) +chk( + renamedProblems.some((line) => line.includes('pool.js') && line.includes('new browser output')), + 'the output under its new name is reported as unclassified', +) + +chk( + checkBrowserAssetContract([...healthy, { name: 'compile-core.browser-chunk.js', staticImports: [] }]) + .some((line) => line.includes('new browser output')), + 'a newly split chunk is reported until it is classified', +) + +chk( + checkBrowserAssetContract( + healthy.map((output) => (output.name === 'stage-worker.browser.js' + ? { ...output, staticImports: ['./chunk-ABC.js'] } + : output)), + ).some((line) => line.includes('statically imports ./chunk-ABC.js')), + 'an asset that stopped being self-contained is reported', +) + +// What the build actually feeds the check: esbuild's metafile, keyed by path, +// carrying both import kinds plus sourcemap entries. +const shaped = browserOutputsFromMetafile({ + outputs: { + 'dist/stage-worker.browser.js': { + imports: [ + { path: 'toolchainSetupURL', kind: 'dynamic-import' }, + { path: './chunk-XYZ.js', kind: 'import-statement' }, + ], + }, + 'dist/stage-worker.browser.js.map': { imports: [] }, + }, +}) +chk(shaped.length === 1 && shaped[0].name === 'stage-worker.browser.js', 'metafile paths are reduced to file names, sourcemaps dropped') +chk( + shaped[0].staticImports.length === 1 && shaped[0].staticImports[0] === './chunk-XYZ.js', + 'a dynamic import (the host toolchain setup URL) is not counted; a static one is', +) + +// resolveBrowserAssets is the path every consumer derives from the ./browser entry. +const posix = resolveBrowserAssets('/app/node_modules/@dimina-kit/compiler/dist/compile-core.browser.js') +chk(posix.dir === '/app/node_modules/@dimina-kit/compiler/dist', 'POSIX: dir is the entry directory') +chk( + posix.files.join('|') === assetNames.map((name) => `${posix.dir}/${name}`).join('|'), + 'POSIX: every asset resolves as a sibling of the entry, in manifest order', +) + +const win = resolveBrowserAssets('C:\\app\\node_modules\\@dimina-kit\\compiler\\dist\\compile-core.browser.js') +chk(win.files[0] === 'C:\\app\\node_modules\\@dimina-kit\\compiler\\dist\\stage-worker.browser.js', 'Windows separators are preserved') + +let threw = false +try { resolveBrowserAssets('compile-core.browser.js') } catch { threw = true } +chk(threw, 'a bare file name (not a path) is rejected rather than silently resolved to ""') + +// If dist is already built, the manifest must describe what is actually there. +const dist = path.join(root, 'dist') +if (existsSync(path.join(dist, 'compile-core.browser.js'))) { + const missing = resolveBrowserAssets(path.join(dist, 'compile-core.browser.js')).files.filter((file) => !existsSync(file)) + chk(missing.length === 0, `built dist contains every listed asset${missing.length ? `: missing ${missing.join(', ')}` : ''}`) +} else { + console.log('ℹ️ dist not built — skipped the on-disk check (the browser build runs it itself)') +} + +console.log(failed ? '\n❌ FAIL' : '\n✅ PASS: browser static-asset manifest and contract check behave') +process.exitCode = failed ? 1 : 0 diff --git a/packages/compiler/src/browser-assets.js b/packages/compiler/src/browser-assets.js new file mode 100644 index 00000000..9a77433d --- /dev/null +++ b/packages/compiler/src/browser-assets.js @@ -0,0 +1,149 @@ +/** + * The authoritative statement of this package's browser static-asset contract, + * for Node-side consumers that copy or serve the browser bundles (a workbench's + * dev/build server, a `copy-web-compiler` script): + * + * The files listed in COMPILER_BROWSER_ASSETS are single-file ESM bundles + * with no static imports, sitting side by side in dist/ under exactly these + * names. Hosts must serve them RAW — running them through a bundler again + * breaks them, and nothing forces the host to notice: the stage worker is + * only ever `new Worker(url)`'d and the other two are only ever fetched and + * imported from a Blob URL, so no bundler ever sees the reference and + * rewrites it. + * + * build-compiler.js asserts both halves at build time out of esbuild's own + * metafile: every browser output is classified here (static asset or + * bundler-only), and every static asset really has zero static imports. So + * renaming or splitting an output without updating this list fails the build, + * instead of 404-ing in some host months later. + * + * Pure string manipulation (no node:path) so the module loads in any runtime, + * and it ships as both ESM (dist/browser-assets.js) and CJS + * (dist/browser-assets.cjs — see the exports map's `require` condition, for + * CommonJS hosts). + */ + +/** + * @typedef {object} CompilerBrowserAsset + * @property {string} name File name in dist/, and the name to serve it under. + * @property {string} loadedBy How the browser gets it — why it must stay raw. + */ + +/** The browser bundles a host has to host as static assets. */ +export const COMPILER_BROWSER_ASSETS = /** @type {readonly CompilerBrowserAsset[]} */ ([ + { + name: 'stage-worker.browser.js', + loadedBy: "new Worker(url, { type: 'module' }) — the URL createCompilerPool's createWorker hands the browser", + }, + { + name: 'pool.browser.js', + loadedBy: 'fetch + Blob-URL import, for hosts that load the pool at runtime instead of bundling `@dimina-kit/compiler/pool`', + }, + { + name: 'compile-core.browser.js', + loadedBy: 'fetch + Blob-URL import, on the single-threaded fallback path (no pool, no stage workers)', + }, +]) + +/** + * Browser outputs that are NOT static assets: hosts reach them through their own + * bundler (`import { installOxc } from '@dimina-kit/compiler/toolchain'`), so + * copying them next to the assets above only ships a file nobody fetches. Listed + * here so the build-time check can classify every browser output it produces — + * a new output that is neither an asset nor bundler-only fails the build. + */ +export const BUNDLER_ONLY_BROWSER_OUTPUTS = /** @type {readonly string[]} */ (['toolchain.browser.js']) + +/** + * Shape esbuild's metafile into the output list {@link checkBrowserAssetContract} + * takes: sourcemaps dropped, and only `import-statement` imports kept. Dynamic + * imports are deliberately ignored — the stage worker reaches the host's toolchain + * setup module through `import(toolchainSetupURL)` at runtime, which is the + * contract, not a violation of it. + * + * @param {{ outputs?: Record }} metafile + * @returns {BrowserOutput[]} + */ +export function browserOutputsFromMetafile(metafile) { + return Object.entries(metafile.outputs || {}) + .map(([file, output]) => { + const cut = Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\')) + return { + name: cut < 0 ? file : file.slice(cut + 1), + staticImports: (output.imports || []) + .filter((entry) => entry.kind === 'import-statement') + .map((entry) => entry.path), + } + }) + .filter((output) => !output.name.endsWith('.map')) +} + +/** + * @typedef {object} BrowserOutput + * @property {string} name Emitted file name, without directories. + * @property {string[]} [staticImports] Modules the output still imports with an `import … from` statement. + */ + +/** + * Check a browser build's outputs against the contract above. Split out of + * build-compiler.js so the failure modes it exists to catch can be tested + * directly (see scripts/test-browser-assets.js) instead of only by breaking a + * real build. + * + * Dynamic imports are fine — the stage worker imports the host's toolchain setup + * module by URL at runtime. A STATIC import means the output is no longer one + * self-contained file, so a host copying it alone ships a broken asset. + * + * @param {BrowserOutput[]} outputs every non-sourcemap output the browser build emitted + * @returns {string[]} one line per problem; empty when the contract holds + */ +export function checkBrowserAssetContract(outputs) { + const assetNames = COMPILER_BROWSER_ASSETS.map((asset) => asset.name) + const classified = new Set([...assetNames, ...BUNDLER_ONLY_BROWSER_OUTPUTS]) + const emitted = new Set(outputs.map((output) => output.name)) + const problems = [] + + for (const name of assetNames) { + if (!emitted.has(name)) { + problems.push(`browser-assets.js lists dist/${name}, but the browser build did not emit it`) + } + } + for (const output of outputs) { + if (!classified.has(output.name)) { + problems.push(`dist/${output.name} is a new browser output: add it to COMPILER_BROWSER_ASSETS (hosts must serve it) or to BUNDLER_ONLY_BROWSER_OUTPUTS (hosts reach it through their own bundler) in src/browser-assets.js`) + continue + } + if (!assetNames.includes(output.name)) continue + const staticImports = output.staticImports || [] + if (staticImports.length > 0) { + problems.push(`dist/${output.name} is served raw as a static asset but statically imports ${staticImports.join(', ')}`) + } + } + return problems +} + +/** + * @typedef {object} ResolvedBrowserAssets + * @property {string} dir Directory holding all of the assets. + * @property {string[]} files Absolute path of each asset, in COMPILER_BROWSER_ASSETS order. + */ + +/** + * Resolve the on-disk asset paths from the resolved path of the `./browser` + * entry — the one path every consumer can obtain without knowing this layout: + * + * resolveBrowserAssets(require.resolve('@dimina-kit/compiler/browser')) + * + * Preserves whichever path separator the input uses (POSIX or Windows), so the + * result is joinable and copyable as-is on the host platform. + * + * @param {string} browserEntryPath + * @returns {ResolvedBrowserAssets} + */ +export function resolveBrowserAssets(browserEntryPath) { + const cut = Math.max(browserEntryPath.lastIndexOf('/'), browserEntryPath.lastIndexOf('\\')) + if (cut < 0) throw new Error(`resolveBrowserAssets: not a path to compile-core.browser.js: ${browserEntryPath}`) + const sep = browserEntryPath[cut] + const dir = browserEntryPath.slice(0, cut) + return { dir, files: COMPILER_BROWSER_ASSETS.map((asset) => dir + sep + asset.name) } +} diff --git a/packages/compiler/tsconfig.types.json b/packages/compiler/tsconfig.types.json index 0eb9fc46..7c52769b 100644 --- a/packages/compiler/tsconfig.types.json +++ b/packages/compiler/tsconfig.types.json @@ -20,6 +20,7 @@ "src/pool.js", "src/pool-node.js", "src/stage-worker.js", - "src/toolchain.js" + "src/toolchain.js", + "src/browser-assets.js" ] } diff --git a/packages/compiler/types-fixture/consumer.ts b/packages/compiler/types-fixture/consumer.ts index b6c50887..23538026 100644 --- a/packages/compiler/types-fixture/consumer.ts +++ b/packages/compiler/types-fixture/consumer.ts @@ -5,6 +5,7 @@ // this file with "unused '@ts-expect-error' directive". import { collectOutputs, STAGE_NAMES } from '@dimina-kit/compiler' +import { COMPILER_BROWSER_ASSETS, resolveBrowserAssets } from '@dimina-kit/compiler/browser-assets' import { initToolchain } from '@dimina-kit/compiler/browser' import { createCompilerPool } from '@dimina-kit/compiler/pool' import { createNodeCompilerPool } from '@dimina-kit/compiler/pool-node' @@ -14,6 +15,13 @@ import { installOxc } from '@dimina-kit/compiler/toolchain' const stages: string[] = STAGE_NAMES void stages +const assetNames: string[] = COMPILER_BROWSER_ASSETS.map(asset => asset.name) +void assetNames +const assetDir: string = resolveBrowserAssets('/pkg/dist/compile-core.browser.js').dir +void assetDir +// @ts-expect-error resolveBrowserAssets takes the resolved entry path, not the asset list +resolveBrowserAssets(COMPILER_BROWSER_ASSETS) + const outputs: Record = collectOutputs({ fs: {}, targetPath: '/dist' }) void outputs // @ts-expect-error targetPath is required From 041e8f20e2aac138ee7c1a9dbd9cec5982594b77 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 22:50:41 +0800 Subject: [PATCH 04/14] =?UTF-8?q?fix(compiler):=20=E9=9D=99=E6=80=81?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E8=87=AA=E6=A3=80=E8=AE=A4=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E3=80=81=E8=AE=A4=E6=89=80=E6=9C=89=20import=20=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=EF=BC=8C=E5=B9=B6=E5=92=8C=20exports=20=E5=AF=B9?= =?UTF-8?q?=E8=B4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 评审指出三处会静默放行的漏洞：产物被挪进 dist 子目录后，只比对文件名的检查看不出 差别；externalize 的 CJS 依赖在 metafile 里记成 require-call，只筛 import-statement 的检查漏掉它；清单里的文件名和 package.json 的 exports 各写各的，改名只落一边不会 报错。 现在产物名保留 dist 以下的目录，任何 import 类型都算破坏自包含（stage worker 的 import(toolchainSetupURL) 不受影响：specifier 是运行时变量，esbuild 记不下来，真实 浏览器构建实测三个资源零 import），并新增 checkAssetsAgainstExports 双向对账。 README 里"必须原样托管"的说法收窄到自己按 URL 托管的宿主——通过包名 import pool、 用 new URL 解析 stage worker 的宿主，打包器看得见引用，本来就该由它处理。 --- packages/compiler/README.md | 12 +- packages/compiler/scripts/build-compiler.js | 21 ++- .../compiler/scripts/test-browser-assets.js | 71 ++++++---- packages/compiler/src/browser-assets.js | 121 +++++++++++++----- 4 files changed, 161 insertions(+), 64 deletions(-) diff --git a/packages/compiler/README.md b/packages/compiler/README.md index a56cab13..c8aaeeb7 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -39,11 +39,15 @@ import { createCompilerPool } from '@dimina-kit/compiler/pool' | `dist/stage-worker.browser.js` | 浏览器 Worker | 常驻 stage worker（已内联 core + memfs），pool 用它做并行 | | `dist/pool.browser.js` | 浏览器主线程 | 编排池 `createCompilerPool` | -### 浏览器静态资源：哪三个文件必须原样托管 +### 浏览器静态资源：自己按 URL 托管这三个文件时的规矩 -上表里带 `.browser.js` 的三个产物，宿主要**原样拷贝、原样托管**，不能再过一遍自己的打包器。stage worker 只被 `new Worker(url)` 引用，另外两个只被 fetch 下来从 Blob URL import——没有一处是打包器能看见的静态 import，所以打包器要么整个漏掉这些文件，要么把它们改写坏，两种情况都不报错，只在运行时 404 或行为异常。 +上表里带 `.browser.js` 的三个产物都是自包含的单文件 ESM，本身不 import 任何东西。 -文件名由本包给出，不要在宿主里手抄： +**宿主自己按 URL 托管它们时**（拷进 `public/` 再 `new Worker('/stage-worker.browser.js')`，或 fetch 下来从 Blob URL import），要**原样拷贝、按这些文件名托管**，不要再过一遍自己的打包器：宿主源码里没有一处是打包器能看见的静态引用，打包器根本不知道这些文件存在，既不会拷也不会改写，缺失只在运行时表现为 404。 + +**宿主通过包名引用它们时**（`import { createCompilerPool } from '@dimina-kit/compiler/pool'`，或 `new Worker(new URL('@dimina-kit/compiler/stage-worker', import.meta.url))`，见下面的接入示例），打包器看得见这条引用、会自己处理，这一节整节都用不上。 + +按 URL 托管的话，文件名由本包给出，不要在宿主里手抄： ```js const { resolveBrowserAssets } = require('@dimina-kit/compiler/browser-assets') @@ -54,7 +58,7 @@ for (const file of files) fs.copyFileSync(file, path.join(publicDir, path.basena `./browser-assets` 同时发 ESM 和 CJS（走 `require` 条件），只做字符串拼接，不依赖 `node:path`。 -每次浏览器构建都会拿 esbuild 的 metafile 对着这份清单自检：产物改了名、被拆出新 chunk、或某个静态资源开始带静态 import（不再是自包含的单文件），构建当场失败，而不是几个月后在某个宿主那里 404。`toolchain.browser.js` 不在清单里——它由宿主用自己的打包器 import（`@dimina-kit/compiler/toolchain`），拷过去也没人 fetch。 +每次浏览器构建都会拿 esbuild 的 metafile 对着这份清单自检：产物改了名、被挪进子目录、被拆出新 chunk、或某个资源开始 import 别的文件（`import` / `require` / 动态 import 都算，一带上就不再是自包含的单文件），以及清单里的文件名和 `package.json` 的 exports 对不上（改名只改了一边），构建当场失败，而不是几个月后在某个宿主那里 404。`toolchain.browser.js` 不在清单里——它由宿主用自己的打包器 import（`@dimina-kit/compiler/toolchain`），拷过去也没人 fetch。 ## 架构 diff --git a/packages/compiler/scripts/build-compiler.js b/packages/compiler/scripts/build-compiler.js index addd371f..b25d894c 100644 --- a/packages/compiler/scripts/build-compiler.js +++ b/packages/compiler/scripts/build-compiler.js @@ -3,7 +3,7 @@ import { readFile } from 'node:fs/promises' import { fileURLToPath } from 'node:url' import { createRequire } from 'node:module' import path from 'node:path' -import { COMPILER_BROWSER_ASSETS, browserOutputsFromMetafile, checkBrowserAssetContract } from '../src/browser-assets.js' +import { COMPILER_BROWSER_ASSETS, browserOutputsFromMetafile, checkAssetsAgainstExports, checkBrowserAssetContract } from '../src/browser-assets.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) const root = path.resolve(__dirname, '..') @@ -258,15 +258,22 @@ for (const [format, outfile] of [['esm', 'browser-assets.js'], ['cjs', 'browser- } console.log('✅ built dist/browser-assets.js + dist/browser-assets.cjs') -// The browser bundles double as static assets a host copies and serves. Their names -// and the "no static imports" rule are stated once in src/browser-assets.js, and -// enforced here so a rename or a newly split chunk fails the build instead of -// 404-ing (or half-loading) inside a host months later. +// The browser bundles double as static files a host copies and serves. Their names +// and the "self-contained, imports nothing" rule are stated once in +// src/browser-assets.js, and enforced here so a rename or a newly split chunk fails +// the build instead of 404-ing (or half-loading) inside a host months later. if (MODE === 'browser') { - const problems = checkBrowserAssetContract(browserOutputsFromMetafile({ outputs: browserMetafileOutputs })) + // esbuild keys the metafile by paths relative to the working directory, so the + // asset check sees the same `dist/…` prefix a host would copy from — and an + // output that lands in a subdirectory keeps that subdirectory in its name. + const outdirPrefix = `${path.relative(process.cwd(), path.join(root, 'dist')).split(path.sep).join('/')}/` + const problems = [ + ...checkBrowserAssetContract(browserOutputsFromMetafile({ outputs: browserMetafileOutputs }, outdirPrefix)), + ...checkAssetsAgainstExports(JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8')).exports), + ] if (problems.length > 0) { console.error(problems.map((line) => ` ✗ ${line}`).join('\n')) process.exit(1) } - console.log(`✅ browser static-asset contract holds (${COMPILER_BROWSER_ASSETS.length} assets, no static imports)`) + console.log(`✅ browser static-asset contract holds (${COMPILER_BROWSER_ASSETS.length} assets, self-contained, names match the exports map)`) } diff --git a/packages/compiler/scripts/test-browser-assets.js b/packages/compiler/scripts/test-browser-assets.js index be2db3ac..5bb15e20 100644 --- a/packages/compiler/scripts/test-browser-assets.js +++ b/packages/compiler/scripts/test-browser-assets.js @@ -11,6 +11,7 @@ import { BUNDLER_ONLY_BROWSER_OUTPUTS, COMPILER_BROWSER_ASSETS, browserOutputsFromMetafile, + checkAssetsAgainstExports, checkBrowserAssetContract, resolveBrowserAssets, } from '../src/browser-assets.js' @@ -22,8 +23,8 @@ const chk = (cond, msg) => { if (!cond) { failed = true; console.error(`❌ ${ms const assetNames = COMPILER_BROWSER_ASSETS.map((asset) => asset.name) const healthy = [ - ...assetNames.map((name) => ({ name, staticImports: [] })), - ...BUNDLER_ONLY_BROWSER_OUTPUTS.map((name) => ({ name, staticImports: [] })), + ...assetNames.map((name) => ({ name, imports: [] })), + ...BUNDLER_ONLY_BROWSER_OUTPUTS.map((name) => ({ name, imports: [] })), ] chk(checkBrowserAssetContract(healthy).length === 0, 'a build that emits exactly the listed outputs passes') @@ -48,37 +49,63 @@ chk( ) chk( - checkBrowserAssetContract([...healthy, { name: 'compile-core.browser-chunk.js', staticImports: [] }]) + checkBrowserAssetContract([...healthy, { name: 'compile-core.browser-chunk.js', imports: [] }]) .some((line) => line.includes('new browser output')), 'a newly split chunk is reported until it is classified', ) -chk( - checkBrowserAssetContract( - healthy.map((output) => (output.name === 'stage-worker.browser.js' - ? { ...output, staticImports: ['./chunk-ABC.js'] } - : output)), - ).some((line) => line.includes('statically imports ./chunk-ABC.js')), - 'an asset that stopped being self-contained is reported', -) +// An asset only works when it is one file with nothing to fetch alongside it, so +// every import kind breaks it — not just `import … from`. An externalized CJS +// dependency shows up as require-call, a lazily pulled chunk as dynamic-import. +for (const kind of ['import-statement', 'require-call', 'dynamic-import']) { + chk( + checkBrowserAssetContract( + healthy.map((output) => (output.name === 'stage-worker.browser.js' + ? { ...output, imports: [{ path: './chunk-ABC.js', kind }] } + : output)), + ).some((line) => line.includes('not self-contained') && line.includes(`./chunk-ABC.js (${kind})`)), + `an asset that imports something (${kind}) is reported`, + ) +} -// What the build actually feeds the check: esbuild's metafile, keyed by path, -// carrying both import kinds plus sourcemap entries. +// An output emitted into a subdirectory is NOT the dist-root file a host copies, +// so it must not pass as one. +const nested = checkBrowserAssetContract([ + ...healthy.filter((output) => output.name !== 'pool.browser.js'), + { name: 'assets/pool.browser.js', imports: [] }, +]) +chk(nested.some((line) => line.includes('dist/pool.browser.js') && line.includes('did not emit it')), 'an asset moved into a subdirectory is reported as missing from dist') +chk(nested.some((line) => line.includes('dist/assets/pool.browser.js') && line.includes('new browser output')), 'the subdirectory copy is reported under its full path') + +// What the build actually feeds the check: esbuild's metafile, keyed by paths +// relative to the working directory, plus sourcemap entries. const shaped = browserOutputsFromMetafile({ outputs: { - 'dist/stage-worker.browser.js': { - imports: [ - { path: 'toolchainSetupURL', kind: 'dynamic-import' }, - { path: './chunk-XYZ.js', kind: 'import-statement' }, - ], - }, + 'dist/stage-worker.browser.js': { imports: [{ path: './chunk-XYZ.js', kind: 'import-statement' }] }, + 'dist/assets/pool.browser.js': { imports: [] }, 'dist/stage-worker.browser.js.map': { imports: [] }, }, }) -chk(shaped.length === 1 && shaped[0].name === 'stage-worker.browser.js', 'metafile paths are reduced to file names, sourcemaps dropped') chk( - shaped[0].staticImports.length === 1 && shaped[0].staticImports[0] === './chunk-XYZ.js', - 'a dynamic import (the host toolchain setup URL) is not counted; a static one is', + shaped.map((output) => output.name).join('|') === 'stage-worker.browser.js|assets/pool.browser.js', + 'metafile paths keep the directory below dist, sourcemaps dropped', +) +chk( + shaped[0].imports.length === 1 && shaped[0].imports[0].kind === 'import-statement', + 'each import is carried through with its kind', +) + +// The manifest and the exports map name the same files; a rename has to land in both. +chk(checkAssetsAgainstExports(pkg.exports).length === 0, "this package's own exports map agrees with the manifest") +chk( + checkAssetsAgainstExports({ './pool': { default: './dist/pool-v2.browser.js' } }) + .some((line) => line.includes('pool-v2.browser.js') && line.includes('does not list')), + 'an export pointing at an unlisted browser bundle is reported', +) +chk( + checkAssetsAgainstExports({ './pool': { default: './dist/pool-v2.browser.js' } }) + .some((line) => line.includes('pool.browser.js') && line.includes('one half of a rename')), + 'a listed asset no export points at is reported', ) // resolveBrowserAssets is the path every consumer derives from the ./browser entry. diff --git a/packages/compiler/src/browser-assets.js b/packages/compiler/src/browser-assets.js index 9a77433d..84105bec 100644 --- a/packages/compiler/src/browser-assets.js +++ b/packages/compiler/src/browser-assets.js @@ -4,18 +4,27 @@ * dev/build server, a `copy-web-compiler` script): * * The files listed in COMPILER_BROWSER_ASSETS are single-file ESM bundles - * with no static imports, sitting side by side in dist/ under exactly these - * names. Hosts must serve them RAW — running them through a bundler again - * breaks them, and nothing forces the host to notice: the stage worker is - * only ever `new Worker(url)`'d and the other two are only ever fetched and - * imported from a Blob URL, so no bundler ever sees the reference and - * rewrites it. + * that import nothing at all, sitting side by side in dist/ under exactly + * these names. * - * build-compiler.js asserts both halves at build time out of esbuild's own + * That contract is for the host that loads them BY URL — a static file it + * serves itself. Nothing in such a host's source references these files (the + * stage worker is only ever `new Worker(url)`'d, the other two are fetched and + * imported from a Blob URL), so its bundler never sees them: it copies nothing, + * rewrites nothing, and the miss surfaces as a 404 at runtime. That host has to + * copy them byte for byte, under these names, and serve them as they are. + * + * A host that instead reaches them through the package — `import + * { createCompilerPool } from '@dimina-kit/compiler/pool'`, or + * `new Worker(new URL('@dimina-kit/compiler/stage-worker', import.meta.url))` — + * hands its bundler a reference it can follow, and needs none of this. + * + * build-compiler.js asserts all of it at build time out of esbuild's own * metafile: every browser output is classified here (static asset or - * bundler-only), and every static asset really has zero static imports. So - * renaming or splitting an output without updating this list fails the build, - * instead of 404-ing in some host months later. + * bundler-only), every static asset really is import-free, and the names here + * still match what package.json's exports map points at. So renaming or + * splitting an output without updating this list fails the build, instead of + * 404-ing in some host months later. * * Pure string manipulation (no node:path) so the module loads in any runtime, * and it ships as both ESM (dist/browser-assets.js) and CJS @@ -26,10 +35,10 @@ /** * @typedef {object} CompilerBrowserAsset * @property {string} name File name in dist/, and the name to serve it under. - * @property {string} loadedBy How the browser gets it — why it must stay raw. + * @property {string} loadedBy How a URL-loading browser host gets it. */ -/** The browser bundles a host has to host as static assets. */ +/** The browser bundles a host that loads them by URL has to serve as static files. */ export const COMPILER_BROWSER_ASSETS = /** @type {readonly CompilerBrowserAsset[]} */ ([ { name: 'stage-worker.browser.js', @@ -56,23 +65,26 @@ export const BUNDLER_ONLY_BROWSER_OUTPUTS = /** @type {readonly string[]} */ ([' /** * Shape esbuild's metafile into the output list {@link checkBrowserAssetContract} - * takes: sourcemaps dropped, and only `import-statement` imports kept. Dynamic - * imports are deliberately ignored — the stage worker reaches the host's toolchain - * setup module through `import(toolchainSetupURL)` at runtime, which is the - * contract, not a violation of it. + * takes: sourcemaps dropped, and `name` kept as the path RELATIVE TO dist — + * `assets/stage-worker.browser.js` must not pass as the dist-root file hosts + * actually copy, so the directory is part of the name, not stripped from it. + * + * Every recorded import is carried through with its kind. The check rejects all + * of them (see below), so nothing here decides what counts as a violation. * * @param {{ outputs?: Record }} metafile + * @param {string} [outdirPrefix] the metafile keys' prefix for the output directory; + * a key outside it keeps its whole path and is then reported as unclassified. * @returns {BrowserOutput[]} */ -export function browserOutputsFromMetafile(metafile) { +export function browserOutputsFromMetafile(metafile, outdirPrefix = 'dist/') { + const prefix = outdirPrefix.endsWith('/') ? outdirPrefix : `${outdirPrefix}/` return Object.entries(metafile.outputs || {}) .map(([file, output]) => { - const cut = Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\')) + const key = file.split('\\').join('/') return { - name: cut < 0 ? file : file.slice(cut + 1), - staticImports: (output.imports || []) - .filter((entry) => entry.kind === 'import-statement') - .map((entry) => entry.path), + name: key.startsWith(prefix) ? key.slice(prefix.length) : key, + imports: (output.imports || []).map((entry) => ({ path: entry.path, kind: entry.kind })), } }) .filter((output) => !output.name.endsWith('.map')) @@ -80,8 +92,10 @@ export function browserOutputsFromMetafile(metafile) { /** * @typedef {object} BrowserOutput - * @property {string} name Emitted file name, without directories. - * @property {string[]} [staticImports] Modules the output still imports with an `import … from` statement. + * @property {string} name Emitted file, as a path relative to dist. + * @property {{ path: string, kind: string }[]} [imports] Everything the output still + * references at module level, as esbuild's metafile records it (kind is + * `import-statement`, `require-call`, `dynamic-import`, …). */ /** @@ -90,9 +104,13 @@ export function browserOutputsFromMetafile(metafile) { * directly (see scripts/test-browser-assets.js) instead of only by breaking a * real build. * - * Dynamic imports are fine — the stage worker imports the host's toolchain setup - * module by URL at runtime. A STATIC import means the output is no longer one - * self-contained file, so a host copying it alone ships a broken asset. + * An asset that still imports anything — under any kind — is no longer one + * self-contained file, so a host copying it alone ships something that 404s or + * half-loads. `import-statement` is only the most obvious kind: an externalized + * CommonJS dependency is recorded as `require-call`, and a split chunk pulled in + * lazily as `dynamic-import`. All of them are rejected. The stage worker's + * `import(toolchainSetupURL)` is not among them: its specifier is a runtime + * variable, so esbuild cannot resolve it and records no import for it. * * @param {BrowserOutput[]} outputs every non-sourcemap output the browser build emitted * @returns {string[]} one line per problem; empty when the contract holds @@ -110,13 +128,54 @@ export function checkBrowserAssetContract(outputs) { } for (const output of outputs) { if (!classified.has(output.name)) { - problems.push(`dist/${output.name} is a new browser output: add it to COMPILER_BROWSER_ASSETS (hosts must serve it) or to BUNDLER_ONLY_BROWSER_OUTPUTS (hosts reach it through their own bundler) in src/browser-assets.js`) + problems.push(`dist/${output.name} is a new browser output: add it to COMPILER_BROWSER_ASSETS (hosts serve it themselves) or to BUNDLER_ONLY_BROWSER_OUTPUTS (hosts reach it through their own bundler) in src/browser-assets.js`) continue } if (!assetNames.includes(output.name)) continue - const staticImports = output.staticImports || [] - if (staticImports.length > 0) { - problems.push(`dist/${output.name} is served raw as a static asset but statically imports ${staticImports.join(', ')}`) + for (const entry of output.imports || []) { + problems.push(`dist/${output.name} is served as a standalone static file but is not self-contained: it still imports ${entry.path} (${entry.kind})`) + } + } + return problems +} + +/** @param {unknown} node @param {string[]} out @returns {string[]} */ +function collectExportTargets(node, out) { + if (typeof node === 'string') { out.push(node); return out } + if (node && typeof node === 'object') { + for (const value of Object.values(node)) collectExportTargets(value, out) + } + return out +} + +/** + * Cross-check this manifest against the package's own exports map. Both describe + * the same files under the same names, and a rename that lands in only one of + * them is silent: the manifest still resolves paths that no longer exist, or + * `import '@dimina-kit/compiler/pool'` still points at a file the build no longer + * emits. So every `.browser.js` an export points at must be classified here, and + * every name classified here must still be some export's target. + * + * @param {unknown} exportsMap the package.json `exports` object + * @param {string} [distPrefix] how those targets spell the dist directory + * @returns {string[]} one line per problem; empty when both sides agree + */ +export function checkAssetsAgainstExports(exportsMap, distPrefix = './dist/') { + const classified = [...COMPILER_BROWSER_ASSETS.map((asset) => asset.name), ...BUNDLER_ONLY_BROWSER_OUTPUTS] + const targets = new Set( + collectExportTargets(exportsMap, []) + .filter((target) => target.endsWith('.browser.js')) + .map((target) => (target.startsWith(distPrefix) ? target.slice(distPrefix.length) : target)), + ) + const problems = [] + for (const target of targets) { + if (!classified.includes(target)) { + problems.push(`package.json exports point at ${distPrefix}${target}, which src/browser-assets.js does not list — add it to COMPILER_BROWSER_ASSETS or BUNDLER_ONLY_BROWSER_OUTPUTS`) + } + } + for (const name of classified) { + if (!targets.has(name)) { + problems.push(`src/browser-assets.js lists ${name}, but no package.json exports entry points at ${distPrefix}${name} — one half of a rename is missing`) } } return problems From e7c7fba9b1754102e26750883af12c766d94de86 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 19:49:37 +0800 Subject: [PATCH 05/14] =?UTF-8?q?feat(compiler):=20=E7=BB=99=20pool=20?= =?UTF-8?q?=E7=9A=84=E5=A4=B1=E8=B4=A5=E4=B8=80=E4=B8=AA=E7=A8=B3=E5=AE=9A?= =?UTF-8?q?=E7=9A=84=E9=94=99=E8=AF=AF=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 宿主要区分「工具链没加载上」和「用户项目编不过」，只能去匹配错误信息文本—— worker 里 `import(toolchainSetupURL)` 失败后，pool 把它和普通编译错误一样记成 compiler-stage-error，两者从外面看没有区别。措辞一改，宿主的判断就静默失效。 - 新增 src/error-codes.js：`COMPILER_ERROR_CODES` 七个码 + `isInfrastructureError`， 两个 pool 各自再导出，宿主 import 常量而不是抄字符串。 - stage worker 给工具链导入失败打上 compiler-toolchain-setup-failed，并把码放进 `{ type:'error' }` 回复；pool 原样转发，没带码的才记为 compiler-stage-error。 - pool.js / pool-node.js / worker-slot.js 改用同一张表，不再各写各的字面量。 回归测试 scripts/test-error-codes.js（不需要构建，已进包的 test）：去掉 pool 那句 转发后，三条断言立刻失败。test-stage-toolchain.js 补一条断言，验证真实 worker bundle 确实带上了这个码。 Co-Authored-By: Claude Opus 5 (1M context) --- packages/compiler/README.md | 34 +++- packages/compiler/package.json | 3 +- packages/compiler/scripts/test-error-codes.js | 153 ++++++++++++++++++ .../compiler/scripts/test-stage-toolchain.js | 17 ++ packages/compiler/src/error-codes.js | 61 +++++++ packages/compiler/src/pool-node.js | 11 +- packages/compiler/src/pool.js | 12 +- packages/compiler/src/stage-worker.js | 13 +- packages/compiler/src/worker-slot.js | 11 +- packages/compiler/tsconfig.types.json | 1 + 10 files changed, 299 insertions(+), 17 deletions(-) create mode 100644 packages/compiler/scripts/test-error-codes.js create mode 100644 packages/compiler/src/error-codes.js diff --git a/packages/compiler/README.md b/packages/compiler/README.md index c8aaeeb7..b5a731b8 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -381,6 +381,35 @@ async function filesFromDir(dir, prefix = '') { // 只读递来的 han - **`targetPath` 来自环境。** compiler 产物目录取 `process.env.TARGET_PATH`，否则 `os.tmpdir()/dimina-fe-dist-<时间戳>`（浏览器 os shim 下通常 `/tmp/...`）。`setupCompile` 会先 `rmSync` 清空它——别把 `TARGET_PATH` 指到源码目录或共享目录。用 `setupCompile` 返回的 `targetPath` 喂 `collectOutputs`。 - **不是全 fail-fast。** 缺 fs 方法、坏 appid、坏 `project.config.json`、miniprogram_npm 构建失败会 **reject**；但**样式预处理器失败（如当前浏览器构建暂不支持 `.less`）会被吞掉、降级用原始 CSS**，PostCSS 解析失败返回空串，资源拷贝失败只 `console.log`，logic esbuild 压缩失败回退未压缩代码。**用 pool 时把这些拿出来的办法是 `createCompilerPool({ onLog })`**——它把 worker 内编译器的 `console.*` 诊断（带 stage 标签）转发给你；也可在产物为空/缺失时二次校验。 +### 错误码：怎么判断该重试还是该把错误给用户看 + +两个 pool（`./pool` 与 `./pool-node`）reject 的每个错误都带 `err.code`，取值就是 `COMPILER_ERROR_CODES` 这张表；两个 pool 都把它连同判定函数一起再导出，宿主不用抄字符串： + +```js +import { COMPILER_ERROR_CODES, isInfrastructureError } from '@dimina-kit/compiler/pool' + +try { + await pool.compile({ files }) +} catch (err) { + if (isInfrastructureError(err)) fallbackToAnotherCompilePath(err) // 机器坏了，换条路还有戏 + else showToUser(err.message) // 项目本身编不过，重试没用 +} +``` + +| code | 含义 | 算基础设施故障 | +| --- | --- | --- | +| `compiler-stage-error` | 编译器拒绝了这个项目：源码或配置要用户自己改 | 否 | +| `compiler-toolchain-setup-failed` | stage worker `import(toolchainSetupURL)` 失败：模块 404、断网、wasm 拿不到 | 是 | +| `compiler-worker-timeout` | worker 静默超过不活动窗口（卡死的 wasm 循环连心跳都发不出） | 是 | +| `compiler-worker-crashed` | worker 挂了：`error` 事件、`postMessage` 抛错、异常退出 | 是 | +| `compiler-worker-dead` | 请求打到了已判死、还没重建的 slot | 是 | +| `compiler-toolchain-dead` | 仅 Node：esbuild 常驻服务进程没了，该 realm 之后每次调用都失败 | 是 | +| `compiler-pool-disposed` | 池已回收，不会再编译任何东西 | 否 | + +`isInfrastructureError` 之外还有一层：worker 死亡类的三个码（timeout / crashed / dead，Node 上再加 toolchain-dead）由 pool 自己用来做那一次透明重试，宿主一般不用关心。 + +工具链导入失败的码是 **worker 自己打的**——只有它能区分「宿主的 wasm 资源没加载上」和「用户项目编不过」，两者到 pool 手里都是同一种 `{ type:'error' }` 回复。pool 原样转发这个码，其余没带码的一律记为 `compiler-stage-error`。 + ## 依赖前置 编译器实体源码在 `dimina` 子模块里，dart-sass 等在其 fe workspace。构建前确保子模块已初始化、依赖已装： @@ -401,7 +430,7 @@ pnpm --filter @dimina-kit/compiler build:types # 仅 dist/types/*.d.ts ## 测试 -`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）只包含不需要构建的静态资源契约测试；下面这些各自要先构建，按需单跑。 +`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）只包含不需要构建的两份契约测试——静态资源清单（`test:browser-assets`）和错误码（`test:error-codes`）；下面这些各自要先构建，按需单跑。 测试里用 memfs 扮演「下游 fs」： @@ -417,6 +446,8 @@ pnpm --filter @dimina-kit/compiler test:idle-shrink # idle 收缩:超时终 pnpm --filter @dimina-kit/compiler test:stage-worker-message-order # stage worker 回复严格按请求到达序串行(build 在途时 introspect 不抢 FIFO 配对) pnpm --filter @dimina-kit/compiler test:stage-load-retry # stage 工具链加载失败不缓存,chunk 恢复后 preloadStage 可重试成功 pnpm --filter @dimina-kit/compiler test:browser-assets # 静态资源清单:改名/新 chunk/出现静态 import 都会被构建期检查拦下 +pnpm --filter @dimina-kit/compiler test:error-codes # 错误码:worker 自己判定的失败(工具链导入)带着码原样传到调用方,其余记为 compiler-stage-error +pnpm --filter @dimina-kit/compiler test:stage-toolchain # 真实 stage worker bundle:每个 stage 都加载工具链、按 URL 记忆、导入失败带错误码 ``` `pool` 的浏览器端到端验证在 `dimina-web-client`（`npm run test:pool`，Playwright 驱动，产物与单线程逐结构一致）。`pool-node` 的宿主端验证在 `@dimina-kit/devkit` 测试套件（真实 fork + `openProject`）。 @@ -431,6 +462,7 @@ pnpm --filter @dimina-kit/compiler test:browser-assets # 静态资 - `src/stage-worker-node.js` — Node 常驻 stage worker：spawn 时按 workerData 里的 stage 身份预热本 stage 工具链，恢复 storeInfo → `runStage(stage, { sourcemap })` 写共享 staging 目录；应答 `{ type: 'introspect' }` 报告本 realm 已加载的重依赖。 - `src/toolchain.js` — 写 `toolchainSetupURL` 模块的可选助手（`installOxc` / `installEsbuildFromURL`，后者内置 esbuild-wasm 静态资源的 Blob-URL 兜底）。导出为 `@dimina-kit/compiler/toolchain`。 - `src/browser-assets.js` — 浏览器静态资源清单与契约（`COMPILER_BROWSER_ASSETS` / `resolveBrowserAssets`，见上文），构建期检查也用它。导出为 `@dimina-kit/compiler/browser-assets`。 +- `src/error-codes.js` — 两个 pool 共用的错误码表 `COMPILER_ERROR_CODES` 与判定 `isInfrastructureError`（见上文），经 `./pool` 与 `./pool-node` 再导出。 - `src/shims/fs.js` — **无后端的 fs 转发层**（`setFs`/`resetFs`/`getFs`）；compiler 所有 `fs.xxx` 走它，未注入即抛错。 - `src/shims/*` — 其余 node 内置与原生依赖的浏览器替身（oxc/esbuild/less/`os.homedir`/…）。 - `scripts/build-compiler.js` — esbuild 打包。onLoad 给 logic/view/style-compiler 与 utils 追加 `__reset*` 导出（喂 `resetCompilerState`，不改子模块源码）；浏览器分支内联真实 `cssnano`+`autoprefixer`（autoprefixer pin 到 node 运行时解析的同一份，避免 esbuild 解析到多加 `-ms-` 前缀的另一版本）；browser 模式产出 core / stage-worker / pool 三个单文件 bundle，并按 metafile 对 `src/browser-assets.js` 的清单自检（漏产物、多产物、静态资源出现静态 import 都直接失败）；node 模式开 `splitting`（stage 编译器成为运行时 chunk——单文件会把 chunk 的 external `import 'sass'` 提升回入口顶层，懒加载会静默失效）。 diff --git a/packages/compiler/package.json b/packages/compiler/package.json index e7d93901..8f06f748 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -69,7 +69,8 @@ "test:realm-reuse": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-realm-reuse.js", "test:pool-node": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-node.js", "test:pool-scopehash": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-scopehash.js", - "test": "pnpm run test:browser-assets", + "test": "pnpm run test:browser-assets && pnpm run test:error-codes", + "test:error-codes": "node scripts/test-error-codes.js", "test:browser-assets": "node scripts/test-browser-assets.js", "test:crypto-shim": "node scripts/test-crypto-shim.js", "test:pool-hardening": "node scripts/build-compiler.js node && node scripts/test-pool-hardening.js", diff --git a/packages/compiler/scripts/test-error-codes.js b/packages/compiler/scripts/test-error-codes.js new file mode 100644 index 00000000..68e73d70 --- /dev/null +++ b/packages/compiler/scripts/test-error-codes.js @@ -0,0 +1,153 @@ +// Every pool rejection carries `err.code`, and a code the WORKER chose survives the trip +// out to the caller. That second half is what this test protects: a stage worker is the +// only place that can tell "the host's wasm toolchain didn't load" apart from "this +// project doesn't compile" — both arrive at the pool as the same { type:'error' } reply. +// If the pool overwrote the worker's code with the generic compiler-stage-error, a host +// would be back to matching error-message text to decide whether retrying or falling back +// to another compile path is worth it. +// +// Drives the REAL src/pool.js against fake Worker-shaped objects (no build, no browser), +// the same technique as scripts/test-pool-worker-hardening.js. +import { + COMPILER_ERROR_CODES, + INFRASTRUCTURE_ERROR_CODES, + WORKER_DEATH_CODES, + isInfrastructureError, +} from '../src/error-codes.js' +import { + COMPILER_ERROR_CODES as POOL_CODES, + createCompilerPool, + isInfrastructureError as poolIsInfrastructureError, +} from '../src/pool.js' + +let failed = false +const chk = (cond, msg) => { if (!cond) { failed = true; console.error(`❌ ${msg}`) } else console.log(`✅ ${msg}`) } + +const STAGES = ['logic', 'view', 'style'] +const FILES = { 'app.json': '{"pages":["pages/index/index"]}' } + +// A fake Worker that answers every message normally except the one message `failOn` +// names, which it answers with a worker-side error reply carrying `code` (undefined for +// a plain compile error). `spawns` counts constructions so a test can prove the pool did +// not retry. +function fakeWorkerFactory({ failOn, code, message = 'boom' }) { + const spawns = { count: 0 } + const createWorker = () => { + spawns.count += 1 + const w = { + onmessage: null, + onerror: null, + onmessageerror: null, + terminate() {}, + postMessage(m) { + queueMicrotask(() => { + let resp + if (m.type === failOn) resp = { type: 'error', error: message, code } + else if (m.type === 'warmup') resp = { type: 'ready', ms: 1 } + else if (m.type === 'setup') { + resp = { + type: 'setup-done', + bundle: { appId: 'app1', name: 'demo', pages: [], storeInfo: {}, targetPath: '/work/dist' }, + scaffold: { 'app-config.json': '{}' }, + } + } else if (m.type === 'compile-subset') { + resp = { type: 'done', result: { appId: 'app1', name: 'demo', files: { [`${m.stages[0]}.out`]: 'x' } } } + } + if (w.onmessage) w.onmessage({ data: resp }) + }) + }, + } + return w + } + return { createWorker, spawns } +} + +async function compileAgainst(config) { + const { createWorker, spawns } = fakeWorkerFactory(config) + const pool = createCompilerPool({ + createWorker, + toolchainSetupURL: 'data:text/javascript,export default {}', + stages: STAGES, + }) + try { + const files = await pool.compile({ files: FILES }) + return { ok: true, files, spawns } + } catch (err) { + return { ok: false, err, spawns } + } finally { + await pool.dispose() + } +} + +async function main() { + // --- the constants themselves ------------------------------------------------- + { + const values = Object.values(COMPILER_ERROR_CODES) + chk(new Set(values).size === values.length, `every code is a distinct string (${values.length} codes)`) + chk(Object.isFrozen(COMPILER_ERROR_CODES), 'COMPILER_ERROR_CODES is frozen — a host cannot mutate the shared table') + chk(!INFRASTRUCTURE_ERROR_CODES.has(COMPILER_ERROR_CODES.stageError), + 'a compile error is NOT infrastructure — retrying a broken project only hides the diagnostic') + chk([...WORKER_DEATH_CODES].every((c) => INFRASTRUCTURE_ERROR_CODES.has(c)), + 'every worker-death code is also an infrastructure code') + chk(isInfrastructureError({ code: COMPILER_ERROR_CODES.toolchainSetupFailed }) + && isInfrastructureError({ code: COMPILER_ERROR_CODES.toolchainDead }), + 'isInfrastructureError() accepts a failed toolchain import and a dead toolchain service') + chk(!isInfrastructureError({ code: COMPILER_ERROR_CODES.stageError }) + && !isInfrastructureError(null) && !isInfrastructureError('compiler-worker-dead') && !isInfrastructureError(new Error('x')), + 'isInfrastructureError() rejects compile errors, non-objects and uncoded errors') + chk(POOL_CODES === COMPILER_ERROR_CODES && poolIsInfrastructureError === isInfrastructureError, + 'the pool re-exports the same table and predicate, so a host never copies the strings') + } + + // --- a worker-classified failure keeps its own code all the way out ------------ + { + const r = await compileAgainst({ + failOn: 'warmup', + code: COMPILER_ERROR_CODES.toolchainSetupFailed, + message: '[compiler] toolchain setup failed importing https://host/toolchain.js: Failed to fetch', + }) + chk(!r.ok && r.err.code === COMPILER_ERROR_CODES.toolchainSetupFailed, + `a warmup that fails to import the toolchain rejects with compiler-toolchain-setup-failed (got ${r.ok ? 'resolved' : r.err.code})`) + chk(!r.ok && isInfrastructureError(r.err), + 'that rejection is classified as infrastructure, so a host can fall back to another compile path') + chk(!r.ok && String(r.err.message).includes('Failed to fetch'), + `the worker's own message survives (got ${r.ok ? '' : JSON.stringify(String(r.err.message).slice(0, 80))})`) + chk(r.spawns.count === STAGES.length, + `an error REPLY is not worker death: the pool does not respawn or retry (spawned ${r.spawns.count}, expected ${STAGES.length})`) + } + + // --- an uncoded worker error is the project's fault ---------------------------- + { + const r = await compileAgainst({ failOn: 'setup', message: 'SyntaxError: Unexpected token in app.json' }) + chk(!r.ok && r.err.code === COMPILER_ERROR_CODES.stageError, + `a worker error reply with no code defaults to compiler-stage-error (got ${r.ok ? 'resolved' : r.err.code})`) + chk(!r.ok && !isInfrastructureError(r.err), 'a compile error is never retried on fresh machinery') + chk(!r.ok && r.err.stage === STAGES[0], `the rejection still names the stage that reported it (got ${r.ok ? '' : r.err.stage})`) + } + + // --- the same propagation on the compile step, not just warmup ----------------- + { + const r = await compileAgainst({ + failOn: 'compile-subset', + code: COMPILER_ERROR_CODES.toolchainSetupFailed, + message: '[compiler] toolchain setup failed importing https://host/toolchain.js: 404', + }) + chk(!r.ok && r.err.code === COMPILER_ERROR_CODES.toolchainSetupFailed, + `a toolchain import that fails at compile-subset time keeps its code too (got ${r.ok ? 'resolved' : r.err.code})`) + } + + // --- a disposed pool says so, rather than looking like a worker fault ---------- + { + const { createWorker } = fakeWorkerFactory({ failOn: null }) + const pool = createCompilerPool({ createWorker, toolchainSetupURL: 'data:text/javascript,export default {}', stages: STAGES }) + await pool.dispose() + const err = await pool.compile({ files: FILES }).then(() => null, (e) => e) + chk(!!err && err.code === COMPILER_ERROR_CODES.poolDisposed, + `compile() after dispose() rejects with compiler-pool-disposed (got ${err && err.code})`) + } + + console.log(failed ? '\n❌ error-code assertions failed.' : '\n✅ pool rejections carry stable codes, and a worker-classified failure keeps its own.') + process.exit(failed ? 1 : 0) +} + +await main() diff --git a/packages/compiler/scripts/test-stage-toolchain.js b/packages/compiler/scripts/test-stage-toolchain.js index 4fe0aeab..226123a2 100644 --- a/packages/compiler/scripts/test-stage-toolchain.js +++ b/packages/compiler/scripts/test-stage-toolchain.js @@ -166,6 +166,23 @@ function findCompiledCss(files) { chk(!!css, `style-only compile-subset produced a real compiled CSS product (found "${css && css[0]}": ${css && JSON.stringify(css[1])})`) } +// --- B2: a toolchain the worker cannot import is reported with a code, not just a +// message. The pool forwards that code untouched, so a host decides "the wasm assets +// didn't load, fall back" without matching the message text (see src/error-codes.js). +{ + const worker = await loadWorkerInstance() + const reply = await worker.send({ + type: 'warmup', + toolchainSetupURL: UNREACHABLE_TOOLCHAIN_URL, + stages: ['logic'], + }) + chk(reply && reply.type === 'error', `an unimportable toolchainSetupURL fails warmup — got ${JSON.stringify(reply && reply.type)}`) + chk(reply && reply.code === 'compiler-toolchain-setup-failed', + `the failure reply carries code compiler-toolchain-setup-failed (got ${JSON.stringify(reply && reply.code)})`) + chk(reply && /toolchain setup failed importing/.test(String(reply.error)), + 'the reply still explains which URL could not be imported') +} + // --- C: logic / view stage worker behavior is unchanged — still imports the // toolchain exactly once per warmup --------------------------------------------- for (const stage of ['logic', 'view']) { diff --git a/packages/compiler/src/error-codes.js b/packages/compiler/src/error-codes.js new file mode 100644 index 00000000..c2a4cc36 --- /dev/null +++ b/packages/compiler/src/error-codes.js @@ -0,0 +1,61 @@ +/** + * Every error a pool rejects with carries `err.code`, and this is the whole set of + * values. Hosts branch on it: an infrastructure failure is worth retrying on a fresh + * worker or falling back to another compile path, a compile error must be shown to + * the user as-is. Without a code the only way to tell those apart is matching the + * error message text, which silently stops working the moment a message is reworded + * — including messages that come from the browser, not from this package (a failed + * dynamic import, an aborted wasm fetch). + * + * Re-exported from `@dimina-kit/compiler/pool` and `@dimina-kit/compiler/pool-node`, + * so a host imports the constants instead of copying the strings. + */ +export const COMPILER_ERROR_CODES = Object.freeze({ + /** The compiler rejected the project: source or config the user has to fix. Retrying changes nothing. */ + stageError: 'compiler-stage-error', + /** A stage worker could not `import(toolchainSetupURL)` — module 404, network down, wasm unreachable. */ + toolchainSetupFailed: 'compiler-toolchain-setup-failed', + /** The worker went silent past its inactivity window (a wedged wasm loop blocks even heartbeats). */ + workerTimeout: 'compiler-worker-timeout', + /** The worker died: an `error` event, a `postMessage` throw, or an unexpected exit. */ + workerCrashed: 'compiler-worker-crashed', + /** A request reached a slot already judged dead, before it was respawned. */ + workerDead: 'compiler-worker-dead', + /** Node only: esbuild's resident service died, so every call in that realm fails from now on. */ + toolchainDead: 'compiler-toolchain-dead', + /** The pool was disposed; nothing will be compiled on it again. */ + poolDisposed: 'compiler-pool-disposed', +}) + +/** + * Failures of the machinery rather than of the project: a fresh worker, or a different + * compile path, has a real chance of succeeding. `compiler-stage-error` is deliberately + * absent — re-running a broken project just hides the diagnostic the user needs. + */ +export const INFRASTRUCTURE_ERROR_CODES = Object.freeze(new Set([ + COMPILER_ERROR_CODES.toolchainSetupFailed, + COMPILER_ERROR_CODES.toolchainDead, + COMPILER_ERROR_CODES.workerTimeout, + COMPILER_ERROR_CODES.workerCrashed, + COMPILER_ERROR_CODES.workerDead, +])) + +/** + * The pool's own retry predicate: these mean the worker is gone, so replaying the whole + * attempt on a respawned one is safe. Narrower than {@link INFRASTRUCTURE_ERROR_CODES}, + * which also covers failures a *host* may want to react to without the pool retrying. + */ +export const WORKER_DEATH_CODES = Object.freeze(new Set([ + COMPILER_ERROR_CODES.workerTimeout, + COMPILER_ERROR_CODES.workerCrashed, + COMPILER_ERROR_CODES.workerDead, +])) + +/** + * @param {unknown} err + * @returns {boolean} true when retrying on fresh machinery, or falling back to another + * compile path, is worth doing — false for compile errors and for programming errors. + */ +export function isInfrastructureError(err) { + return !!err && typeof err === 'object' && INFRASTRUCTURE_ERROR_CODES.has(/** @type {{ code?: string }} */ (err).code) +} diff --git a/packages/compiler/src/pool-node.js b/packages/compiler/src/pool-node.js index 60cf61d4..0c8a9ce6 100644 --- a/packages/compiler/src/pool-node.js +++ b/packages/compiler/src/pool-node.js @@ -26,10 +26,13 @@ import nodeFs from 'node:fs' import nodePath from 'node:path' import process from 'node:process' import { setupCompile, resetCompilerState, STAGE_NAMES } from './compile-core.js' +import { COMPILER_ERROR_CODES, WORKER_DEATH_CODES as BROWSER_WORKER_DEATH_CODES } from './error-codes.js' import { createWorkerSlot, settleAll } from './worker-slot.js' import { publishToDist } from '../../../dimina/fe/packages/compiler/src/common/publish.js' import { getAppConfigInfo, getAppId, getAppName } from '../../../dimina/fe/packages/compiler/src/env.js' +export { COMPILER_ERROR_CODES, INFRASTRUCTURE_ERROR_CODES, isInfrastructureError } from './error-codes.js' + const { Worker } = createRequire(import.meta.url)('node:worker_threads') // Default INACTIVITY ceiling per stage build. The stage worker heartbeats every 2s while @@ -41,7 +44,7 @@ const DEFAULT_SEND_TIMEOUT_MS = 120000 // 'compiler-toolchain-dead' belongs here even though the worker THREAD is alive: the // realm's esbuild service child process is gone, so the realm is just as unusable as a // crashed worker — the same one-retry-on-fresh-workers policy applies. -const WORKER_DEATH_CODES = new Set(['compiler-worker-timeout', 'compiler-worker-crashed', 'compiler-worker-dead', 'compiler-toolchain-dead']) +const WORKER_DEATH_CODES = new Set([...BROWSER_WORKER_DEATH_CODES, COMPILER_ERROR_CODES.toolchainDead]) // esbuild's node lib drives a spawned long-lived binary child (its "service"). When that // child dies (spawn ENOENT in a packaged app, OOM kill, AV kill), esbuild reports every @@ -200,10 +203,10 @@ export function createNodeCompilerPool({ // the worker so the next attempt (the transparent retry, or the next build once // the environment is healed) respawns a fresh realm with a fresh service. // shrink() is safe here: settleAll above guarantees no request is in flight. - err.code = 'compiler-toolchain-dead' + err.code = COMPILER_ERROR_CODES.toolchainDead workers[i].slot.shrink() } else { - err.code = 'compiler-stage-error' // worker-reported compile error — never retried + err.code = COMPILER_ERROR_CODES.stageError // worker-reported compile error — never retried } if (!firstErr) firstErr = err } @@ -224,7 +227,7 @@ export function createNodeCompilerPool({ function build(outputDir, workPath, useAppIdDir = true, options = {}) { if (disposed) { - return Promise.reject(Object.assign(new Error('[compiler] pool has been disposed'), { code: 'compiler-pool-disposed' })) + return Promise.reject(Object.assign(new Error('[compiler] pool has been disposed'), { code: COMPILER_ERROR_CODES.poolDisposed })) } // New activity: a pending shrink is off the table until this pool drains again. cancelIdleShrink() diff --git a/packages/compiler/src/pool.js b/packages/compiler/src/pool.js index b64fa35f..c1dcbef5 100644 --- a/packages/compiler/src/pool.js +++ b/packages/compiler/src/pool.js @@ -19,8 +19,11 @@ // (esbuild.wasm / oxc wasm are host-hosted assets) // - the source itself (a { relPath: content } map). OPFS is intentionally NOT here: // it's an optional zero-copy source-distribution the downstream can layer on. +import { COMPILER_ERROR_CODES, WORKER_DEATH_CODES } from './error-codes.js' import { createWorkerSlot, settleAll } from './worker-slot.js' +export { COMPILER_ERROR_CODES, INFRASTRUCTURE_ERROR_CODES, isInfrastructureError } from './error-codes.js' + const DEFAULT_STAGES = ['logic', 'view', 'style'] // Default INACTIVITY ceiling for a setup/compile-subset round trip. The stage worker @@ -36,8 +39,6 @@ const DEFAULT_SEND_TIMEOUT_MS = 30000 // eventually — an unguarded warmup would wedge the serial compile chain forever. const DEFAULT_WARMUP_TIMEOUT_MS = 120000 -const WORKER_DEATH_CODES = new Set(['compiler-worker-timeout', 'compiler-worker-crashed', 'compiler-worker-dead']) - /** * @param {{ * createWorker: () => Worker, // required: spawn a module worker running dist/stage-worker.browser.js @@ -123,9 +124,12 @@ export function createCompilerPool(options) { if (!r || r.type === 'error') { // Stable classification for downstream: worker-reported compile/setup errors get // their own code, distinct from the worker-death codes that gate the retry. + // A worker that classified its own failure (toolchain setup, which is machinery + // rather than the user's project) sends its code along — keep it, since only the + // worker can tell those apart. throw Object.assign( new Error(r && r.error ? r.error : `[compiler] ${description} failed in stage '${entry.stage}' worker`), - { code: 'compiler-stage-error', stage: entry.stage }, + { code: (r && r.code) || COMPILER_ERROR_CODES.stageError, stage: entry.stage }, ) } return r @@ -215,7 +219,7 @@ export function createCompilerPool(options) { function compile(input = {}) { const run = chain.then(async () => { if (disposed) { - throw Object.assign(new Error('[compiler] pool has been disposed'), { code: 'compiler-pool-disposed' }) + throw Object.assign(new Error('[compiler] pool has been disposed'), { code: COMPILER_ERROR_CODES.poolDisposed }) } const files = input.files || input if (!files || typeof files !== 'object' || !Object.keys(files).length) { diff --git a/packages/compiler/src/stage-worker.js b/packages/compiler/src/stage-worker.js index e09a64be..874637ae 100644 --- a/packages/compiler/src/stage-worker.js +++ b/packages/compiler/src/stage-worker.js @@ -16,6 +16,7 @@ // layer it on top (hydrate OPFS -> a files map before calling the pool). import { Volume, createFsFromVolume } from 'memfs' import { setupCompile, compileStage, collectOutputs, resetCompilerState } from './compile-core.js' +import { COMPILER_ERROR_CODES } from './error-codes.js' // The compiler logs diagnostics (missing components, unsupported wx APIs, style // preprocessor fallbacks, asset-copy failures, …) via console.* inside this worker, @@ -60,7 +61,13 @@ function ensureToolchain(url) { toolchainReady = import(/* @vite-ignore */ pending) .catch((err) => { toolchainReady = null - throw new Error(`[compiler] toolchain setup failed importing ${pending}: ${(err && err.message) || err}`) + // Coded, because only this worker can tell "the host's wasm assets didn't load" + // apart from "the project doesn't compile". The pool forwards the code, and the + // host uses it to retry or fall back instead of matching the message text. + throw Object.assign( + new Error(`[compiler] toolchain setup failed importing ${pending}: ${(err && err.message) || err}`), + { code: COMPILER_ERROR_CODES.toolchainSetupFailed }, + ) }) } return toolchainReady @@ -182,7 +189,9 @@ self.onmessage = async (e) => { return } } catch (err) { - self.postMessage({ type: 'error', error: String((err && err.stack) || err) }) + // `code` only when this worker classified the failure itself; the pool defaults the + // rest to compiler-stage-error. + self.postMessage({ type: 'error', error: String((err && err.stack) || err), code: (err && err.code) || undefined }) } finally { if (beacon) clearInterval(beacon) } diff --git a/packages/compiler/src/worker-slot.js b/packages/compiler/src/worker-slot.js index e8c23a4f..d24f60b7 100644 --- a/packages/compiler/src/worker-slot.js +++ b/packages/compiler/src/worker-slot.js @@ -18,6 +18,7 @@ // dead transport's terminate() settlement (worker_threads returns a Promise — until it // resolves the old worker may still be writing shared disk), then spawns the next // generation. Messages from a superseded generation are dropped, never FIFO-paired. +import { COMPILER_ERROR_CODES } from './error-codes.js' // Await every promise (so no request is left in flight across an attempt boundary — // a retry must never overlap the previous attempt's traffic), then surface the first @@ -47,7 +48,7 @@ export function createWorkerSlot({ name, spawnTransport, onEvent }) { let reviving = null // in-flight ensureAlive(), so concurrent callers share one respawn const codedError = (message, code) => Object.assign(new Error(message), { code }) - const disposedError = () => codedError(`${name}: pool has been disposed`, 'compiler-pool-disposed') + const disposedError = () => codedError(`${name}: pool has been disposed`, COMPILER_ERROR_CODES.poolDisposed) function armTimer(entry) { if (!(entry.timeoutMs < Infinity)) return @@ -57,7 +58,7 @@ export function createWorkerSlot({ name, spawnTransport, onEvent }) { entry.timer = setTimeout(() => { judgeDead( `${name} timed out after ${entry.timeoutMs}ms waiting for a reply to '${entry.description}'`, - 'compiler-worker-timeout', + COMPILER_ERROR_CODES.workerTimeout, ) }, entry.timeoutMs) } @@ -103,7 +104,7 @@ export function createWorkerSlot({ name, spawnTransport, onEvent }) { function handleCrash(gen, message) { if (disposed || dead || gen !== generation) return - judgeDead(message, 'compiler-worker-crashed') + judgeDead(message, COMPILER_ERROR_CODES.workerCrashed) } function ensureAlive() { @@ -137,7 +138,7 @@ export function createWorkerSlot({ name, spawnTransport, onEvent }) { if (dead) { return reject(codedError( `${name} is dead — ensureAlive() must run before request()`, - 'compiler-worker-dead', + COMPILER_ERROR_CODES.workerDead, )) } const entry = { resolve, reject, timeoutMs, description, timer: null } @@ -146,7 +147,7 @@ export function createWorkerSlot({ name, spawnTransport, onEvent }) { try { transport.postMessage(msg) } catch (err) { - judgeDead(`${name} postMessage failed: ${(err && err.message) || err}`, 'compiler-worker-crashed') + judgeDead(`${name} postMessage failed: ${(err && err.message) || err}`, COMPILER_ERROR_CODES.workerCrashed) } }) } diff --git a/packages/compiler/tsconfig.types.json b/packages/compiler/tsconfig.types.json index 7c52769b..15d52840 100644 --- a/packages/compiler/tsconfig.types.json +++ b/packages/compiler/tsconfig.types.json @@ -16,6 +16,7 @@ }, "include": [ "src/compile-core.js", + "src/error-codes.js", "src/browser-entry.js", "src/pool.js", "src/pool-node.js", From 5896ec98b2c3b1b73f8df4213a2f983ba3edf270 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 23:35:23 +0800 Subject: [PATCH 06/14] =?UTF-8?q?fix(compiler):=20=E6=8C=89=20review=20?= =?UTF-8?q?=E6=84=8F=E8=A7=81=E6=94=B6=E7=B4=A7=E9=94=99=E8=AF=AF=E7=A0=81?= =?UTF-8?q?=E7=9A=84=E8=BE=B9=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 补三个之前落在 compiler-stage-error 里的码：工具链没被打进宿主应用 （compiler-toolchain-unavailable，可以换编译路径）、编译成功但拷不到 outputDir（compiler-output-write-failed）、调用方自己传错 （compiler-invalid-input）。这三种都不该让宿主以为是用户项目编不过。 - INFRASTRUCTURE_ERROR_CODES / WORKER_DEATH_CODES 改成真正只读： Object.freeze 拦不住 Set 的 add/delete/clear，而这两个集合决定本包自己 什么时候重试，宿主 add 一下就悄悄改了行为。 - worker 回复里的 code 只认表里的值：memfs 抛的 ENOENT 之类不再原样传给 宿主，按 compiler-stage-error 记，报错文字照旧完整保留。 - Node 侧「报错文字 → 错误码」的判定和两条打包提示拆到 src/failure-hints.js。 它不牵连 worker_threads 和编译器实体，test:error-codes 能直接驱动——之前 这条规则只有跑一次真实 Node 构建才验证得到。pool-node 继续再导出两条提示。 - test:error-codes 补齐上述行为的回归；README 的错误码表补上新码和只读约定。 Co-Authored-By: Claude Opus 5 (1M context) --- packages/compiler/README.md | 8 +- packages/compiler/scripts/test-error-codes.js | 77 ++++++++++++++++ .../compiler/scripts/test-stage-toolchain.js | 6 +- packages/compiler/src/error-codes.js | 71 ++++++++++++--- packages/compiler/src/failure-hints.js | 90 +++++++++++++++++++ packages/compiler/src/pool-node.js | 83 ++++++----------- packages/compiler/src/pool.js | 14 ++- 7 files changed, 273 insertions(+), 76 deletions(-) create mode 100644 packages/compiler/src/failure-hints.js diff --git a/packages/compiler/README.md b/packages/compiler/README.md index b5a731b8..b53a0ccd 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -404,11 +404,14 @@ try { | `compiler-worker-crashed` | worker 挂了：`error` 事件、`postMessage` 抛错、异常退出 | 是 | | `compiler-worker-dead` | 请求打到了已判死、还没重建的 slot | 是 | | `compiler-toolchain-dead` | 仅 Node：esbuild 常驻服务进程没了，该 realm 之后每次调用都失败 | 是 | +| `compiler-toolchain-unavailable` | 仅 Node：工具链根本没被打进宿主应用——oxc-parser 在这个平台上找不到运行时绑定，或 esbuild 二进制被塞在 app.asar 里 spawn 不出来。要改的是宿主的打包配置；换个新 worker 没用，但换一条编译路径（比如另装的 dmcc）还有戏 | 是 | +| `compiler-output-write-failed` | 仅 Node：项目编译成功了，把暂存目录拷到 `outputDir` 失败（权限、磁盘满） | 否 | +| `compiler-invalid-input` | 调用本身就错了，比如 `compile()` 没给文件。是调用方的 bug，不是项目的 | 否 | | `compiler-pool-disposed` | 池已回收，不会再编译任何东西 | 否 | -`isInfrastructureError` 之外还有一层：worker 死亡类的三个码（timeout / crashed / dead，Node 上再加 toolchain-dead）由 pool 自己用来做那一次透明重试，宿主一般不用关心。 +`isInfrastructureError` 之外还有一层：worker 死亡类的三个码（timeout / crashed / dead，Node 上再加 toolchain-dead）由 pool 自己用来做那一次透明重试，宿主一般不用关心。这两组码通过 `INFRASTRUCTURE_ERROR_CODES` 和 `WORKER_DEATH_CODES` 导出，都是只读的 Set：`.add()` / `.delete()` / `.clear()` 直接抛错，免得宿主改一下就悄悄改掉了本包自己的重试行为（`Object.freeze` 对 Set 拦不住这些方法）。 -工具链导入失败的码是 **worker 自己打的**——只有它能区分「宿主的 wasm 资源没加载上」和「用户项目编不过」，两者到 pool 手里都是同一种 `{ type:'error' }` 回复。pool 原样转发这个码，其余没带码的一律记为 `compiler-stage-error`。 +工具链导入失败的码是 **worker 自己打的**——只有它能区分「宿主的 wasm 资源没加载上」和「用户项目编不过」，两者到 pool 手里都是同一种 `{ type:'error' }` 回复。pool 原样转发这个码，但只认表里这些值：worker 回复里带的是别的东西（比如 memfs 抛的 `ENOENT`）就按 `compiler-stage-error` 记，宿主不会拿到一个自己分支里没有的码；没带码的同样记为 `compiler-stage-error`。 ## 依赖前置 @@ -463,6 +466,7 @@ pnpm --filter @dimina-kit/compiler test:stage-toolchain # 真实 stag - `src/toolchain.js` — 写 `toolchainSetupURL` 模块的可选助手（`installOxc` / `installEsbuildFromURL`，后者内置 esbuild-wasm 静态资源的 Blob-URL 兜底）。导出为 `@dimina-kit/compiler/toolchain`。 - `src/browser-assets.js` — 浏览器静态资源清单与契约（`COMPILER_BROWSER_ASSETS` / `resolveBrowserAssets`，见上文），构建期检查也用它。导出为 `@dimina-kit/compiler/browser-assets`。 - `src/error-codes.js` — 两个 pool 共用的错误码表 `COMPILER_ERROR_CODES` 与判定 `isInfrastructureError`（见上文），经 `./pool` 与 `./pool-node` 再导出。 +- `src/failure-hints.js` — Node 侧「一条原始报错文字该记哪个码」的判定（`errorCodeForMessage` / `tagFailure`），以及 oxc 绑定缺失、esbuild 二进制被封在 app.asar 这两种打包问题的中文提示（`oxcNativeBindingHint` / `esbuildAsarSpawnHint`，经 `./pool-node` 再导出）。单独成文件是为了让它不牵连 `worker_threads` 和编译器实体，`test:error-codes` 能直接驱动。 - `src/shims/fs.js` — **无后端的 fs 转发层**（`setFs`/`resetFs`/`getFs`）；compiler 所有 `fs.xxx` 走它，未注入即抛错。 - `src/shims/*` — 其余 node 内置与原生依赖的浏览器替身（oxc/esbuild/less/`os.homedir`/…）。 - `scripts/build-compiler.js` — esbuild 打包。onLoad 给 logic/view/style-compiler 与 utils 追加 `__reset*` 导出（喂 `resetCompilerState`，不改子模块源码）；浏览器分支内联真实 `cssnano`+`autoprefixer`（autoprefixer pin 到 node 运行时解析的同一份，避免 esbuild 解析到多加 `-ms-` 前缀的另一版本）；browser 模式产出 core / stage-worker / pool 三个单文件 bundle，并按 metafile 对 `src/browser-assets.js` 的清单自检（漏产物、多产物、静态资源出现静态 import 都直接失败）；node 模式开 `splitting`（stage 编译器成为运行时 chunk——单文件会把 chunk 的 external `import 'sass'` 提升回入口顶层，懒加载会静默失效）。 diff --git a/packages/compiler/scripts/test-error-codes.js b/packages/compiler/scripts/test-error-codes.js index 68e73d70..31c21672 100644 --- a/packages/compiler/scripts/test-error-codes.js +++ b/packages/compiler/scripts/test-error-codes.js @@ -12,6 +12,7 @@ import { COMPILER_ERROR_CODES, INFRASTRUCTURE_ERROR_CODES, WORKER_DEATH_CODES, + isCompilerErrorCode, isInfrastructureError, } from '../src/error-codes.js' import { @@ -19,6 +20,10 @@ import { createCompilerPool, isInfrastructureError as poolIsInfrastructureError, } from '../src/pool.js' +// The Node pool's message→code classification lives in its own module precisely so it can +// be driven here: importing pool-node.js would pull in worker_threads and the compiler +// core, which is why this rule used to go untested until a real Node build ran. +import { errorCodeForMessage, tagFailure } from '../src/failure-hints.js' let failed = false const chk = (cond, msg) => { if (!cond) { failed = true; console.error(`❌ ${msg}`) } else console.log(`✅ ${msg}`) } @@ -97,6 +102,66 @@ async function main() { 'isInfrastructureError() rejects compile errors, non-objects and uncoded errors') chk(POOL_CODES === COMPILER_ERROR_CODES && poolIsInfrastructureError === isInfrastructureError, 'the pool re-exports the same table and predicate, so a host never copies the strings') + chk(isCompilerErrorCode(COMPILER_ERROR_CODES.workerDead) + && !isCompilerErrorCode('ENOENT') && !isCompilerErrorCode(undefined), + 'isCompilerErrorCode() accepts a published code and rejects anything else') + } + + // --- the exported sets are read-only, not merely frozen ------------------------ + // Object.freeze() leaves a Set fully mutable, and these two drive the pool's OWN + // retry decisions: one .add() in host code would silently change when this package + // retries a build. + { + for (const [name, set] of [['INFRASTRUCTURE_ERROR_CODES', INFRASTRUCTURE_ERROR_CODES], ['WORKER_DEATH_CODES', WORKER_DEATH_CODES]]) { + for (const method of ['add', 'delete', 'clear']) { + let threw = false + try { set[method]('compiler-stage-error') } catch { threw = true } + chk(threw, `${name}.${method}() throws instead of changing the pool's retry policy`) + } + chk(set.has(COMPILER_ERROR_CODES.workerDead) && set.size > 0, `${name} is still readable (has + size)`) + } + } + + // --- Node message → code, the classification the disk pool applies ------------- + { + const oxc = 'Error: Cannot find native binding. npm has a bug related to optional dependencies' + const asar = 'Error: spawn /Applications/Demo.app/Contents/Resources/app.asar/node_modules/@esbuild/darwin-arm64/bin/esbuild ENOENT' + chk(errorCodeForMessage(oxc) === COMPILER_ERROR_CODES.toolchainUnavailable, + `a missing oxc-parser binding is a packaging failure, not the project's fault (got ${errorCodeForMessage(oxc)})`) + chk(errorCodeForMessage(asar) === COMPILER_ERROR_CODES.toolchainUnavailable, + `esbuild's binary unspawnable inside app.asar is a packaging failure too (got ${errorCodeForMessage(asar)})`) + chk(isInfrastructureError({ code: errorCodeForMessage(oxc) }), + 'so the host may fall back to another compile path instead of showing the user a compile error') + chk(errorCodeForMessage('The service was stopped') === COMPILER_ERROR_CODES.toolchainDead + && errorCodeForMessage('The service is no longer running') === COMPILER_ERROR_CODES.toolchainDead, + "a dead esbuild service is compiler-toolchain-dead — that realm's worker has to be recycled") + chk(errorCodeForMessage('SyntaxError: Unexpected token in app.json') === COMPILER_ERROR_CODES.stageError, + 'an ordinary compile failure stays compiler-stage-error') + + const tagged = tagFailure(new Error(oxc), 'logic') + chk(tagged.code === COMPILER_ERROR_CODES.toolchainUnavailable && tagged.stage === 'logic', + 'tagFailure() records both the code the message earned and the stage it came from') + chk(/electron-builder|binding-wasm32-wasi/.test(tagged.message), + 'and appends the packaging hint, because the raw oxc message says nothing about packaging') + const preset = tagFailure(Object.assign(new Error(oxc), { code: COMPILER_ERROR_CODES.stageError, stage: 'view' }), 'logic') + chk(preset.code === COMPILER_ERROR_CODES.stageError && preset.stage === 'view' && preset.message === oxc, + 'a code set closer to the failure wins — tagFailure() never overwrites one') + chk(tagFailure('not an error object').code === COMPILER_ERROR_CODES.stageError, + 'a thrown non-Error still comes out as a coded Error') + chk(tagFailure(new Error('EACCES: permission denied'), null, COMPILER_ERROR_CODES.outputWriteFailed).code + === COMPILER_ERROR_CODES.outputWriteFailed, + 'a caller that already knows the category (copying to outputDir failed) passes it in') + } + + // --- a call the host got wrong is not the project's fault ---------------------- + { + const { createWorker } = fakeWorkerFactory({ failOn: null }) + const pool = createCompilerPool({ createWorker, toolchainSetupURL: 'data:text/javascript,export default {}', stages: STAGES }) + const err = await pool.compile({ files: {} }).then(() => null, (e) => e) + await pool.dispose() + chk(!!err && err.code === COMPILER_ERROR_CODES.invalidInput, + `compile() with an empty files map rejects with compiler-invalid-input (got ${err && err.code})`) + chk(!!err && !isInfrastructureError(err), 'and is not retryable — a fresh worker cannot fix the caller') } // --- a worker-classified failure keeps its own code all the way out ------------ @@ -125,6 +190,18 @@ async function main() { chk(!r.ok && r.err.stage === STAGES[0], `the rejection still names the stage that reported it (got ${r.ok ? '' : r.err.stage})`) } + // --- a code the worker made up does not reach the host ------------------------- + // A stage worker can fail on something whose error already carries an unrelated `code` + // (memfs throws ENOENT, node throws EACCES). Passing that straight through would give + // hosts a value their branches do not cover and their `catch` cannot classify. + { + const r = await compileAgainst({ failOn: 'setup', code: 'ENOENT', message: "ENOENT: no such file, open '/work/app.json'" }) + chk(!r.ok && r.err.code === COMPILER_ERROR_CODES.stageError, + `a worker reply carrying a non-published code (ENOENT) is normalized to compiler-stage-error (got ${r.ok ? 'resolved' : r.err.code})`) + chk(!r.ok && String(r.err.message).includes('ENOENT'), + 'the original message still reaches the user — only the code is normalized') + } + // --- the same propagation on the compile step, not just warmup ----------------- { const r = await compileAgainst({ diff --git a/packages/compiler/scripts/test-stage-toolchain.js b/packages/compiler/scripts/test-stage-toolchain.js index 226123a2..f73cf0e9 100644 --- a/packages/compiler/scripts/test-stage-toolchain.js +++ b/packages/compiler/scripts/test-stage-toolchain.js @@ -166,9 +166,9 @@ function findCompiledCss(files) { chk(!!css, `style-only compile-subset produced a real compiled CSS product (found "${css && css[0]}": ${css && JSON.stringify(css[1])})`) } -// --- B2: a toolchain the worker cannot import is reported with a code, not just a -// message. The pool forwards that code untouched, so a host decides "the wasm assets -// didn't load, fall back" without matching the message text (see src/error-codes.js). +// --- a toolchain URL the worker cannot import fails with a code, not just a message. +// The pool forwards that code untouched, so a host decides "the wasm assets didn't +// load, fall back" without matching the message text (see src/error-codes.js). { const worker = await loadWorkerInstance() const reply = await worker.send({ diff --git a/packages/compiler/src/error-codes.js b/packages/compiler/src/error-codes.js index c2a4cc36..2eb6e25e 100644 --- a/packages/compiler/src/error-codes.js +++ b/packages/compiler/src/error-codes.js @@ -1,11 +1,14 @@ /** - * Every error a pool rejects with carries `err.code`, and this is the whole set of - * values. Hosts branch on it: an infrastructure failure is worth retrying on a fresh - * worker or falling back to another compile path, a compile error must be shown to - * the user as-is. Without a code the only way to tell those apart is matching the - * error message text, which silently stops working the moment a message is reworded - * — including messages that come from the browser, not from this package (a failed - * dynamic import, an aborted wasm fetch). + * Every failure a pool rejects with carries `err.code`, and this is the whole set of + * values — from the argument check on the way in to the last copy on the way out. (A + * bug inside the pool itself still surfaces as a plain TypeError; that is a defect + * here, not a category a host should branch on.) Hosts branch on the code: an + * infrastructure failure is worth retrying on a fresh worker or falling back to + * another compile path, a compile error must be shown to the user as-is. Without a + * code the only way to tell those apart is matching the error message text, which + * silently stops working the moment a message is reworded — including messages that + * come from the browser, not from this package (a failed dynamic import, an aborted + * wasm fetch). * * Re-exported from `@dimina-kit/compiler/pool` and `@dimina-kit/compiler/pool-node`, * so a host imports the constants instead of copying the strings. @@ -23,33 +26,79 @@ export const COMPILER_ERROR_CODES = Object.freeze({ workerDead: 'compiler-worker-dead', /** Node only: esbuild's resident service died, so every call in that realm fails from now on. */ toolchainDead: 'compiler-toolchain-dead', + /** + * Node only: the toolchain is not in the installed app at all — oxc-parser resolves no + * runtime binding for this platform, or esbuild's binary sits inside app.asar where it + * cannot be spawned. The host's packaging is what has to change; a fresh worker does + * not help, but another compile path (a separately installed dmcc) still can. + */ + toolchainUnavailable: 'compiler-toolchain-unavailable', + /** Node only: the project compiled, copying the staging dir to outputDir did not (permissions, full disk). */ + outputWriteFailed: 'compiler-output-write-failed', + /** The call itself was wrong — e.g. `compile()` with no files. The caller's bug, not the project's. */ + invalidInput: 'compiler-invalid-input', /** The pool was disposed; nothing will be compiled on it again. */ poolDisposed: 'compiler-pool-disposed', }) +const CODE_VALUES = new Set(Object.values(COMPILER_ERROR_CODES)) + +/** + * Is this one of the published codes? The pool uses it as a gate on codes that arrive + * from a worker: whatever a worker reply claims, only a value from the table above + * reaches the host, so a stray runtime code (a memfs `ENOENT`) cannot pass itself off + * as something hosts branch on. + * @param {unknown} value + * @returns {boolean} + */ +export function isCompilerErrorCode(value) { + return CODE_VALUES.has(/** @type {string} */ (value)) +} + +/** + * A Set a host can read but not change. `Object.freeze` alone does not do this: a frozen + * Set still takes `.add()` and `.delete()`, and the sets below drive the pool's OWN retry + * decisions — one `.add()` in host code would quietly change when this package retries. + * @param {string} name + * @param {string[]} values + * @returns {Set} + */ +function readonlyCodeSet(name, values) { + const set = new Set(values) + for (const method of ['add', 'delete', 'clear']) { + Object.defineProperty(set, method, { + value: () => { throw new TypeError(`${name} is read-only`) }, + writable: false, + configurable: false, + }) + } + return Object.freeze(set) +} + /** * Failures of the machinery rather than of the project: a fresh worker, or a different * compile path, has a real chance of succeeding. `compiler-stage-error` is deliberately * absent — re-running a broken project just hides the diagnostic the user needs. */ -export const INFRASTRUCTURE_ERROR_CODES = Object.freeze(new Set([ +export const INFRASTRUCTURE_ERROR_CODES = readonlyCodeSet('INFRASTRUCTURE_ERROR_CODES', [ COMPILER_ERROR_CODES.toolchainSetupFailed, COMPILER_ERROR_CODES.toolchainDead, + COMPILER_ERROR_CODES.toolchainUnavailable, COMPILER_ERROR_CODES.workerTimeout, COMPILER_ERROR_CODES.workerCrashed, COMPILER_ERROR_CODES.workerDead, -])) +]) /** * The pool's own retry predicate: these mean the worker is gone, so replaying the whole * attempt on a respawned one is safe. Narrower than {@link INFRASTRUCTURE_ERROR_CODES}, * which also covers failures a *host* may want to react to without the pool retrying. */ -export const WORKER_DEATH_CODES = Object.freeze(new Set([ +export const WORKER_DEATH_CODES = readonlyCodeSet('WORKER_DEATH_CODES', [ COMPILER_ERROR_CODES.workerTimeout, COMPILER_ERROR_CODES.workerCrashed, COMPILER_ERROR_CODES.workerDead, -])) +]) /** * @param {unknown} err diff --git a/packages/compiler/src/failure-hints.js b/packages/compiler/src/failure-hints.js new file mode 100644 index 00000000..d4c69495 --- /dev/null +++ b/packages/compiler/src/failure-hints.js @@ -0,0 +1,90 @@ +// Reading a raw Node failure message: which of the published error codes it earns, and +// what to tell the user when the cause is a packaging mistake rather than their project. +// +// Kept apart from pool-node.js (which pulls in worker_threads and the compiler core) so +// this classification is pure string work anyone — including scripts/test-error-codes.js +// — can import and drive directly. +import process from 'node:process' +import { COMPILER_ERROR_CODES } from './error-codes.js' + +// esbuild's node lib drives a spawned long-lived binary child (its "service"). When that +// child dies (spawn ENOENT in a packaged app, OOM kill, AV kill), esbuild reports every +// call with one of these two phrases — and the service NEVER restarts inside that realm, +// so the warm worker is permanently broken and must be recycled, not kept. +export function isDeadToolchainServiceError(message) { + return /The service (was stopped|is no longer running)/.test(String(message)) +} + +/** + * Map a failure message to an actionable packaging hint when it is esbuild failing to + * spawn its native binary from inside an Electron app.asar archive. Electron patches + * child_process.execFile for asar paths but NOT child_process.spawn (which esbuild + * uses), so an in-archive binary path always ENOENTs at spawn even though fs sees the + * file — the raw message points at a path that plainly exists, which is why it needs + * a hint. Returns null for every other message. + * @param {string} message + * @returns {string | null} + */ +export function esbuildAsarSpawnHint(message) { + const msg = String(message) + if (!/app\.asar/.test(msg) || !/esbuild/i.test(msg) || !/ENOENT/.test(msg)) return null + return 'esbuild 的原生二进制无法从 app.asar 内 spawn（Electron 只为 execFile 打 asar 补丁）：' + + "打包配置需 asarUnpack '**/node_modules/esbuild/**' 与 '**/node_modules/@esbuild/**'，" + + '并确保 ESBUILD_BINARY_PATH 指向 app.asar.unpacked 下的真实二进制（@dimina-kit/devkit 在 asar 内运行时会自动设置）' +} + +/** + * Map a failure message to an actionable packaging hint when it is oxc-parser's + * "missing runtime binding" error (thrown when NEITHER the platform-native + * `@oxc-parser/binding-` package NOR the `@oxc-parser/binding-wasm32-wasi` + * fallback resolves at runtime). Neither package is a direct dependency of a + * typical host, so app bundlers (e.g. electron-builder's dependency collection) + * silently drop them — and the raw oxc message says nothing about packaging. + * Returns null for every other message. + * @param {string} message + * @returns {string | null} + */ +export function oxcNativeBindingHint(message) { + if (!/Cannot find native binding/i.test(String(message))) return null + return 'oxc-parser 的运行时绑定没有被打进宿主应用：@dimina-kit/compiler 的 Node 编译路径需要 ' + + `@oxc-parser/binding-${process.platform}-${process.arch}（平台原生绑定）或 ` + + '@oxc-parser/binding-wasm32-wasi（wasm 兜底）二者之一实际存在于包内。' + + '打包分发（如 electron-builder）时请把其中一个显式声明为宿主依赖，避免依赖收集时被丢弃' +} + +/** + * Which published code a raw failure message earns. The two packaging failures the hints + * above recognize are NOT the project's fault — oxc's binding or esbuild's binary is + * missing from the installed app — so they must not land in the compile-error bucket, + * which hosts are told never to retry and never to fall back from. + * + * @param {string} message + * @returns {string} one of COMPILER_ERROR_CODES + */ +export function errorCodeForMessage(message) { + if (isDeadToolchainServiceError(message)) return COMPILER_ERROR_CODES.toolchainDead + if (oxcNativeBindingHint(message) || esbuildAsarSpawnHint(message)) return COMPILER_ERROR_CODES.toolchainUnavailable + return COMPILER_ERROR_CODES.stageError +} + +/** + * Give a raw main-thread failure the same treatment a stage reply gets: a published code, + * the packaging hint its message earned, and the stage it came from. An error that already + * carries a code keeps it — a code set closer to the failure knows more than this does. + * + * @param {unknown} err + * @param {string | null} [stage] the stage to record when the error does not name one + * @param {string} [forcedCode] a code the call site already knows (e.g. a write failure), + * used instead of reading the message + * @returns {Error & { code: string, stage?: string }} + */ +export function tagFailure(err, stage, forcedCode) { + const e = err instanceof Error ? err : new Error(`[compiler] ${String(err)}`) + if (!e.code) { + const hint = oxcNativeBindingHint(e.message) || esbuildAsarSpawnHint(e.message) + if (hint) e.message = `${e.message} — ${hint}` + e.code = forcedCode || errorCodeForMessage(e.message) + } + if (stage && !e.stage) e.stage = stage + return e +} diff --git a/packages/compiler/src/pool-node.js b/packages/compiler/src/pool-node.js index 0c8a9ce6..75b205fe 100644 --- a/packages/compiler/src/pool-node.js +++ b/packages/compiler/src/pool-node.js @@ -27,11 +27,15 @@ import nodePath from 'node:path' import process from 'node:process' import { setupCompile, resetCompilerState, STAGE_NAMES } from './compile-core.js' import { COMPILER_ERROR_CODES, WORKER_DEATH_CODES as BROWSER_WORKER_DEATH_CODES } from './error-codes.js' +import { errorCodeForMessage, esbuildAsarSpawnHint, oxcNativeBindingHint, tagFailure } from './failure-hints.js' import { createWorkerSlot, settleAll } from './worker-slot.js' import { publishToDist } from '../../../dimina/fe/packages/compiler/src/common/publish.js' import { getAppConfigInfo, getAppId, getAppName } from '../../../dimina/fe/packages/compiler/src/env.js' export { COMPILER_ERROR_CODES, INFRASTRUCTURE_ERROR_CODES, isInfrastructureError } from './error-codes.js' +// The packaging hints stay part of this entry's published surface: a Node host that +// catches a compile failure reads them from `@dimina-kit/compiler/pool-node`. +export { esbuildAsarSpawnHint, oxcNativeBindingHint } from './failure-hints.js' const { Worker } = createRequire(import.meta.url)('node:worker_threads') @@ -46,14 +50,6 @@ const DEFAULT_SEND_TIMEOUT_MS = 120000 // crashed worker — the same one-retry-on-fresh-workers policy applies. const WORKER_DEATH_CODES = new Set([...BROWSER_WORKER_DEATH_CODES, COMPILER_ERROR_CODES.toolchainDead]) -// esbuild's node lib drives a spawned long-lived binary child (its "service"). When that -// child dies (spawn ENOENT in a packaged app, OOM kill, AV kill), esbuild reports every -// call with one of these two phrases — and the service NEVER restarts inside that realm, -// so the warm worker is permanently broken and must be recycled, not kept. -function isDeadToolchainServiceError(message) { - return /The service (was stopped|is no longer running)/.test(String(message)) -} - // Default idle window before the pool shrinks (terminates its resident stage workers to // release their memory — a warm worker set holds hundreds of MB of toolchain + compile // allocations). Shrinking is transparent: the next build's ensureAlive respawns fresh @@ -164,12 +160,20 @@ export function createNodeCompilerPool({ // outputDir resolved exactly like publishToDist resolves it (against cwd), so // when it sits inside the project the npm scan skips the published output — // a previous build's copies must never become the next build's input. - const ctx = await setupCompile({ - fs: nodeFs, - workPath, - options: { fileTypes }, - npmScanExclude: [nodePath.resolve(process.cwd(), outputDir)], - }) + // Setup runs on the main thread, so its failures never pass through the stage-result + // normalization below — code them here, or a bad app.json (and an unpackaged oxc + // binding, which setup hits first) would reject with no code at all. + let ctx + try { + ctx = await setupCompile({ + fs: nodeFs, + workPath, + options: { fileTypes }, + npmScanExclude: [nodePath.resolve(process.cwd(), outputDir)], + }) + } catch (err) { + throw tagFailure(err, 'setup') + } const { storeInfo, pages } = ctx // 2) Fan out to the resident stage workers. They restore the same storeInfo (so their @@ -198,22 +202,26 @@ export function createNodeCompilerPool({ const err = new Error(`[compiler] stage "${r && r.stage}" failed: ${cause}${hint ? ` — ${hint}` : ''}`) if (info && info.stack) err.stack = info.stack err.stage = r && r.stage - if (isDeadToolchainServiceError(cause)) { + err.code = errorCodeForMessage(cause) + if (err.code === COMPILER_ERROR_CODES.toolchainDead) { // The realm's toolchain service child is dead and never comes back — terminate // the worker so the next attempt (the transparent retry, or the next build once // the environment is healed) respawns a fresh realm with a fresh service. // shrink() is safe here: settleAll above guarantees no request is in flight. - err.code = COMPILER_ERROR_CODES.toolchainDead workers[i].slot.shrink() - } else { - err.code = COMPILER_ERROR_CODES.stageError // worker-reported compile error — never retried } if (!firstErr) firstErr = err } if (firstErr) throw firstErr // 3) Publish the staging dir to the caller's outputDir (dmcc-identical layout). - publishToDist(outputDir, useAppIdDir) + // Everything compiled; only the copy can still fail here (permissions, full disk), + // which is neither the project's fault nor a reason to retry on a fresh worker. + try { + publishToDist(outputDir, useAppIdDir) + } catch (err) { + throw tagFailure(err, null, COMPILER_ERROR_CODES.outputWriteFailed) + } return { appId: getAppId(), @@ -269,43 +277,6 @@ export function createNodeCompilerPool({ // user-facing compile-log lines a host (e.g. devkit/devtools' log panel) already scrapes. const STAGE_TITLES = { logic: '编译页面逻辑', view: '编译页面文件', style: '编译样式文件' } -/** - * Map a failure message to an actionable packaging hint when it is esbuild failing to - * spawn its native binary from inside an Electron app.asar archive. Electron patches - * child_process.execFile for asar paths but NOT child_process.spawn (which esbuild - * uses), so an in-archive binary path always ENOENTs at spawn even though fs sees the - * file — the raw message points at a path that plainly exists, which is why it needs - * a hint. Returns null for every other message. - * @param {string} message - * @returns {string | null} - */ -export function esbuildAsarSpawnHint(message) { - const msg = String(message) - if (!/app\.asar/.test(msg) || !/esbuild/i.test(msg) || !/ENOENT/.test(msg)) return null - return 'esbuild 的原生二进制无法从 app.asar 内 spawn（Electron 只为 execFile 打 asar 补丁）：' - + "打包配置需 asarUnpack '**/node_modules/esbuild/**' 与 '**/node_modules/@esbuild/**'，" - + '并确保 ESBUILD_BINARY_PATH 指向 app.asar.unpacked 下的真实二进制（@dimina-kit/devkit 在 asar 内运行时会自动设置）' -} - -/** - * Map a failure message to an actionable packaging hint when it is oxc-parser's - * "missing runtime binding" error (thrown when NEITHER the platform-native - * `@oxc-parser/binding-` package NOR the `@oxc-parser/binding-wasm32-wasi` - * fallback resolves at runtime). Neither package is a direct dependency of a - * typical host, so app bundlers (e.g. electron-builder's dependency collection) - * silently drop them — and the raw oxc message says nothing about packaging. - * Returns null for every other message. - * @param {string} message - * @returns {string | null} - */ -export function oxcNativeBindingHint(message) { - if (!/Cannot find native binding/i.test(String(message))) return null - return 'oxc-parser 的运行时绑定没有被打进宿主应用：@dimina-kit/compiler 的 Node 编译路径需要 ' - + `@oxc-parser/binding-${process.platform}-${process.arch}（平台原生绑定）或 ` - + '@oxc-parser/binding-wasm32-wasi（wasm 兜底）二者之一实际存在于包内。' - + '打包分发（如 electron-builder）时请把其中一个显式声明为宿主依赖，避免依赖收集时被丢弃' -} - // Lazy singleton pool — a DROP-IN replacement for dmcc's `build(targetPath, workPath, // useAppIdDir, options)` with ONE deliberate divergence on the error path: // • the first call spins up the resident workers; every later call (a watch rebuild) diff --git a/packages/compiler/src/pool.js b/packages/compiler/src/pool.js index c1dcbef5..0c2d0456 100644 --- a/packages/compiler/src/pool.js +++ b/packages/compiler/src/pool.js @@ -19,7 +19,7 @@ // (esbuild.wasm / oxc wasm are host-hosted assets) // - the source itself (a { relPath: content } map). OPFS is intentionally NOT here: // it's an optional zero-copy source-distribution the downstream can layer on. -import { COMPILER_ERROR_CODES, WORKER_DEATH_CODES } from './error-codes.js' +import { COMPILER_ERROR_CODES, WORKER_DEATH_CODES, isCompilerErrorCode } from './error-codes.js' import { createWorkerSlot, settleAll } from './worker-slot.js' export { COMPILER_ERROR_CODES, INFRASTRUCTURE_ERROR_CODES, isInfrastructureError } from './error-codes.js' @@ -126,10 +126,13 @@ export function createCompilerPool(options) { // their own code, distinct from the worker-death codes that gate the retry. // A worker that classified its own failure (toolchain setup, which is machinery // rather than the user's project) sends its code along — keep it, since only the - // worker can tell those apart. + // worker can tell those apart, but only if it is one of the published codes. A + // runtime code that happens to ride along on the reply (a memfs `ENOENT`) is not + // something a host can branch on, so it lands in the compile-error bucket. + const workerCode = r && isCompilerErrorCode(r.code) ? r.code : null throw Object.assign( new Error(r && r.error ? r.error : `[compiler] ${description} failed in stage '${entry.stage}' worker`), - { code: (r && r.code) || COMPILER_ERROR_CODES.stageError, stage: entry.stage }, + { code: workerCode || COMPILER_ERROR_CODES.stageError, stage: entry.stage }, ) } return r @@ -223,7 +226,10 @@ export function createCompilerPool(options) { } const files = input.files || input if (!files || typeof files !== 'object' || !Object.keys(files).length) { - throw new Error('[compiler] pool.compile expects { files: { relPath: content }, workPath?, options? } (or a non-empty files map)') + throw Object.assign( + new Error('[compiler] pool.compile expects { files: { relPath: content }, workPath?, options? } (or a non-empty files map)'), + { code: COMPILER_ERROR_CODES.invalidInput }, + ) } const workPath = input.workPath || defaultWorkPath const options = input.options || {} From 8f78ae2c0de0defeffd8e4dfa25cdc63ee656e8a Mon Sep 17 00:00:00 2001 From: lbb00 Date: Thu, 3 Sep 2026 00:42:19 +0800 Subject: [PATCH 07/14] =?UTF-8?q?fix(compiler):=20node:fs=20=E7=9A=84=20EA?= =?UTF-8?q?CCES=20=E4=B8=8D=E5=86=8D=E5=BD=93=E6=88=90=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E5=99=A8=E7=9A=84=E9=94=99=E8=AF=AF=E7=A0=81=E6=BC=8F=E5=87=BA?= =?UTF-8?q?=E5=8E=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tagFailure() 原本"错误已经带码就不动它"。但这条路上的失败大多来自 node:fs， 它抛的错天生带着 .code = 'EACCES' / 'ENOSPC'，于是往 outputDir 拷产物失败时， 宿主拿到的是 EACCES，而不是承诺的 compiler-output-write-failed——一个它自己的 分支里根本没有的码，等于没有分类。 改成：调用方传了码就用调用方的；没传时只保留表里认得的码，其余按报错文字重新 分类。原来"离失败更近的码优先"的意图对表内的码仍然成立。 原有测试只把 EACCES 写进 message、没设 .code，所以实现坏着它也是绿的；补的两条 断言按真实的 node:fs 错误形状构造，实现改回去就红。 Co-Authored-By: Claude Opus 5 (1M context) --- packages/compiler/README.md | 2 ++ packages/compiler/scripts/test-error-codes.js | 12 ++++++++++++ packages/compiler/src/failure-hints.js | 16 +++++++++++----- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/compiler/README.md b/packages/compiler/README.md index b53a0ccd..d01e362c 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -413,6 +413,8 @@ try { 工具链导入失败的码是 **worker 自己打的**——只有它能区分「宿主的 wasm 资源没加载上」和「用户项目编不过」，两者到 pool 手里都是同一种 `{ type:'error' }` 回复。pool 原样转发这个码，但只认表里这些值：worker 回复里带的是别的东西（比如 memfs 抛的 `ENOENT`）就按 `compiler-stage-error` 记，宿主不会拿到一个自己分支里没有的码；没带码的同样记为 `compiler-stage-error`。 +主线程上抛出来的失败同样只会带表里的码。这条路上最常见的来源是 `node:fs`，而它的错误天生带着 `EACCES`、`ENOSPC` 这类 libc 名字：往 `outputDir` 拷产物失败一律记 `compiler-output-write-failed`，其余的按报错文字重新分类，libc 名字不会漏出来。 + ## 依赖前置 编译器实体源码在 `dimina` 子模块里，dart-sass 等在其 fe workspace。构建前确保子模块已初始化、依赖已装： diff --git a/packages/compiler/scripts/test-error-codes.js b/packages/compiler/scripts/test-error-codes.js index 31c21672..8602ba58 100644 --- a/packages/compiler/scripts/test-error-codes.js +++ b/packages/compiler/scripts/test-error-codes.js @@ -151,6 +151,18 @@ async function main() { chk(tagFailure(new Error('EACCES: permission denied'), null, COMPILER_ERROR_CODES.outputWriteFailed).code === COMPILER_ERROR_CODES.outputWriteFailed, 'a caller that already knows the category (copying to outputDir failed) passes it in') + + // The real failures this path sees come from node:fs, and those errors arrive with + // .code already set to a libc name. Letting an existing .code win would publish + // 'EACCES'/'ENOSPC' out of the pool — codes that are in no table the host switches on. + const fsDenied = Object.assign(new Error('EACCES: permission denied, rename ...'), { code: 'EACCES' }) + chk(tagFailure(fsDenied, null, COMPILER_ERROR_CODES.outputWriteFailed).code === COMPILER_ERROR_CODES.outputWriteFailed, + 'a node:fs error that already carries EACCES still comes out as compiler-output-write-failed') + const fsFull = Object.assign(new Error('ENOSPC: no space left on device'), { code: 'ENOSPC' }) + chk(tagFailure(fsFull, 'setup').code === COMPILER_ERROR_CODES.stageError, + 'and with no forced code it gets reclassified, not passed through') + chk(tagFailure(fsFull, 'setup').stage === 'setup', + 'reclassifying the code leaves the stage alone') } // --- a call the host got wrong is not the project's fault ---------------------- diff --git a/packages/compiler/src/failure-hints.js b/packages/compiler/src/failure-hints.js index d4c69495..67771bb7 100644 --- a/packages/compiler/src/failure-hints.js +++ b/packages/compiler/src/failure-hints.js @@ -5,7 +5,7 @@ // this classification is pure string work anyone — including scripts/test-error-codes.js // — can import and drive directly. import process from 'node:process' -import { COMPILER_ERROR_CODES } from './error-codes.js' +import { COMPILER_ERROR_CODES, isCompilerErrorCode } from './error-codes.js' // esbuild's node lib drives a spawned long-lived binary child (its "service"). When that // child dies (spawn ENOENT in a packaged app, OOM kill, AV kill), esbuild reports every @@ -69,8 +69,12 @@ export function errorCodeForMessage(message) { /** * Give a raw main-thread failure the same treatment a stage reply gets: a published code, - * the packaging hint its message earned, and the stage it came from. An error that already - * carries a code keeps it — a code set closer to the failure knows more than this does. + * the packaging hint its message earned, and the stage it came from. + * + * Only a code from the published table survives. A code set closer to the failure does know + * more, but most failures here come out of node:fs, and those arrive with .code already set + * to a libc name — leaving it alone would publish 'EACCES' or 'ENOSPC' out of the pool, and + * the host switches on the table, so an unknown code reads as "no category at all". * * @param {unknown} err * @param {string | null} [stage] the stage to record when the error does not name one @@ -80,10 +84,12 @@ export function errorCodeForMessage(message) { */ export function tagFailure(err, stage, forcedCode) { const e = err instanceof Error ? err : new Error(`[compiler] ${String(err)}`) - if (!e.code) { + if (forcedCode) { + e.code = forcedCode + } else if (!isCompilerErrorCode(e.code)) { const hint = oxcNativeBindingHint(e.message) || esbuildAsarSpawnHint(e.message) if (hint) e.message = `${e.message} — ${hint}` - e.code = forcedCode || errorCodeForMessage(e.message) + e.code = errorCodeForMessage(e.message) } if (stage && !e.stage) e.stage = stage return e From 0d5e7fa37f9bae2e08fb1fa5426d933e8f96693a Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 20:00:53 +0800 Subject: [PATCH 08/14] =?UTF-8?q?feat(compiler):=20=E8=AE=A9=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E7=AD=89=E4=BA=8C=E8=BF=9B=E5=88=B6=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E5=8E=9F=E6=A0=B7=E7=A9=BF=E8=BF=87=E7=BC=96=E8=AF=91=E6=B1=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 小程序里的图片以前过不了池子，两个方向都断： - 出口：collectOutputs 一律用 utf8 读产物，图片的每个非法字节被换成 U+FFFD 且无法还原。现在每条产物先按严格 UTF-8 解码，解得出来才是 字符串，解不出来原样返回字节。 - 入口：worker 收到的图片是 Uint8Array（postMessage 结构化克隆的结果）， 而 memfs 的 Volume.fromJSON 见到 Uint8Array 会把它建成一个目录，不报错 ——图片在 worker 里根本不存在。新增 src/seed-memfs.js 把这类值单独 writeFileSync 写入。 files map 的类型因此从 Record 放宽成 Record，下游把一条产物当字符串用之前要先判类型。 test:binary-seed 不需要构建，锁住 memfs 这个行为；test:binary-outputs 用 base 示例跑完整编译，断言页面引用的两张 PNG 逐字节相同、文本产物仍是字符串 且不含替换字符。 --- packages/compiler/README.md | 16 ++-- packages/compiler/package.json | 4 +- .../compiler/scripts/test-binary-outputs.js | 79 +++++++++++++++++++ packages/compiler/scripts/test-binary-seed.js | 54 +++++++++++++ packages/compiler/src/compile-core.js | 26 +++++- packages/compiler/src/pool.js | 11 ++- packages/compiler/src/seed-memfs.js | 38 +++++++++ packages/compiler/src/stage-worker.js | 4 +- packages/compiler/tsconfig.types.json | 1 + packages/compiler/types-fixture/consumer.ts | 5 +- 10 files changed, 222 insertions(+), 16 deletions(-) create mode 100644 packages/compiler/scripts/test-binary-outputs.js create mode 100644 packages/compiler/scripts/test-binary-seed.js create mode 100644 packages/compiler/src/seed-memfs.js diff --git a/packages/compiler/README.md b/packages/compiler/README.md index d01e362c..6cdeed6e 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -104,7 +104,7 @@ const pool = createCompilerPool({ await pool.warmup() // 起 3 个常驻 stage worker;每个各自初始化一次 wasm,之后 compile 复用 const { appId, name, files } = await pool.compile({ files: source, workPath: '/project' }) -// 入参 source 与返回 files 都是文本 map:{ 相对路径: 文本内容 };二进制资源不能靠返回 files,见「已知限制」 +// 入参 source 与返回 files 都是 { 相对路径: 内容 };内容是文本就给 string,图片等二进制给 Uint8Array pool.dispose() // 用完终止 worker ``` @@ -229,7 +229,7 @@ try { 几个值得知道的行为： -- **写真实磁盘**（不经 `files` map），二进制静态资源完好——不受浏览器 `collectOutputs` utf8 限制。 +- **写真实磁盘**（不经 `files` map），产物直接落盘，不用先读回内存再交给你。 - **fs 用原生、`worker_threads` 仍 shim**：Node 构建不给 `node:fs` 挂 shim（产物/二进制资源走真实磁盘）；但 `worker_threads` 依旧 shim 掉，为的是关掉 dmcc 编译器自带的 worker 消息处理——pool/stage worker 自己改用 `createRequire('node:worker_threads')` 拿到真实的 `Worker`/`parentPort`。 - **sourcemap 开关走导出而非 worker message**：dmcc 的 logic/view stage 编译器原生只从 worker 的 `parentPort` message 里读 sourcemap 开关，而这条路径已经被上面那条 `worker_threads` shim 短路——构建时给这两个 stage 追加 `__setEnableSourcemap` 导出，`runStage` 前显式调用来代替。 - **`build()` 里的 `path` 与 dmcc 语义等价、算法不同**：dmcc 读 `mainPages[1]` 是因为它的 style task 用 `unshift` 把 `app` 塞进了共享数组（`[1]` 才是原本的第一页）；本包的 style stage 不改变原数组，等价写法就是 `mainPages[0]`。 @@ -248,7 +248,7 @@ pool 覆盖不了的场景才用 core。它导出 `compileMiniApp`（单线程 compileMiniApp({ fs, workPath? }): Promise<{ appId, name, files }> // = setup + 三 stage + collect 的单-realm 组合 setupCompile({ fs, workPath?, options? }): Promise<{ storeInfo, pages, appId, name, targetPath, workPath }> compileStage({ stage, pages, storeInfo, fs }): Promise // stage: 'logic'|'view'|'style';产物写回 fs -collectOutputs({ fs, targetPath }): Record // 遍历 fs 收 targetPath 前缀下产物 +collectOutputs({ fs, targetPath }): Record // 遍历 fs 收 targetPath 前缀下产物(文本给 string,二进制给字节) resetCompilerState(): void // 清模块级缓存;常驻 realm 复用前必调 STAGE_NAMES: string[] // ['logic','view','style'] initToolchain(): Promise // no-op,占位保 API 稳定 @@ -361,7 +361,7 @@ async function filesFromDir(dir, prefix = '') { // 只读递来的 han } ``` -> OPFS 只是**源码分发的真相源**（一次写、多 worker 独立读、零克隆），编译仍在 memfs 上；不需要 SharedArrayBuffer（但上面 oxc wasm 钩子仍需 COOP/COEP）。示例只处理**文本** fixture——真实图片/svg 资源要保留二进制（见「已知限制」）。 +> OPFS 只是**源码分发的真相源**（一次写、多 worker 独立读、零克隆），编译仍在 memfs 上；不需要 SharedArrayBuffer（但上面 oxc wasm 钩子仍需 COOP/COEP）。示例只处理**文本** fixture——真实图片/svg 资源读成 `Uint8Array` 放进同一个 map 即可。 ## fs 契约与约定 @@ -372,12 +372,13 @@ async function filesFromDir(dir, prefix = '') { // 只读递来的 han - **产物写回同一个 fs。** compiler `writeFileSync` 把产物写进你的 fs，所以传入的 fs 必须**可写**。产物目录 `targetPath` 见下。 - **编译会修改你的 fs。** 缺 `project.config.json`/appid 时，会往 `${workPath}/project.config.json` 写入一个 appid（`dmlocalpreview`）——传入的 fs 不能当成只读快照。 - **同步契约，不需要 `fs.promises`。** `DiminaFs` 只要求同步方法（`existsSync`/`readFileSync`/`readdirSync{withFileTypes}`/`statSync`/`writeFileSync`/`mkdirSync{recursive}`/`copyFileSync`/`rmSync`）——编译路径不碰 async fs。纯异步后端（只有 Promise 版读写）没法当 fs 用。 +- **`readFileSync` 不带编码参数时必须返回字节。** `collectOutputs` 靠这个来区分文本产物和图片：先按严格 UTF-8 解码，解不出来就原样把字节交给调用方。后端若无视编码参数一律返回字符串，二进制产物就会在这一步坏掉。 `@dimina/compiler` 自己并不知道 fs 被换掉了。 ## 已知限制与错误处理 -- **`files` 目前只可靠承载文本产物。** `collectOutputs` 用 utf8 读回 `targetPath` 下所有产物；图片/svg 等二进制静态资源（compiler `copyFileSync` 到 `main/static`）会被按 utf8 读坏。纯文本项目/fixture 无碍；要用真实二进制资源，需下游自行从注入的 fs 读 `main/static` 的字节，别依赖返回的 `files`。 +- **`files` 里的一条产物可能是字符串，也可能是 `Uint8Array`。** `collectOutputs` 对每条产物先按严格 UTF-8 解码，能解出来就是字符串（JS/CSS/JSON 等），解不出来（图片、字体等 compiler `copyFileSync` 到 `main/static` 的资源）就原样给字节。下游拿到一条产物当字符串用之前要先判类型——`typeof v === 'string'`。入参方向同理：源码里的图片直接以 `Uint8Array` 放进 `files` 即可，pool 会把它当文件写进 worker 的 memfs。 - **`targetPath` 来自环境。** compiler 产物目录取 `process.env.TARGET_PATH`，否则 `os.tmpdir()/dimina-fe-dist-<时间戳>`（浏览器 os shim 下通常 `/tmp/...`）。`setupCompile` 会先 `rmSync` 清空它——别把 `TARGET_PATH` 指到源码目录或共享目录。用 `setupCompile` 返回的 `targetPath` 喂 `collectOutputs`。 - **不是全 fail-fast。** 缺 fs 方法、坏 appid、坏 `project.config.json`、miniprogram_npm 构建失败会 **reject**；但**样式预处理器失败（如当前浏览器构建暂不支持 `.less`）会被吞掉、降级用原始 CSS**，PostCSS 解析失败返回空串，资源拷贝失败只 `console.log`，logic esbuild 压缩失败回退未压缩代码。**用 pool 时把这些拿出来的办法是 `createCompilerPool({ onLog })`**——它把 worker 内编译器的 `console.*` 诊断（带 stage 标签）转发给你；也可在产物为空/缺失时二次校验。 @@ -435,7 +436,7 @@ pnpm --filter @dimina-kit/compiler build:types # 仅 dist/types/*.d.ts ## 测试 -`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）只包含不需要构建的两份契约测试——静态资源清单（`test:browser-assets`）和错误码（`test:error-codes`）；下面这些各自要先构建，按需单跑。 +`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）只包含不需要构建的三份契约测试——静态资源清单（`test:browser-assets`）、错误码（`test:error-codes`）和二进制入参播种（`test:binary-seed`）；下面这些各自要先构建，按需单跑。 测试里用 memfs 扮演「下游 fs」： @@ -453,6 +454,8 @@ pnpm --filter @dimina-kit/compiler test:stage-load-retry # stage 工 pnpm --filter @dimina-kit/compiler test:browser-assets # 静态资源清单:改名/新 chunk/出现静态 import 都会被构建期检查拦下 pnpm --filter @dimina-kit/compiler test:error-codes # 错误码:worker 自己判定的失败(工具链导入)带着码原样传到调用方,其余记为 compiler-stage-error pnpm --filter @dimina-kit/compiler test:stage-toolchain # 真实 stage worker bundle:每个 stage 都加载工具链、按 URL 记忆、导入失败带错误码 +pnpm --filter @dimina-kit/compiler test:binary-seed # 入参里的 Uint8Array 播种成文件(memfs 自己的 fromJSON 会把它变成目录) +pnpm --filter @dimina-kit/compiler test:binary-outputs # 页面引用的图片走完整编译后逐字节相同,文本产物仍是字符串 ``` `pool` 的浏览器端到端验证在 `dimina-web-client`（`npm run test:pool`，Playwright 驱动，产物与单线程逐结构一致）。`pool-node` 的宿主端验证在 `@dimina-kit/devkit` 测试套件（真实 fork + `openProject`）。 @@ -461,6 +464,7 @@ pnpm --filter @dimina-kit/compiler test:stage-toolchain # 真实 stag - `src/compile-core.js` — 内联编排 dmcc 的 compile 函数；导出 `compileMiniApp` 与四个接缝 `setupCompile`/`compileStage`/`collectOutputs`/`resetCompilerState`（+ `STAGE_NAMES`、`preloadStage`）。相对路径引用 `dimina` 子模块的 compiler 源码；三个 stage 编译器经动态 import 懒加载（`resetCompilerState` 只清已加载的 stage），node bundle 借 esbuild splitting 把它们编成运行时 chunk。 - `src/browser-entry.js` — 浏览器 core 入口，导出上述接缝 + `initToolchain()`（no-op）。 +- `src/seed-memfs.js` — 把 `files` map 播种成一个 memfs 卷。文本走 `Volume.fromJSON`，`Uint8Array` 单独 `writeFileSync` 写入——`fromJSON` 见到 `Uint8Array` 会当成目录，不报错。 - `src/pool.js` — **浏览器编排池** `createCompilerPool`：常驻 stage worker 池、并行派发、并集合并、realm 复用；不含编译器（轻量，~3KB）。 - `src/stage-worker.js` — **包自带的常驻 stage worker**（内联 core + memfs）：warmup 时 `import(toolchainSetupURL)` 装 wasm 钩子，每次编译 seed 私有 memfs → `setupCompile` + 指定 stage → `collectOutputs`；并把编译器 `console.*` 诊断转发给 pool 的 `onLog`。 - `src/pool-node.js` — **Node 编排池** `createNodeCompilerPool` + dmcc drop-in 默认导出 `build()`：常驻 worker_threads、真实磁盘、全局 build 串行、死 worker 懒复活、idle 自动收缩（`idleShrinkMs`）。 diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 8f06f748..5dfe6c14 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -69,7 +69,9 @@ "test:realm-reuse": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-realm-reuse.js", "test:pool-node": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-node.js", "test:pool-scopehash": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-scopehash.js", - "test": "pnpm run test:browser-assets && pnpm run test:error-codes", + "test": "pnpm run test:browser-assets && pnpm run test:error-codes && pnpm run test:binary-seed", + "test:binary-seed": "node scripts/test-binary-seed.js", + "test:binary-outputs": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-binary-outputs.js", "test:error-codes": "node scripts/test-error-codes.js", "test:browser-assets": "node scripts/test-browser-assets.js", "test:crypto-shim": "node scripts/test-crypto-shim.js", diff --git a/packages/compiler/scripts/test-binary-outputs.js b/packages/compiler/scripts/test-binary-outputs.js new file mode 100644 index 00000000..bb1e5356 --- /dev/null +++ b/packages/compiler/scripts/test-binary-outputs.js @@ -0,0 +1,79 @@ +// An image that a page references must survive the whole compile: seeded as bytes, +// copied by the compiler into the product, and handed back byte-identical. It used to +// come back corrupted — collectOutputs read every product with 'utf8', which replaces +// each invalid byte with U+FFFD and cannot be undone, so downstreams were told in the +// README to go read the fs themselves for real assets. +// +// Drives the real compile seams against the `base` example, which references +// pages/project-mixed/pages/detail/ui.png from its wxml. +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import path from 'node:path' +import { seedMemfs } from '../src/seed-memfs.js' + +const APP = process.env.APP_DIR + || fileURLToPath(new URL('../../../dimina/fe/example/base', import.meta.url)) + +const TEXT_EXT = new Set([ + '.json', '.js', '.ts', '.wxml', '.ddml', '.wxss', '.ddss', '.less', + '.scss', '.sass', '.wxs', '.dds', '.css', +]) + +let failed = 0 +const chk = (cond, msg) => { if (cond) { console.log(`✅ ${msg}`) } else { console.log(`❌ ${msg}`); failed++ } } +const sameBytes = (a, b) => a && b && a.length === b.length && [...a].every((x, i) => x === b[i]) + +// Unlike the older seams tests, this one seeds binary files too — that filter was the +// workaround for the gap under test here. +const files = {} +const readDir = (dir) => { + for (const name of readdirSync(dir)) { + if (name === 'node_modules' || name === '.git') continue + const full = path.join(dir, name) + if (statSync(full).isDirectory()) { readDir(full); continue } + const rel = path.relative(APP, full).split(path.sep).join('/') + files[rel] = TEXT_EXT.has(path.extname(name).toLowerCase()) + ? readFileSync(full, 'utf8') + : new Uint8Array(readFileSync(full)) + } +} +readDir(APP) + +const binarySources = Object.entries(files).filter(([, v]) => v instanceof Uint8Array) +chk(binarySources.length > 0, `the fixture really carries binary sources (${binarySources.map(([k]) => k).join(', ')})`) + +const { setupCompile, compileStage, collectOutputs, STAGE_NAMES } = await import('../dist/compile-core.node.js') + +const workPath = '/work' +const fs = seedMemfs(files, workPath) +const ctx = await setupCompile({ fs, workPath }) +for (const stage of STAGE_NAMES) { + await compileStage({ stage, pages: ctx.pages, storeInfo: ctx.storeInfo, fs }) +} +const out = collectOutputs({ fs, targetPath: ctx.targetPath }) + +const products = Object.entries(out) +const byteProducts = products.filter(([, v]) => v instanceof Uint8Array) +chk(byteProducts.length > 0, + `the compiler's copied assets come back as bytes, not decoded text (${byteProducts.length} of ${products.length} products: ${byteProducts.map(([k]) => k).join(', ')})`) + +for (const [srcPath, srcBytes] of binarySources) { + const match = byteProducts.find(([, v]) => sameBytes(v, srcBytes)) + const copied = products.some(([k]) => k.endsWith(path.basename(srcPath))) + // Only assets a page actually references are copied into the product; one that is + // never referenced legitimately has no product to compare against. + if (match) chk(true, `${srcPath} reached the product byte-identical (as ${match[0]})`) + else chk(!copied, `${srcPath} is not referenced by any page, so it has no product (nothing named like it was emitted)`) +} + +const appConfig = out[Object.keys(out).find((k) => k.endsWith('app-config.json'))] +chk(typeof appConfig === 'string' && JSON.parse(appConfig), 'UTF-8 products are still plain strings (app-config.json parses)') +const jsProducts = products.filter(([k, v]) => k.endsWith('.js') && typeof v === 'string') +chk(jsProducts.length > 0, `compiled JS products are still strings (${jsProducts.length} of them)`) +const cssProducts = products.filter(([k, v]) => k.endsWith('.css') && typeof v === 'string') +chk(cssProducts.length > 0, `compiled CSS products are still strings (${cssProducts.length} of them)`) +const mojibake = products.filter(([, v]) => typeof v === 'string' && v.includes('�')) +chk(mojibake.length === 0, `no product came back with replacement characters (${mojibake.map(([k]) => k).join(', ') || 'none'})`) + +console.log(failed ? `\n❌ ${failed} binary-output assertion(s) failed.` : '\n✅ referenced assets survive the compile byte-identical; text products are unchanged.') +process.exit(failed ? 1 : 0) diff --git a/packages/compiler/scripts/test-binary-seed.js b/packages/compiler/scripts/test-binary-seed.js new file mode 100644 index 00000000..df0c71a9 --- /dev/null +++ b/packages/compiler/scripts/test-binary-seed.js @@ -0,0 +1,54 @@ +// A binary source file must reach the compiler as a FILE. postMessage hands the stage +// worker every image as a Uint8Array, and memfs' own `Volume.fromJSON` turns such a +// value into a DIRECTORY without erroring — the app then referenced an asset that was +// never in the volume. src/seed-memfs.js exists to close exactly that hole, so this +// test drives it directly (no build, no browser). +import { Volume } from 'memfs' +import { seedMemfs } from '../src/seed-memfs.js' + +let failed = 0 +const chk = (cond, msg) => { if (cond) { console.log(`✅ ${msg}`) } else { console.log(`❌ ${msg}`); failed++ } } +const sameBytes = (a, b) => a && b && a.length === b.length && [...a].every((x, i) => x === b[i]) + +const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0xff, 0x00, 0xc3, 0x28]) + +// The memfs behavior this module compensates for. If it ever changes, seed-memfs.js +// can be simplified — until then, dropping it silently loses every binary asset. +{ + const vol = Volume.fromJSON({ 'img/logo.png': PNG }, '/work') + chk(vol.toJSON()['/work/img/logo.png'] === null, + 'Volume.fromJSON turns a Uint8Array value into a directory, no error raised — the reason seedMemfs exists') +} + +{ + const fs = seedMemfs({ 'img/logo.png': PNG, 'app.json': '{"pages":[]}' }, '/work') + chk(fs.statSync('/work/img/logo.png').isFile(), 'a Uint8Array source lands as a file, not a directory') + chk(sameBytes(fs.readFileSync('/work/img/logo.png'), PNG), + 'its bytes are seeded verbatim, including the ones that are not valid UTF-8') + chk(fs.readFileSync('/work/app.json', 'utf8') === '{"pages":[]}', 'text sources are unaffected') +} + +{ + // A Uint8Array that is a WINDOW into a larger buffer (what slicing an upload gives + // you) must write its own bytes only, not the whole backing buffer. + const backing = new Uint8Array([0, 0, 0, 1, 2, 3, 0, 0]) + const view = backing.subarray(3, 6) + const fs = seedMemfs({ 'a/b/c.bin': view }, '/work') + chk(sameBytes(fs.readFileSync('/work/a/b/c.bin'), new Uint8Array([1, 2, 3])), + `a byte view writes only its own range (got ${[...fs.readFileSync('/work/a/b/c.bin')]})`) + chk(fs.statSync('/work/a/b').isDirectory(), 'missing parent directories are created for a binary source') +} + +{ + const fs = seedMemfs({ '/elsewhere/logo.png': PNG, 'app.json': '{}' }, '/work') + chk(sameBytes(fs.readFileSync('/elsewhere/logo.png'), PNG), + 'an absolute key is honored as-is, matching memfs fromJSON') +} + +{ + const fs = seedMemfs({ 'img/logo.png': PNG }, '/work/') + chk(fs.statSync('/work/img/logo.png').isFile(), 'a workPath with a trailing slash does not double the separator') +} + +console.log(failed ? `\n❌ ${failed} binary-seed assertion(s) failed.` : '\n✅ binary sources reach the compiler as files with their exact bytes.') +process.exit(failed ? 1 : 0) diff --git a/packages/compiler/src/compile-core.js b/packages/compiler/src/compile-core.js index ee397b3c..11fe8548 100644 --- a/packages/compiler/src/compile-core.js +++ b/packages/compiler/src/compile-core.js @@ -126,6 +126,24 @@ function ensureAppIdFs(fs, configPath) { } } +// Products are read as BYTES and only turned into a string when they really are +// UTF-8 text. Reading a PNG with 'utf8' replaces every invalid byte with U+FFFD and +// nothing can undo that — the images the compiler copies into `main/static` used to +// come back corrupted, so callers had to go read the fs themselves to get real +// assets. Strict decoding is exact in both directions: text is byte-identical to +// before, and anything that isn't valid UTF-8 stays raw bytes. +const utf8Strict = new TextDecoder('utf-8', { fatal: true }) +function decodeProduct(bytes) { + // A fs backend that ignores the missing encoding and hands back a string is taken + // at its word — same as before this function existed. + if (typeof bytes === 'string') return bytes + try { + return utf8Strict.decode(bytes) + } catch { + return bytes + } +} + // Walk the injected fs under targetPath and collect { relPath: content }. Uses // only readdirSync({withFileTypes}) + readFileSync — inside the fs contract. // Fail-fast: a missing target dir, an unreadable product, or a fs that ignores @@ -148,7 +166,7 @@ function readOutputs(fs, target) { } const full = `${dir}/${e.name}` if (e.isDirectory()) walk(full) - else out[full.slice(prefix.length)] = fs.readFileSync(full, 'utf8') + else out[full.slice(prefix.length)] = decodeProduct(fs.readFileSync(full)) } } walk(prefix.slice(0, -1)) @@ -432,8 +450,10 @@ export async function compileStage({ stage, pages, storeInfo: bundle, fs, source /** * Collect the compiled products from the injected fs under `targetPath` into a * `{ relPath: content }` map. Uses `fs` directly (no shim), so no setup needed. + * UTF-8 text comes back as a string; anything else (images and other binary assets + * the compiler copies into `main/static`) comes back as raw bytes. * @param {{ fs: object, targetPath: string }} opts - * @returns {Record} + * @returns {Record} */ export function collectOutputs({ fs, targetPath } = {}) { return readOutputs(fs, targetPath) @@ -483,7 +503,7 @@ let compileChain = Promise.resolve() * options: forwarded to `setupCompile` -> dmcc's `storeInfo` (custom file-type * dialect, e.g. { fileTypes: { template: ['qdml'], style: ['qdss'], * viewScript: ['qds'] } }). - * @returns {Promise<{ appId: string, name: string, files: Record }>} + * @returns {Promise<{ appId: string, name: string, files: Record }>} */ export function compileMiniApp(opts = {}) { const result = compileChain.then(() => runCompile(opts)) diff --git a/packages/compiler/src/pool.js b/packages/compiler/src/pool.js index 0c2d0456..0d429894 100644 --- a/packages/compiler/src/pool.js +++ b/packages/compiler/src/pool.js @@ -210,14 +210,19 @@ export function createCompilerPool(options) { * Single argument, no ambiguity: pass { files, workPath, options }. A bare * { relPath: content } map is also accepted (uses the default workPath, no options). * @param {{ - * files: Record, + * files: Record, * workPath?: string, * options?: { fileTypes?: { template?: string[], style?: string[], viewScript?: string[] } }, - * } | Record} input + * } | Record} input + * Source files are text or raw bytes: an image belongs in the map as a Uint8Array, + * not as a decoded string (postMessage carries it as bytes, and the stage worker + * seeds it into its memfs as a file). * options.fileTypes lets a caller register a custom template/style/view-script * dialect (e.g. { template: ['qdml'], style: ['qdss'], viewScript: ['qds'] }) — * forwarded to the setup worker's `setupCompile` (dmcc's storeInfo). - * @returns {Promise<{ appId: string, name: string, files: Record }>} + * @returns {Promise<{ appId: string, name: string, files: Record }>} + * Compiled products: UTF-8 text as strings, binary assets (images copied into + * `main/static`) as raw bytes. */ function compile(input = {}) { const run = chain.then(async () => { diff --git a/packages/compiler/src/seed-memfs.js b/packages/compiler/src/seed-memfs.js new file mode 100644 index 00000000..91674c7d --- /dev/null +++ b/packages/compiler/src/seed-memfs.js @@ -0,0 +1,38 @@ +// Turn a `{ relPath: content }` source map into a private memfs for one compile. +// +// Split out of the stage worker because of one memfs behavior that fails silently: +// `Volume.fromJSON` only understands string (and Buffer) values — hand it a plain +// Uint8Array and the path becomes a DIRECTORY, with no error anywhere. postMessage +// delivers every binary source file as exactly that (structured clone turns a Buffer +// into a Uint8Array), so an image in `files` used to end up as an empty directory and +// the compiled app referenced an asset that was never there. +import { Volume, createFsFromVolume } from 'memfs' + +function joinUnder(workPath, relPath) { + if (relPath.startsWith('/')) return relPath + const base = workPath.endsWith('/') ? workPath.slice(0, -1) : workPath + return `${base}/${relPath}` +} + +/** + * @param {Record} files source map; string values are + * seeded through memfs' own fromJSON, byte values are written in afterwards. + * @param {string} workPath project root inside the volume, e.g. '/work' + * @returns {object} a node:fs-shaped object over the fresh volume + */ +export function seedMemfs(files, workPath) { + const text = {} + const binary = [] + for (const [relPath, content] of Object.entries(files || {})) { + if (content instanceof Uint8Array) binary.push([relPath, content]) + else text[relPath] = content + } + const fs = createFsFromVolume(Volume.fromJSON(text, workPath)) + for (const [relPath, bytes] of binary) { + const full = joinUnder(workPath, relPath) + const slash = full.lastIndexOf('/') + if (slash > 0) fs.mkdirSync(full.slice(0, slash), { recursive: true }) + fs.writeFileSync(full, bytes) + } + return fs +} diff --git a/packages/compiler/src/stage-worker.js b/packages/compiler/src/stage-worker.js index 874637ae..1ccdc857 100644 --- a/packages/compiler/src/stage-worker.js +++ b/packages/compiler/src/stage-worker.js @@ -14,9 +14,9 @@ // Source distribution is deliberately OPFS-free: the pool posts the source map and we // seed it into our own memfs. A downstream that wants zero-copy OPFS distribution can // layer it on top (hydrate OPFS -> a files map before calling the pool). -import { Volume, createFsFromVolume } from 'memfs' import { setupCompile, compileStage, collectOutputs, resetCompilerState } from './compile-core.js' import { COMPILER_ERROR_CODES } from './error-codes.js' +import { seedMemfs } from './seed-memfs.js' // The compiler logs diagnostics (missing components, unsupported wx APIs, style // preprocessor fallbacks, asset-copy failures, …) via console.* inside this worker, @@ -74,7 +74,7 @@ function ensureToolchain(url) { } function freshFs(files, workPath) { - return createFsFromVolume(Volume.fromJSON(files, workPath)) + return seedMemfs(files, workPath) } // Run setupCompile ONCE for a compile: parse config, build the scaffold diff --git a/packages/compiler/tsconfig.types.json b/packages/compiler/tsconfig.types.json index 15d52840..196790b2 100644 --- a/packages/compiler/tsconfig.types.json +++ b/packages/compiler/tsconfig.types.json @@ -17,6 +17,7 @@ "include": [ "src/compile-core.js", "src/error-codes.js", + "src/seed-memfs.js", "src/browser-entry.js", "src/pool.js", "src/pool-node.js", diff --git a/packages/compiler/types-fixture/consumer.ts b/packages/compiler/types-fixture/consumer.ts index 23538026..2b533c26 100644 --- a/packages/compiler/types-fixture/consumer.ts +++ b/packages/compiler/types-fixture/consumer.ts @@ -22,8 +22,11 @@ void assetDir // @ts-expect-error resolveBrowserAssets takes the resolved entry path, not the asset list resolveBrowserAssets(COMPILER_BROWSER_ASSETS) -const outputs: Record = collectOutputs({ fs: {}, targetPath: '/dist' }) +const outputs: Record = collectOutputs({ fs: {}, targetPath: '/dist' }) void outputs +// @ts-expect-error products are text OR bytes; a downstream must narrow before treating one as a string +const textOnly: Record = collectOutputs({ fs: {}, targetPath: '/dist' }) +void textOnly // @ts-expect-error targetPath is required collectOutputs({ fs: {} }) From 31e691615fc97fa630e07149a1772c59874917f4 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 23:57:49 +0800 Subject: [PATCH 09/14] =?UTF-8?q?fix(compiler):=20=E6=8C=89=20review=20?= =?UTF-8?q?=E6=84=8F=E8=A7=81=E8=A1=A5=E4=B8=8A=E6=B1=A0=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E4=B8=8E=20BOM=20=E7=9A=84=E5=AD=97=E8=8A=82=E4=BF=9D=E7=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test:binary-outputs 之前只跑了 setupCompile/compileStage/collectOutputs 三个 接缝，全程在同一个 realm 里，宿主真正调的 pool 路径一次都没走到：源码字节 postMessage 进 worker、产物再 postMessage 回来这两趟结构化克隆都没被覆盖。 现在补一段真实浏览器池（dist/pool.browser.js + dist/stage-worker.browser.js） 的往返，harness 自己 structuredClone 每条消息，断言页面引用的图片逐字节相同。 只把 pool.js/stage-worker.js 退回加二进制支持之前，接缝那半仍然通过，这段会红。 - collectOutputs 的 TextDecoder 加 ignoreBOM：默认会把开头的 U+FEFF 吃掉， 带 BOM 的产物出来比进去少三个字节，而且因为仍是合法 UTF-8，没有任何 bytes/text 的信号能发现。补上往返断言。 - README 里 pool 的签名两个 files 都还写成 Record，改成 string | Uint8Array。 Co-Authored-By: Claude Opus 5 (1M context) --- packages/compiler/README.md | 5 +- packages/compiler/package.json | 2 +- .../compiler/scripts/test-binary-outputs.js | 132 +++++++++++++++++- packages/compiler/src/compile-core.js | 5 +- 4 files changed, 137 insertions(+), 7 deletions(-) diff --git a/packages/compiler/README.md b/packages/compiler/README.md index 6cdeed6e..0fc38a2b 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -127,8 +127,9 @@ createCompilerPool(options: { retryOnWorkerDeath?: boolean // 默认 true:worker 死亡导致的编译失败,整次透明重试一次 }): { warmup(): Promise - compile(input: { files: Record, workPath?: string }) - : Promise<{ appId: string, name: string, files: Record }> + // 两个 files 的值:文本给 string,图片等二进制给 Uint8Array(原样穿过,不做编解码) + compile(input: { files: Record, workPath?: string }) + : Promise<{ appId: string, name: string, files: Record }> dispose(): Promise stages: string[] } diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 5dfe6c14..55393e5a 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -71,7 +71,7 @@ "test:pool-scopehash": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-scopehash.js", "test": "pnpm run test:browser-assets && pnpm run test:error-codes && pnpm run test:binary-seed", "test:binary-seed": "node scripts/test-binary-seed.js", - "test:binary-outputs": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-binary-outputs.js", + "test:binary-outputs": "node scripts/build-compiler.js node && node scripts/build-compiler.js browser && node --import ./scripts/register-kit.js scripts/test-binary-outputs.js", "test:error-codes": "node scripts/test-error-codes.js", "test:browser-assets": "node scripts/test-browser-assets.js", "test:crypto-shim": "node scripts/test-crypto-shim.js", diff --git a/packages/compiler/scripts/test-binary-outputs.js b/packages/compiler/scripts/test-binary-outputs.js index bb1e5356..9a6fb207 100644 --- a/packages/compiler/scripts/test-binary-outputs.js +++ b/packages/compiler/scripts/test-binary-outputs.js @@ -4,13 +4,27 @@ // each invalid byte with U+FFFD and cannot be undone, so downstreams were told in the // README to go read the fs themselves for real assets. // -// Drives the real compile seams against the `base` example, which references -// pages/project-mixed/pages/detail/ui.png from its wxml. +// Two levels, because the bytes have to survive two different journeys: +// PART A — the real browser pool (dist/pool.browser.js + dist/stage-worker.browser.js): +// source bytes cross into a worker realm and products cross back, both through +// structured cloning. This is the path a host actually calls. +// PART B — the compile seams (setupCompile/compileStage/collectOutputs) against the +// `base` example, which references pages/project-mixed/pages/detail/ui.png +// from its wxml, plus the byte-exactness rules collectOutputs itself owns. + +// Warm real esbuild/oxc-parser module eval AND esbuild's long-lived service child +// process with the REAL process object BEFORE masking (below) — the service, once +// spawned, is reused for later transforms even under a masked process, so this must +// run first. +import { transform } from 'esbuild' +import 'oxc-parser' import { readdirSync, readFileSync, statSync } from 'node:fs' import { fileURLToPath } from 'node:url' import path from 'node:path' import { seedMemfs } from '../src/seed-memfs.js' +await transform('const __warm = 1', {}) + const APP = process.env.APP_DIR || fileURLToPath(new URL('../../../dimina/fe/example/base', import.meta.url)) @@ -41,6 +55,104 @@ readDir(APP) const binarySources = Object.entries(files).filter(([, v]) => v instanceof Uint8Array) chk(binarySources.length > 0, `the fixture really carries binary sources (${binarySources.map(([k]) => k).join(', ')})`) +const IMAGE_BYTES = binarySources.length > 0 ? binarySources[0][1] : new Uint8Array([0x89, 0x50, 0x4E, 0x47]) + +// --- PART A: the real browser pool, bytes through structured cloning ------------ +// +// The seams in PART B never leave this realm, so they cannot show whether the bytes +// survive postMessage in either direction — the worker seeds its own memfs from the +// cloned source map, and the product map is cloned back. A Uint8Array that arrived as +// a plain object, or a product silently decoded before being posted, would pass every +// PART B assertion and still hand the host a corrupt image. +// +// dart-sass's bundled browser shim checks process.versions.node at module-eval time +// (only reached once the lazily-loaded style-compiler chunk imports it) — masking to a +// browser-shaped stub makes it take the browser branch instead of crashing on +// `Dynamic require of "url" is not supported`. Saved so PART B (a Node-target bundle, +// whose transitive deps module-eval-check process.versions.node too) can restore it. +const realProcess = globalThis.process +globalThis.process = { env: {}, cwd: () => '/' } + +const WORKER_URL = new URL('../dist/stage-worker.browser.js', import.meta.url).href +const TOOLCHAIN_URL = new URL('./toolchain-setup-node-native.js', import.meta.url).href +const WORK_PATH = '/work' + +// A page that references one image, so the compiler has a reason to copy it. +const POOL_FIXTURE = { + 'app.json': JSON.stringify({ pages: ['pages/index/index'] }), + 'project.config.json': JSON.stringify({ appid: 'binary_outputs_001', projectname: 'binary-outputs' }), + 'app.js': 'App({})\n', + 'pages/index/index.js': "Page({ data: { title: 'binary' } })\n", + 'pages/index/index.json': JSON.stringify({ navigationBarTitleText: 'binary' }), + 'pages/index/index.wxss': '.box { padding: 20rpx; }\n', + 'pages/index/index.wxml': '\n \n\n', + 'pages/index/logo.png': IMAGE_BYTES, +} + +// Each stage worker is its own dynamically-imported module instance (own closure state, +// mirroring a real Web Worker realm); ALL of them funnel through one shared `chain` so +// only one is ever in flight — the worker module resolves the bare `self` identifier +// against whatever `globalThis.self` currently is, so serializing every send keeps two +// realms' async continuations from racing over that shared global. +let chain = Promise.resolve() +let instanceCounter = 0 +async function makeStageWorker() { + const worker = { onmessage: null, onerror: null, terminate() {} } + const fakeSelf = { + onmessage: null, + // structuredClone is what a real postMessage does; without it this harness would + // hand the pool the very same Uint8Array instance and prove nothing about cloning. + postMessage(msg) { if (worker.onmessage) worker.onmessage({ data: structuredClone(msg) }) }, + } + globalThis.self = fakeSelf + instanceCounter += 1 + await import(`${WORKER_URL}?n=${instanceCounter}`) + // stage-worker.js does `self.onmessage = async (e) => {...}` at module top level + // against whatever `globalThis.self` was AT IMPORT TIME — capture it now. + const boundOnMessage = fakeSelf.onmessage + worker.postMessage = (msg) => { + const cloned = structuredClone(msg) + chain = chain.then(async () => { + globalThis.self = fakeSelf + try { + await boundOnMessage({ data: cloned }) + } catch (err) { + // The real handler already catches internally and posts { type:'error' }; an + // escaping exception would mean it threw before its own try, so surface it the + // same way rather than hanging the pool's pending request forever. + fakeSelf.postMessage({ type: 'error', error: String((err && err.stack) || err) }) + } + }) + } + return worker +} + +{ + const { createCompilerPool } = await import('../dist/pool.browser.js') + const stageWorkers = [] + for (const _stage of ['logic', 'view', 'style']) stageWorkers.push(await makeStageWorker()) + let nextWorker = 0 + const pool = createCompilerPool({ + createWorker: () => stageWorkers[nextWorker++], + toolchainSetupURL: TOOLCHAIN_URL, + }) + const out = await pool.compile({ files: POOL_FIXTURE, workPath: WORK_PATH }) + await pool.dispose() + + const products = Object.entries(out.files) + const match = products.find(([, v]) => v instanceof Uint8Array && sameBytes(v, IMAGE_BYTES)) + chk(!!match, + `PART A (pool.js + stage-worker.js): the referenced image came back byte-identical through structured cloning${match ? ` (as ${match[0]})` : ` — got ${JSON.stringify(products.filter(([k]) => k.endsWith('.png')).map(([k, v]) => [k, v instanceof Uint8Array ? `bytes(${v.length})` : typeof v]))}`}`) + chk(!!match && !sameBytes(POOL_FIXTURE['pages/index/logo.png'], new Uint8Array(0)), + 'PART A: and the source bytes it was compared against are non-empty') + const poolMojibake = products.filter(([, v]) => typeof v === 'string' && v.includes('�')) + chk(poolMojibake.length === 0, `PART A: no product came back with replacement characters (${poolMojibake.map(([k]) => k).join(', ') || 'none'})`) + const poolJs = out.files['main/pages_index_index.js'] + chk(typeof poolJs === 'string' && poolJs.length > 0, 'PART A: text products are still plain strings (the page module compiled)') +} + +// --- PART B: the compile seams, against the full `base` example ------------------ +globalThis.process = realProcess const { setupCompile, compileStage, collectOutputs, STAGE_NAMES } = await import('../dist/compile-core.node.js') @@ -75,5 +187,19 @@ chk(cssProducts.length > 0, `compiled CSS products are still strings (${cssProdu const mojibake = products.filter(([, v]) => typeof v === 'string' && v.includes('�')) chk(mojibake.length === 0, `no product came back with replacement characters (${mojibake.map(([k]) => k).join(', ') || 'none'})`) -console.log(failed ? `\n❌ ${failed} binary-output assertion(s) failed.` : '\n✅ referenced assets survive the compile byte-identical; text products are unchanged.') +// A product that starts with a UTF-8 BOM must come back with the BOM still on it. The +// compiler copies referenced assets verbatim, and a .svg or .json asset written by an +// editor that emits a BOM is valid UTF-8 — so it decodes to a string, and a decoder +// that eats the BOM would hand the host a file three bytes shorter than the one on +// disk, silently, with no bytes-vs-text signal to notice it by. +{ + const bomBytes = new Uint8Array([0xEF, 0xBB, 0xBF, ...new TextEncoder().encode('{"a":1}')]) + fs.writeFileSync(`${ctx.targetPath}/bom-asset.json`, bomBytes) + const collected = collectOutputs({ fs, targetPath: ctx.targetPath })['bom-asset.json'] + const roundTripped = typeof collected === 'string' ? new TextEncoder().encode(collected) : collected + chk(sameBytes(roundTripped, bomBytes), + `a BOM-prefixed product survives byte-identical (${bomBytes.length} bytes in, ${roundTripped ? roundTripped.length : 'nothing'} out)`) +} + +console.log(failed ? `\n❌ ${failed} binary-output assertion(s) failed.` : '\n✅ referenced assets survive the compile byte-identical, through the pool and through the seams; text products are unchanged.') process.exit(failed ? 1 : 0) diff --git a/packages/compiler/src/compile-core.js b/packages/compiler/src/compile-core.js index 11fe8548..2c436ebb 100644 --- a/packages/compiler/src/compile-core.js +++ b/packages/compiler/src/compile-core.js @@ -132,7 +132,10 @@ function ensureAppIdFs(fs, configPath) { // come back corrupted, so callers had to go read the fs themselves to get real // assets. Strict decoding is exact in both directions: text is byte-identical to // before, and anything that isn't valid UTF-8 stays raw bytes. -const utf8Strict = new TextDecoder('utf-8', { fatal: true }) +// ignoreBOM keeps a leading U+FEFF in the string instead of eating it: by default +// TextDecoder strips it, so a BOM-prefixed file would come out three bytes shorter +// than it went in — exactly the kind of silent rewrite this function exists to avoid. +const utf8Strict = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }) function decodeProduct(bytes) { // A fs backend that ignores the missing encoding and hands back a string is taken // at its word — same as before this function existed. From 82daa05f4aec5d932bbff77aaa28a0a176b1046b Mon Sep 17 00:00:00 2001 From: lbb00 Date: Thu, 3 Sep 2026 00:47:28 +0800 Subject: [PATCH 10/14] =?UTF-8?q?fix(compiler):=20.wasm=20=E7=AD=89?= =?UTF-8?q?=E4=BA=8C=E8=BF=9B=E5=88=B6=E4=BA=A7=E7=89=A9=E6=8C=89=E6=89=A9?= =?UTF-8?q?=E5=B1=95=E5=90=8D=E7=9B=B4=E6=8E=A5=E7=BB=99=E5=AD=97=E8=8A=82?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E5=86=8D=E9=9D=A0=E5=86=85=E5=AE=B9=E7=8C=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 最小的合法 .wasm 只有 8 字节头，整份都是合法 UTF-8，按内容判会被当成文本 交出去，宿主喂给 WebAssembly.instantiate() 直接类型错误。改成先看扩展名： 天生二进制的一律不解码，其余仍按严格 UTF-8 解，解不出来给字节。 同时把 test:binary-outputs 接进包的 test 脚本——这套回归测试之前写了但没跑。 --- packages/compiler/README.md | 6 ++--- packages/compiler/package.json | 5 ++-- .../compiler/scripts/test-binary-outputs.js | 14 ++++++++++- packages/compiler/src/compile-core.js | 24 +++++++++++++++++-- turbo.json | 10 ++++++++ 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/packages/compiler/README.md b/packages/compiler/README.md index 0fc38a2b..9733e240 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -373,13 +373,13 @@ async function filesFromDir(dir, prefix = '') { // 只读递来的 han - **产物写回同一个 fs。** compiler `writeFileSync` 把产物写进你的 fs，所以传入的 fs 必须**可写**。产物目录 `targetPath` 见下。 - **编译会修改你的 fs。** 缺 `project.config.json`/appid 时，会往 `${workPath}/project.config.json` 写入一个 appid（`dmlocalpreview`）——传入的 fs 不能当成只读快照。 - **同步契约，不需要 `fs.promises`。** `DiminaFs` 只要求同步方法（`existsSync`/`readFileSync`/`readdirSync{withFileTypes}`/`statSync`/`writeFileSync`/`mkdirSync{recursive}`/`copyFileSync`/`rmSync`）——编译路径不碰 async fs。纯异步后端（只有 Promise 版读写）没法当 fs 用。 -- **`readFileSync` 不带编码参数时必须返回字节。** `collectOutputs` 靠这个来区分文本产物和图片：先按严格 UTF-8 解码，解不出来就原样把字节交给调用方。后端若无视编码参数一律返回字符串，二进制产物就会在这一步坏掉。 +- **`readFileSync` 不带编码参数时必须返回字节。** `collectOutputs` 靠这个来区分文本产物和图片：扩展名摆明是二进制的直接给字节，其余按严格 UTF-8 解码，解不出来就原样把字节交给调用方。后端若无视编码参数一律返回字符串，二进制产物就会在这一步坏掉。 `@dimina/compiler` 自己并不知道 fs 被换掉了。 ## 已知限制与错误处理 -- **`files` 里的一条产物可能是字符串，也可能是 `Uint8Array`。** `collectOutputs` 对每条产物先按严格 UTF-8 解码，能解出来就是字符串（JS/CSS/JSON 等），解不出来（图片、字体等 compiler `copyFileSync` 到 `main/static` 的资源）就原样给字节。下游拿到一条产物当字符串用之前要先判类型——`typeof v === 'string'`。入参方向同理：源码里的图片直接以 `Uint8Array` 放进 `files` 即可，pool 会把它当文件写进 worker 的 memfs。 +- **`files` 里的一条产物可能是字符串，也可能是 `Uint8Array`。** `collectOutputs` 先看扩展名：`.wasm`、图片、字体、音视频、压缩包这些天生是二进制的，一律原样给字节，不解码——最小的合法 `.wasm` 只有 8 个字节的头，整份都是合法 UTF-8，光看内容判会把它当成文本交出去，宿主拿去喂 `WebAssembly.instantiate()` 直接类型错误。其余产物按严格 UTF-8 解码，能解出来就是字符串（JS/CSS/JSON 等），解不出来就原样给字节。下游拿到一条产物当字符串用之前要先判类型——`typeof v === 'string'`。入参方向同理：源码里的图片直接以 `Uint8Array` 放进 `files` 即可，pool 会把它当文件写进 worker 的 memfs。 - **`targetPath` 来自环境。** compiler 产物目录取 `process.env.TARGET_PATH`，否则 `os.tmpdir()/dimina-fe-dist-<时间戳>`（浏览器 os shim 下通常 `/tmp/...`）。`setupCompile` 会先 `rmSync` 清空它——别把 `TARGET_PATH` 指到源码目录或共享目录。用 `setupCompile` 返回的 `targetPath` 喂 `collectOutputs`。 - **不是全 fail-fast。** 缺 fs 方法、坏 appid、坏 `project.config.json`、miniprogram_npm 构建失败会 **reject**；但**样式预处理器失败（如当前浏览器构建暂不支持 `.less`）会被吞掉、降级用原始 CSS**，PostCSS 解析失败返回空串，资源拷贝失败只 `console.log`，logic esbuild 压缩失败回退未压缩代码。**用 pool 时把这些拿出来的办法是 `createCompilerPool({ onLog })`**——它把 worker 内编译器的 `console.*` 诊断（带 stage 标签）转发给你；也可在产物为空/缺失时二次校验。 @@ -437,7 +437,7 @@ pnpm --filter @dimina-kit/compiler build:types # 仅 dist/types/*.d.ts ## 测试 -`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）只包含不需要构建的三份契约测试——静态资源清单（`test:browser-assets`）、错误码（`test:error-codes`）和二进制入参播种（`test:binary-seed`）；下面这些各自要先构建，按需单跑。 +`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）包含四份契约测试——静态资源清单（`test:browser-assets`）、错误码（`test:error-codes`）、二进制入参播种（`test:binary-seed`）和二进制产物保真（`test:binary-outputs`）。最后一份要拿 `dist` 里的真实 bundle 跑，但它自己不构建：turbo 里 `@dimina-kit/compiler#test` 依赖本包的 `build`，构建只发生一次，测试期间没有人再往 `dist` 写。脱离 turbo 单跑时用带构建的 `test:binary-outputs`（`test:binary-outputs:prebuilt` 是不构建的那个入口）。下面这些各自要先构建，按需单跑。 测试里用 memfs 扮演「下游 fs」： diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 55393e5a..a2c29835 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -69,9 +69,10 @@ "test:realm-reuse": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-realm-reuse.js", "test:pool-node": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-node.js", "test:pool-scopehash": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-scopehash.js", - "test": "pnpm run test:browser-assets && pnpm run test:error-codes && pnpm run test:binary-seed", + "test": "pnpm run test:browser-assets && pnpm run test:error-codes && pnpm run test:binary-seed && pnpm run test:binary-outputs:prebuilt", "test:binary-seed": "node scripts/test-binary-seed.js", - "test:binary-outputs": "node scripts/build-compiler.js node && node scripts/build-compiler.js browser && node --import ./scripts/register-kit.js scripts/test-binary-outputs.js", + "test:binary-outputs": "node scripts/build-compiler.js node && node scripts/build-compiler.js browser && pnpm run test:binary-outputs:prebuilt", + "test:binary-outputs:prebuilt": "node --import ./scripts/register-kit.js scripts/test-binary-outputs.js", "test:error-codes": "node scripts/test-error-codes.js", "test:browser-assets": "node scripts/test-browser-assets.js", "test:crypto-shim": "node scripts/test-crypto-shim.js", diff --git a/packages/compiler/scripts/test-binary-outputs.js b/packages/compiler/scripts/test-binary-outputs.js index 9a6fb207..157e690a 100644 --- a/packages/compiler/scripts/test-binary-outputs.js +++ b/packages/compiler/scripts/test-binary-outputs.js @@ -201,5 +201,17 @@ chk(mojibake.length === 0, `no product came back with replacement characters (${ `a BOM-prefixed product survives byte-identical (${bomBytes.length} bytes in, ${roundTripped ? roundTripped.length : 'nothing'} out)`) } -console.log(failed ? `\n❌ ${failed} binary-output assertion(s) failed.` : '\n✅ referenced assets survive the compile byte-identical, through the pool and through the seams; text products are unchanged.') +// A .wasm module must come back as bytes even though it decodes cleanly as UTF-8. The +// 8-byte header below is a complete, legal module, and every one of its bytes is valid +// UTF-8 — decoding by content alone hands the host a string, and WebAssembly.instantiate() +// takes bytes only. +{ + const wasmBytes = new Uint8Array([0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00]) + fs.writeFileSync(`${ctx.targetPath}/tiny.wasm`, wasmBytes) + const collected = collectOutputs({ fs, targetPath: ctx.targetPath })['tiny.wasm'] + chk(collected instanceof Uint8Array && sameBytes(collected, wasmBytes), + `a .wasm product stays bytes even though it is valid UTF-8 (got ${collected instanceof Uint8Array ? `bytes(${collected.length})` : typeof collected})`) +} + +console.log(failed ? `\n❌ ${failed} binary-output assertion(s) failed.` :'\n✅ referenced assets survive the compile byte-identical, through the pool and through the seams; text products are unchanged.') process.exit(failed ? 1 : 0) diff --git a/packages/compiler/src/compile-core.js b/packages/compiler/src/compile-core.js index 2c436ebb..3d8eaaf7 100644 --- a/packages/compiler/src/compile-core.js +++ b/packages/compiler/src/compile-core.js @@ -136,10 +136,27 @@ function ensureAppIdFs(fs, configPath) { // TextDecoder strips it, so a BOM-prefixed file would come out three bytes shorter // than it went in — exactly the kind of silent rewrite this function exists to avoid. const utf8Strict = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }) -function decodeProduct(bytes) { + +// "Does it decode as UTF-8" catches PNGs and fonts, but it is not a reliable test for +// "is this text": short binary files can be valid UTF-8 by accident. The smallest legal +// .wasm module is its 8-byte header (00 61 73 6D 01 00 00 00) — every byte decodes, so +// it would come back as a string and blow up in WebAssembly.instantiate(), which takes +// bytes only. For file types that are binary by definition, skip the decode entirely. +// Extension-only, so a text product can never be pushed onto this list by its contents. +const BINARY_EXTS = new Set([ + '.wasm', + '.png', '.jpg', '.jpeg', '.gif', '.webp', '.avif', '.bmp', '.ico', '.tif', '.tiff', + '.ttf', '.otf', '.woff', '.woff2', '.eot', + '.mp3', '.wav', '.ogg', '.m4a', '.mp4', '.webm', '.mov', + '.pdf', '.zip', '.gz', '.br', +]) + +function decodeProduct(bytes, relPath) { // A fs backend that ignores the missing encoding and hands back a string is taken // at its word — same as before this function existed. if (typeof bytes === 'string') return bytes + const dot = relPath.lastIndexOf('.') + if (dot > relPath.lastIndexOf('/') && BINARY_EXTS.has(relPath.slice(dot).toLowerCase())) return bytes try { return utf8Strict.decode(bytes) } catch { @@ -169,7 +186,10 @@ function readOutputs(fs, target) { } const full = `${dir}/${e.name}` if (e.isDirectory()) walk(full) - else out[full.slice(prefix.length)] = decodeProduct(fs.readFileSync(full)) + else { + const rel = full.slice(prefix.length) + out[rel] = decodeProduct(fs.readFileSync(full), rel) + } } } walk(prefix.slice(0, -1)) diff --git a/turbo.json b/turbo.json index d9441a5b..81ac47f1 100644 --- a/turbo.json +++ b/turbo.json @@ -45,6 +45,16 @@ "cache": true, "outputs": ["test-report*.json", "coverage/**"] }, + // 全局的 test 只依赖 ^build（上游包），compiler 自己的 dist 不在其中。而 compiler + // 的几份契约测试要拿 dist 里的真实 bundle 来跑：它们自己 build 的话，就会跟 + // devkit#test 触发的 compiler#build 同时往同一个 dist 写——那份 build 会先删 chunk + // 目录再重写，正在读产物的另一个测试就会撞上缺文件。所以这里让 compiler#test 依赖 + // 自己的 build，由 turbo 保证先构建、且只构建一次，测试脚本本身不再构建。 + "@dimina-kit/compiler#test": { + "dependsOn": ["^build", "build"], + "cache": true, + "outputs": ["test-report*.json", "coverage/**"] + }, "test:coverage": { "dependsOn": ["^build"], "cache": true, From 678edeb789366fc46fff9553a65b594f7c56be50 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 21:16:39 +0800 Subject: [PATCH 11/14] =?UTF-8?q?feat(compiler):=20=E6=96=B9=E8=A8=80?= =?UTF-8?q?=E6=89=A9=E5=B1=95=E5=90=8D=E6=9C=89=E4=BA=86=E4=B8=80=E4=BB=BD?= =?UTF-8?q?=E6=9D=83=E5=A8=81=E5=A3=B0=E6=98=8E=EF=BC=8C=E5=AE=BF=E4=B8=BB?= =?UTF-8?q?=E4=B8=8D=E7=94=A8=E5=86=8D=E5=90=84=E6=8A=84=E4=B8=80=E9=81=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `options.fileTypes` 让工程用自己的扩展名（qd 方言的 .qdml/.qdss/.qds 对应 wxml/wxss/wxs）。但这份配置不止编译器要用：编辑器的语言映射、模板校验、预览时 找页面模板，宿主自己也要按同一套规则判断文件角色。现在各处手写 `/\.(wxml|qdml)$/`，漏掉内置的 .ddml 不显眼，真正的问题是同一个方言在几个仓库 里各抄一份，谁改了另一边不知道。 新增 `@dimina-kit/compiler/file-types`（ESM + CJS，无依赖）： - `QD_FILE_TYPES`：qd 方言那一份配置。 - `resolveFileTypes(fileTypes)`：算出编译器这次实际会认的扩展名和内联标签， 合并内置项、规范化、去重、丢掉占用其他角色或 .js/.ts/.json 的项。 - `hasExt(path, exts)`：大小写不敏感的尾部匹配。 合并规则照编译器 env.js 的 normalizeFileTypes 写（直接 import 会把 node:fs 和 整个配置解析拉进来）。抄来的东西会漂，所以 test:file-types 直接读 env.js 源码 比对内置列表、保留扩展名和两条校验正则——上游加一种内置方言或改一条正则，这里 就红。test:pool-filetypes 改成用 QD_FILE_TYPES 本身，顺带证明导出的这份配置就是 能让 .qdml 工程编出来的那份。 --- packages/compiler/README.md | 22 ++- packages/compiler/package.json | 11 +- packages/compiler/scripts/build-compiler.js | 23 +-- packages/compiler/scripts/test-file-types.js | 93 +++++++++++ .../compiler/scripts/test-pool-filetypes.js | 8 +- packages/compiler/src/compile-core.js | 2 +- packages/compiler/src/file-types.js | 158 ++++++++++++++++++ packages/compiler/src/pool-node.js | 2 +- packages/compiler/src/pool.js | 2 +- packages/compiler/tsconfig.types.json | 3 +- packages/compiler/types-fixture/consumer.ts | 21 ++- 11 files changed, 324 insertions(+), 21 deletions(-) create mode 100644 packages/compiler/scripts/test-file-types.js create mode 100644 packages/compiler/src/file-types.js diff --git a/packages/compiler/README.md b/packages/compiler/README.md index 9733e240..1641ea2e 100644 --- a/packages/compiler/README.md +++ b/packages/compiler/README.md @@ -60,6 +60,24 @@ for (const file of files) fs.copyFileSync(file, path.join(publicDir, path.basena 每次浏览器构建都会拿 esbuild 的 metafile 对着这份清单自检：产物改了名、被挪进子目录、被拆出新 chunk、或某个资源开始 import 别的文件（`import` / `require` / 动态 import 都算，一带上就不再是自包含的单文件），以及清单里的文件名和 `package.json` 的 exports 对不上（改名只改了一边），构建当场失败，而不是几个月后在某个宿主那里 404。`toolchain.browser.js` 不在清单里——它由宿主用自己的打包器 import（`@dimina-kit/compiler/toolchain`），拷过去也没人 fetch。 +### 自定义文件类型（方言）：声明一次，编译器和宿主用同一份 + +`options.fileTypes` 让工程用自己的扩展名，比如千岛（qd）方言的 `.qdml`/`.qdss`/`.qds` 对应 `.wxml`/`.wxss`/`.wxs`。麻烦的是这份配置不止编译器要用：编辑器的语言映射、模板校验、预览时找页面模板，宿主自己也要按同一套规则判断文件角色。各处手写 `/\.(wxml|qdml)$/` 的结果是漏掉内置的 `.ddml`，而且同一个方言在几个仓库里各抄一份，谁改了另一边不会知道。 + +`@dimina-kit/compiler/file-types` 就是那一份（同时发 ESM 和 CJS，无依赖）： + +```js +import { QD_FILE_TYPES, resolveFileTypes, hasExt } from '@dimina-kit/compiler/file-types' + +await pool.compile({ files, workPath, options: { fileTypes: QD_FILE_TYPES } }) + +const { templateExts, styleExts, viewScriptExts, viewScriptTags } = resolveFileTypes(QD_FILE_TYPES) +// templateExts: ['.wxml', '.ddml', '.qdml'] ← 内置在前，自定义在后，顺序即查找优先级 +hasExt('pages/index/index.QDML', templateExts) // true，大小写不敏感 +``` + +`resolveFileTypes()` 算的是**编译器这次实际会认的**扩展名和内联标签：合并内置项、规范化（去空白、转小写、补一个前导点）、去重，并丢掉占用其他角色或 `.js`/`.ts`/`.json` 的项——`template: ['js']` 会把页面逻辑当模板解析，所以直接不接受。规则是照编译器 `env.js` 的 `normalizeFileTypes` 写的（直接 import 它会把 `node:fs` 和整个配置解析一起拉进来），`test:file-types` 读 env.js 源码比对内置列表和两条校验正则，上游一改这里就红。 + ## 架构 本包是**编译器与文件系统之间的一层适配**，再往上叠一层**编排**。真正的编译逻辑在 `dimina` 子模块的 `@dimina/compiler`，本包用一个**无后端的 fs 转发 shim** 把它每一次 `fs.xxx` 指向下游注入的 fs；`pool` 则在上面替下游管好 worker 池与并行——下游不再手写任何 worker/合并逻辑。 @@ -437,7 +455,7 @@ pnpm --filter @dimina-kit/compiler build:types # 仅 dist/types/*.d.ts ## 测试 -`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）包含四份契约测试——静态资源清单（`test:browser-assets`）、错误码（`test:error-codes`）、二进制入参播种（`test:binary-seed`）和二进制产物保真（`test:binary-outputs`）。最后一份要拿 `dist` 里的真实 bundle 跑，但它自己不构建：turbo 里 `@dimina-kit/compiler#test` 依赖本包的 `build`，构建只发生一次，测试期间没有人再往 `dist` 写。脱离 turbo 单跑时用带构建的 `test:binary-outputs`（`test:binary-outputs:prebuilt` 是不构建的那个入口）。下面这些各自要先构建，按需单跑。 +`pnpm --filter @dimina-kit/compiler test`（也就是 `turbo run test` 会跑到的那份）包含六份契约测试——静态资源清单（`test:browser-assets`）、错误码（`test:error-codes`）、二进制入参播种（`test:binary-seed`）、二进制产物保真（`test:binary-outputs`）、方言扩展名（`test:file-types`）和方言穿过编译池（`test:pool-filetypes`）。后两类里要拿 `dist` 真实 bundle 跑的那几份自己不构建：turbo 里 `@dimina-kit/compiler#test` 依赖本包的 `build`，构建只发生一次，测试期间没有人再往 `dist` 写。脱离 turbo 单跑时用带构建的 `test:binary-outputs` / `test:pool-filetypes`（带 `:prebuilt` 后缀的是不构建的那个入口）。下面这些各自要先构建，按需单跑。 测试里用 memfs 扮演「下游 fs」： @@ -455,6 +473,7 @@ pnpm --filter @dimina-kit/compiler test:stage-load-retry # stage 工 pnpm --filter @dimina-kit/compiler test:browser-assets # 静态资源清单:改名/新 chunk/出现静态 import 都会被构建期检查拦下 pnpm --filter @dimina-kit/compiler test:error-codes # 错误码:worker 自己判定的失败(工具链导入)带着码原样传到调用方,其余记为 compiler-stage-error pnpm --filter @dimina-kit/compiler test:stage-toolchain # 真实 stage worker bundle:每个 stage 都加载工具链、按 URL 记忆、导入失败带错误码 +pnpm --filter @dimina-kit/compiler test:file-types # 方言扩展名:合并/规范化行为,以及内置列表与校验正则和编译器 env.js 逐字一致 pnpm --filter @dimina-kit/compiler test:binary-seed # 入参里的 Uint8Array 播种成文件(memfs 自己的 fromJSON 会把它变成目录) pnpm --filter @dimina-kit/compiler test:binary-outputs # 页面引用的图片走完整编译后逐字节相同,文本产物仍是字符串 ``` @@ -471,6 +490,7 @@ pnpm --filter @dimina-kit/compiler test:binary-outputs # 页面引 - `src/pool-node.js` — **Node 编排池** `createNodeCompilerPool` + dmcc drop-in 默认导出 `build()`：常驻 worker_threads、真实磁盘、全局 build 串行、死 worker 懒复活、idle 自动收缩（`idleShrinkMs`）。 - `src/stage-worker-node.js` — Node 常驻 stage worker：spawn 时按 workerData 里的 stage 身份预热本 stage 工具链，恢复 storeInfo → `runStage(stage, { sourcemap })` 写共享 staging 目录；应答 `{ type: 'introspect' }` 报告本 realm 已加载的重依赖。 - `src/toolchain.js` — 写 `toolchainSetupURL` 模块的可选助手（`installOxc` / `installEsbuildFromURL`，后者内置 esbuild-wasm 静态资源的 Blob-URL 兜底）。导出为 `@dimina-kit/compiler/toolchain`。 +- `src/file-types.js` — 自定义文件类型（方言）的权威声明：内置扩展名、合并规则 `resolveFileTypes`、qd 方言常量 `QD_FILE_TYPES`。导出为 `@dimina-kit/compiler/file-types`（ESM + CJS）。 - `src/browser-assets.js` — 浏览器静态资源清单与契约（`COMPILER_BROWSER_ASSETS` / `resolveBrowserAssets`，见上文），构建期检查也用它。导出为 `@dimina-kit/compiler/browser-assets`。 - `src/error-codes.js` — 两个 pool 共用的错误码表 `COMPILER_ERROR_CODES` 与判定 `isInfrastructureError`（见上文），经 `./pool` 与 `./pool-node` 再导出。 - `src/failure-hints.js` — Node 侧「一条原始报错文字该记哪个码」的判定（`errorCodeForMessage` / `tagFailure`），以及 oxc 绑定缺失、esbuild 二进制被封在 app.asar 这两种打包问题的中文提示（`oxcNativeBindingHint` / `esbuildAsarSpawnHint`，经 `./pool-node` 再导出）。单独成文件是为了让它不牵连 `worker_threads` 和编译器实体，`test:error-codes` 能直接驱动。 diff --git a/packages/compiler/package.json b/packages/compiler/package.json index a2c29835..c0f4485a 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -47,6 +47,11 @@ "require": "./dist/browser-assets.cjs", "default": "./dist/browser-assets.js" }, + "./file-types": { + "types": "./dist/types/file-types.d.ts", + "require": "./dist/file-types.cjs", + "default": "./dist/file-types.js" + }, "./package.json": "./package.json" }, "files": [ @@ -69,7 +74,8 @@ "test:realm-reuse": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-realm-reuse.js", "test:pool-node": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-node.js", "test:pool-scopehash": "node scripts/build-compiler.js node && node --import ./scripts/register-kit.js scripts/test-pool-scopehash.js", - "test": "pnpm run test:browser-assets && pnpm run test:error-codes && pnpm run test:binary-seed && pnpm run test:binary-outputs:prebuilt", + "test": "pnpm run test:browser-assets && pnpm run test:error-codes && pnpm run test:binary-seed && pnpm run test:binary-outputs:prebuilt && pnpm run test:file-types && pnpm run test:pool-filetypes:prebuilt", + "test:file-types": "node scripts/test-file-types.js", "test:binary-seed": "node scripts/test-binary-seed.js", "test:binary-outputs": "node scripts/build-compiler.js node && node scripts/build-compiler.js browser && pnpm run test:binary-outputs:prebuilt", "test:binary-outputs:prebuilt": "node --import ./scripts/register-kit.js scripts/test-binary-outputs.js", @@ -88,7 +94,8 @@ "test:stage-toolchain": "node scripts/build-compiler.js browser && node scripts/test-stage-toolchain.js", "test:stage-worker-message-order": "node scripts/build-compiler.js node && node scripts/test-stage-worker-message-order.js", "test:stage-load-retry": "node scripts/build-compiler.js node && node scripts/test-stage-load-retry.js", - "test:pool-filetypes": "node scripts/build-compiler.js node && node scripts/build-compiler.js browser && node scripts/test-pool-filetypes.js" + "test:pool-filetypes": "node scripts/build-compiler.js node && node scripts/build-compiler.js browser && pnpm run test:pool-filetypes:prebuilt", + "test:pool-filetypes:prebuilt": "node scripts/test-pool-filetypes.js" }, "dependencies": { "@babel/parser": "^7.29.7", diff --git a/packages/compiler/scripts/build-compiler.js b/packages/compiler/scripts/build-compiler.js index b25d894c..705ab119 100644 --- a/packages/compiler/scripts/build-compiler.js +++ b/packages/compiler/scripts/build-compiler.js @@ -246,17 +246,20 @@ for (const b of builds) { // The static-asset manifest itself: dependency-free string code, emitted in both // modes (either build alone leaves a usable dist) and in both formats, because the // hosts that copy these files are as often CommonJS build scripts as ESM ones. -for (const [format, outfile] of [['esm', 'browser-assets.js'], ['cjs', 'browser-assets.cjs']]) { - await esbuild.build({ - entryPoints: [path.join(root, 'src/browser-assets.js')], - outfile: path.join(root, 'dist', outfile), - bundle: true, - format, - target: ['es2022'], - logLevel: 'warning', - }) +// src/file-types.js（方言扩展名）走同一条路：宿主的编辑器配置和文件分类也常常在 CJS 里。 +for (const name of ['browser-assets', 'file-types']) { + for (const [format, ext] of [['esm', 'js'], ['cjs', 'cjs']]) { + await esbuild.build({ + entryPoints: [path.join(root, `src/${name}.js`)], + outfile: path.join(root, 'dist', `${name}.${ext}`), + bundle: true, + format, + target: ['es2022'], + logLevel: 'warning', + }) + } + console.log(`✅ built dist/${name}.js + dist/${name}.cjs`) } -console.log('✅ built dist/browser-assets.js + dist/browser-assets.cjs') // The browser bundles double as static files a host copies and serves. Their names // and the "self-contained, imports nothing" rule are stated once in diff --git a/packages/compiler/scripts/test-file-types.js b/packages/compiler/scripts/test-file-types.js new file mode 100644 index 00000000..aa302c18 --- /dev/null +++ b/packages/compiler/scripts/test-file-types.js @@ -0,0 +1,93 @@ +// src/file-types.js 把编译器的合并规则抄了一份出来（env.js 直接 import 会拉进 node:fs 和 +// 整个配置解析，宿主只想知道扩展名）。抄来的东西会漂：所以这里直接读 env.js 源码，比对内置 +// 列表、保留扩展名和两条规范化正则——上游加一种内置方言或改一条正则，这个测试就红，而不是 +// 等某个宿主把文件分错类。后半段是合并行为本身。 +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { + BUILTIN_STYLE_EXTS, + BUILTIN_TEMPLATE_DIRECTIVE_PREFIXES, + BUILTIN_TEMPLATE_EXTS, + BUILTIN_VIEW_SCRIPT_EXTS, + BUILTIN_VIEW_SCRIPT_TAGS, + QD_FILE_TYPES, + RESERVED_EXTS, + hasExt, + resolveFileTypes, +} from '../src/file-types.js' + +let failed = 0 +const chk = (cond, msg) => { if (cond) { console.log(`✅ ${msg}`) } else { console.log(`❌ ${msg}`); failed++ } } +const same = (a, b) => JSON.stringify(a) === JSON.stringify(b) + +const ENV_PATH = fileURLToPath(new URL('../../../dimina/fe/packages/compiler/src/env.js', import.meta.url)) +const env = readFileSync(ENV_PATH, 'utf8') + +const listOf = (name) => { + const m = env.match(new RegExp(`const ${name} = \\[([^\\]]*)\\]`)) + if (!m) throw new Error(`${name} 不在 ${ENV_PATH} 里了——编译器改了内置文件类型的写法，先看它现在怎么写`) + return [...m[1].matchAll(/'([^']*)'/g)].map((x) => x[1]) +} +const regexOf = (fnName) => { + const start = env.indexOf(`function ${fnName}(`) + if (start < 0) throw new Error(`${fnName} 不在 ${ENV_PATH} 里了——先看编译器现在怎么校验扩展名`) + const m = env.slice(start).match(/!(\/\S+?\/)\.test\(v\)/) + if (!m) throw new Error(`${fnName} 里的规范化正则找不到了——先看编译器现在怎么校验扩展名`) + return m[1] +} + +chk(same([...BUILTIN_TEMPLATE_EXTS], listOf('DEFAULT_TEMPLATE_EXTS')), + `内置模板扩展名与编译器一致（${BUILTIN_TEMPLATE_EXTS.join(' ')}）`) +chk(same([...BUILTIN_TEMPLATE_DIRECTIVE_PREFIXES], listOf('DEFAULT_TEMPLATE_DIRECTIVE_PREFIXES')), + `内置模板指令前缀与编译器一致（${BUILTIN_TEMPLATE_DIRECTIVE_PREFIXES.join(' ')}）`) +chk(same([...BUILTIN_STYLE_EXTS], listOf('DEFAULT_STYLE_EXTS')), + `内置样式扩展名与编译器一致（${BUILTIN_STYLE_EXTS.join(' ')}）`) +chk(same([...BUILTIN_VIEW_SCRIPT_EXTS], listOf('DEFAULT_VIEW_SCRIPT_EXTS')), + `内置视图脚本扩展名与编译器一致（${BUILTIN_VIEW_SCRIPT_EXTS.join(' ')}）`) +chk(same([...BUILTIN_VIEW_SCRIPT_TAGS], listOf('DEFAULT_VIEW_SCRIPT_TAGS')), + `内置视图脚本标签与编译器一致（${BUILTIN_VIEW_SCRIPT_TAGS.join(' ')}）`) + +{ + const block = env.slice(env.indexOf('const RESERVED_EXTS = new Set([')) + const extra = [...block.slice(0, block.indexOf('])')).matchAll(/'([^']*)'/g)].map((x) => x[1]) + chk(same(RESERVED_EXTS.filter((e) => !BUILTIN_TEMPLATE_EXTS.includes(e) && !BUILTIN_STYLE_EXTS.includes(e) && !BUILTIN_VIEW_SCRIPT_EXTS.includes(e)), extra), + `保留扩展名里非内置的那几个与编译器一致（${extra.join(' ')}）`) +} + +chk(regexOf('normalizeExt') === '/^[a-z0-9_-]+$/', `扩展名校验正则与编译器一致（${regexOf('normalizeExt')}）`) +chk(regexOf('normalizeTag') === '/^[a-z][a-z0-9_-]*$/', `标签名校验正则与编译器一致（${regexOf('normalizeTag')}）`) + +// 合并行为 +{ + const r = resolveFileTypes(QD_FILE_TYPES) + chk(same(r.templateExts, ['.wxml', '.ddml', '.qdml']), `qd 方言的模板扩展名（${r.templateExts.join(' ')}）`) + chk(same(r.styleExts, ['.wxss', '.ddss', '.less', '.scss', '.sass', '.qdss']), `qd 方言的样式扩展名（${r.styleExts.join(' ')}）`) + chk(same(r.viewScriptExts, ['.wxs', '.qds']), `qd 方言的视图脚本扩展名（${r.viewScriptExts.join(' ')}）`) + chk(same(r.viewScriptTags, ['wxs', 'dds', 'qds']), `视图脚本扩展名同时派生内联标签（${r.viewScriptTags.join(' ')}）`) + chk(r.templateDirectivePrefixes.includes('qd'), `自定义模板扩展名派生出指令前缀（${r.templateDirectivePrefixes.join(' ')}）`) +} + +{ + const r = resolveFileTypes() + chk(same(r.templateExts, [...BUILTIN_TEMPLATE_EXTS]), '不传 fileTypes 就只有内置项') + r.templateExts.push('.mine') + chk(!BUILTIN_TEMPLATE_EXTS.includes('.mine'), '返回的是副本，改它不会污染内置列表') +} + +chk(same(resolveFileTypes({ template: ['js', 'ts', 'json', 'wxss'] }).templateExts, ['.wxml', '.ddml']), + '占用逻辑/配置/其他角色扩展名的自定义项被丢弃') +chk(same(resolveFileTypes({ template: ['.QDML', 'qdml', ' qdml '] }).templateExts, ['.wxml', '.ddml', '.qdml']), + '带点、大写、带空白的写法规范化成同一项，且只留一份') +chk(same(resolveFileTypes({ template: ['a/b', 'q*d', '', ' '] }).templateExts, ['.wxml', '.ddml']), + '带路径分隔符或元字符的项被丢弃') +chk(same(resolveFileTypes({ viewScript: ['9qd'] }).viewScriptTags, ['wxs', 'dds']), + '数字开头不能当标签名（扩展名可以，标签名不行）') +chk(same(resolveFileTypes({ viewScript: ['9qd'] }).viewScriptExts, ['.wxs', '.9qd']), + '同一项作为扩展名仍然有效') + +chk(hasExt('pages/index/index.QDML', resolveFileTypes(QD_FILE_TYPES).templateExts), 'hasExt 大小写不敏感') +chk(!hasExt('pages/index/index.json', resolveFileTypes(QD_FILE_TYPES).templateExts), 'hasExt 不误判 .json') +chk(!hasExt(undefined, BUILTIN_TEMPLATE_EXTS), 'hasExt 对非字符串返回 false') + +console.log(failed ? `\n❌ ${failed} 条文件类型断言失败。` : '\n✅ 方言常量与编译器的合并规则一致。') +process.exit(failed ? 1 : 0) diff --git a/packages/compiler/scripts/test-pool-filetypes.js b/packages/compiler/scripts/test-pool-filetypes.js index 949a8162..c784583f 100644 --- a/packages/compiler/scripts/test-pool-filetypes.js +++ b/packages/compiler/scripts/test-pool-filetypes.js @@ -11,8 +11,7 @@ // options-threading fix and passes (green) with it — no reimplementation of the // protocol handlers under test. // -// Fixture mirrors qdmp's e2e "qd-app" (see -// ~/code/qdmp/main/packages/qdmp-devtools/e2e/qd-app): a page using .qdml/.qdss/.qds +// Fixture is a minimal qd-dialect app: a page using .qdml/.qdss/.qds // instead of .wxml/.wxss/.wxs, with a view // script and a `{{ m.shout(title) }}` mustache expression — so a correct compile // must recognize the custom template AND the custom view-script extension. @@ -26,6 +25,7 @@ // must run first. import { transform } from 'esbuild' import 'oxc-parser' +import { QD_FILE_TYPES } from '../src/file-types.js' await transform('const __warm = 1', {}) @@ -53,7 +53,9 @@ const FIXTURE_FILES = { 'pages/index/index.json': JSON.stringify({ navigationBarTitleText: 'QD Ext Index' }), 'pages/index/index.qds': "function shout(text) {\n return text + '!'\n}\n\nmodule.exports = {\n shout: shout,\n}\n", } -const FILE_TYPES_OPTIONS = { fileTypes: { template: ['qdml'], style: ['qdss'], viewScript: ['qds'] } } +// 用导出的常量本身，而不是再抄一份字面量：这样这个测试同时证明 QD_FILE_TYPES 就是能让 +// .qdml/.qdss/.qds 工程编出来的那份配置。 +const FILE_TYPES_OPTIONS = { fileTypes: QD_FILE_TYPES } let failed = false const chk = (cond, msg) => { if (!cond) { failed = true; console.error(`❌ ${msg}`) } else console.log(`✅ ${msg}`) } diff --git a/packages/compiler/src/compile-core.js b/packages/compiler/src/compile-core.js index 3d8eaaf7..54688a89 100644 --- a/packages/compiler/src/compile-core.js +++ b/packages/compiler/src/compile-core.js @@ -516,7 +516,7 @@ let compileChain = Promise.resolve() * Compile a mini-program against a caller-injected fs. Calls are serialized per * realm (see the singleton note above). Convenience wrapper that runs * `setupCompile` + all stages + `collectOutputs` in one realm. - * @param {{ fs: object, workPath?: string, options?: { fileTypes?: { template?: string[], style?: string[], viewScript?: string[] } } }} opts + * @param {{ fs: object, workPath?: string, options?: { fileTypes?: { template?: readonly string[], style?: readonly string[], viewScript?: readonly string[] } } }} opts * fs: a node:fs replacement (sync subset: existsSync/readFileSync/ * readdirSync{withFileTypes}/statSync/writeFileSync/mkdirSync{recursive}/ * copyFileSync/rmSync), already seeded with the project source under diff --git a/packages/compiler/src/file-types.js b/packages/compiler/src/file-types.js new file mode 100644 index 00000000..96f54edb --- /dev/null +++ b/packages/compiler/src/file-types.js @@ -0,0 +1,158 @@ +/** + * 自定义文件类型（方言）的权威声明，供宿主与编译器共用。 + * + * 编译器认哪些扩展名，是 `options.fileTypes` 决定的（见 README「自定义文件类型」）。 + * 但宿主除了把这份配置传给编译器，自己也要按同一套规则判断文件角色——编辑器语言映射、 + * 模板校验、预览时找页面模板。它们各自手写 `/\.(wxml|qdml)$/` 这类正则时，漏掉内置的 + * `.ddml` 只是不显眼，真正的问题是同一个方言在几个仓库里各抄一份，谁改了另一边不会知道。 + * + * 所以这里把两件事放在一处：内置扩展名与合并规则（`resolveFileTypes`），以及千岛（qd） + * 方言这份具体配置（`QD_FILE_TYPES`）。宿主 import 它，而不是再抄一遍。 + * + * 合并规则是照着编译器 `env.js` 的 `normalizeFileTypes` 写的（本包不能直接 import 它： + * env.js 会拉进 node:fs 和整个配置解析）。`test:file-types` 直接读 env.js 源码比对内置 + * 列表和两条规范化正则，上游一改这里就红。 + * + * 纯字符串处理、无依赖，任何运行时都能加载；同时发 ESM（dist/file-types.js）和 + * CJS（dist/file-types.cjs），因为 Electron 主进程那侧常常是 CommonJS。 + */ + +/** 内置模板扩展名。顺序即同名文件的查找优先级。 */ +export const BUILTIN_TEMPLATE_EXTS = Object.freeze(['.wxml', '.ddml']) +/** 内置模板指令前缀（`wx:if` 的 `wx`）。自定义模板扩展名会再派生一个。 */ +export const BUILTIN_TEMPLATE_DIRECTIVE_PREFIXES = Object.freeze(['wx', 'dd', 'a']) +/** 内置样式扩展名，含预处理器。 */ +export const BUILTIN_STYLE_EXTS = Object.freeze(['.wxss', '.ddss', '.less', '.scss', '.sass']) +/** 内置视图脚本扩展名。 */ +export const BUILTIN_VIEW_SCRIPT_EXTS = Object.freeze(['.wxs']) +/** 内置视图脚本内联标签（``）。 */ +export const BUILTIN_VIEW_SCRIPT_TAGS = Object.freeze(['wxs', 'dds']) + +/** + * 自定义项不得占用的扩展名：所有内置角色 + 逻辑（.js/.ts）+ 配置（.json）。 + * 占用会导致跨角色串编——`template: ['js']` 会把页面逻辑当模板解析。 + */ +export const RESERVED_EXTS = Object.freeze([ + ...BUILTIN_TEMPLATE_EXTS, + ...BUILTIN_STYLE_EXTS, + ...BUILTIN_VIEW_SCRIPT_EXTS, + '.js', + '.ts', + '.json', +]) + +/** + * 千岛（qd）方言：`.qdml`/`.qdss`/`.qds` 分别对应 `.wxml`/`.wxss`/`.wxs`。 + * 编译器、编辑器语言映射和宿主自己的文件分类都以这一份为准。 + * @type {Readonly} + */ +export const QD_FILE_TYPES = Object.freeze({ + template: Object.freeze(['qdml']), + style: Object.freeze(['qdss']), + viewScript: Object.freeze(['qds']), +}) + +/** + * 列表声明成只读：本包发出去的 `QD_FILE_TYPES` 是冻结的，写成可变数组的话，下游 + * 「再 push 一个扩展名」能通过类型检查，运行时才抛 TypeError。传入方向不受影响， + * 普通数组照样能喂给 `resolveFileTypes`。 + * + * @typedef {object} FileTypes + * @property {readonly string[]} [template] 追加的模板扩展名，如 ['qdml']（点可带可不带） + * @property {readonly string[]} [style] 追加的样式扩展名 + * @property {readonly string[]} [viewScript] 追加的视图脚本扩展名，同时派生同名内联标签 + */ + +/** + * @typedef {object} ResolvedFileTypes + * @property {string[]} templateExts 内置在前、自定义在后 + * @property {string[]} templateDirectivePrefixes 模板指令前缀 + * @property {string[]} styleExts + * @property {string[]} viewScriptExts + * @property {string[]} viewScriptTags 内联标签名（不带点） + */ + +/** + * 规范化成扩展名：去空白、转小写、补一个前导点。只接受字母、数字、连字符和下划线； + * 空串、路径分隔符和其他元字符返回 null 由调用方丢弃——扩展名会用来拼尾部匹配正则。 + * @param {unknown} raw + * @returns {string | null} + */ +function normalizeExt(raw) { + if (typeof raw !== 'string') return null + const v = raw.trim().toLowerCase().replace(/^\.+/, '') + if (!/^[a-z0-9_-]+$/.test(v)) return null + return `.${v}` +} + +/** + * 规范化成内联标签名：同上但必须以字母开头且不带点——标签名会拼进选择器， + * 放行元字符会让 `'qds,view'` 误选到 。 + * @param {unknown} raw + * @returns {string | null} + */ +function normalizeTag(raw) { + if (typeof raw !== 'string') return null + const v = raw.trim().toLowerCase().replace(/^\.+/, '') + if (!/^[a-z][a-z0-9_-]*$/.test(v)) return null + return v +} + +/** + * @param {readonly string[]} builtins + * @param {unknown} custom + * @param {(raw: unknown) => string | null} normalizer + * @param {Set} [reserved] + * @returns {string[]} + */ +function mergeUnique(builtins, custom, normalizer, reserved) { + const out = [...builtins] + const seen = new Set(out) + if (Array.isArray(custom)) { + for (const raw of custom) { + const n = normalizer(raw) + if (n && !seen.has(n) && !reserved?.has(n)) { + seen.add(n) + out.push(n) + } + } + } + return out +} + +/** + * 算出一次编译里编译器实际认的扩展名与标签——内置的加上这份 `fileTypes` 追加的。 + * 宿主用它做文件分类，就不会和编译器给出不同答案。 + * @param {FileTypes} [fileTypes] + * @returns {ResolvedFileTypes} + */ +export function resolveFileTypes(fileTypes = {}) { + const ft = fileTypes || {} + const reserved = new Set(RESERVED_EXTS) + const templateExts = mergeUnique(BUILTIN_TEMPLATE_EXTS, ft.template, normalizeExt, reserved) + return { + templateExts, + templateDirectivePrefixes: [...new Set([ + ...BUILTIN_TEMPLATE_DIRECTIVE_PREFIXES, + ...templateExts.map((extension) => { + const name = extension.slice(1) + return name.endsWith('ml') ? name.slice(0, -2) : name + }).filter(Boolean), + ])], + styleExts: mergeUnique(BUILTIN_STYLE_EXTS, ft.style, normalizeExt, reserved), + viewScriptExts: mergeUnique(BUILTIN_VIEW_SCRIPT_EXTS, ft.viewScript, normalizeExt, reserved), + viewScriptTags: mergeUnique(BUILTIN_VIEW_SCRIPT_TAGS, ft.viewScript, normalizeTag), + } +} + +/** + * 路径是不是这组扩展名之一。大小写不敏感，`resolveFileTypes` 的任一列表都能直接喂进来。 + * @param {string} filePath + * @param {readonly string[]} exts + * @returns {boolean} + */ +export function hasExt(filePath, exts) { + if (typeof filePath !== 'string') return false + const lower = filePath.toLowerCase() + return exts.some((ext) => lower.endsWith(ext)) +} diff --git a/packages/compiler/src/pool-node.js b/packages/compiler/src/pool-node.js index 75b205fe..cc9f63e3 100644 --- a/packages/compiler/src/pool-node.js +++ b/packages/compiler/src/pool-node.js @@ -65,7 +65,7 @@ const DEFAULT_IDLE_SHRINK_MS = 300000 let chain = Promise.resolve() /** - * @typedef {{ template?: string[], style?: string[], viewScript?: string[] }} FileTypes + * @typedef {{ template?: readonly string[], style?: readonly string[], viewScript?: readonly string[] }} FileTypes * @typedef {{ sourcemap?: boolean, fileTypes?: FileTypes }} BuildOptions * @typedef {{ appId: string, name: string, path: string }} BuildResult */ diff --git a/packages/compiler/src/pool.js b/packages/compiler/src/pool.js index 0d429894..e6668402 100644 --- a/packages/compiler/src/pool.js +++ b/packages/compiler/src/pool.js @@ -212,7 +212,7 @@ export function createCompilerPool(options) { * @param {{ * files: Record, * workPath?: string, - * options?: { fileTypes?: { template?: string[], style?: string[], viewScript?: string[] } }, + * options?: { fileTypes?: { template?: readonly string[], style?: readonly string[], viewScript?: readonly string[] } }, * } | Record} input * Source files are text or raw bytes: an image belongs in the map as a Uint8Array, * not as a decoded string (postMessage carries it as bytes, and the stage worker diff --git a/packages/compiler/tsconfig.types.json b/packages/compiler/tsconfig.types.json index 196790b2..d5238bb6 100644 --- a/packages/compiler/tsconfig.types.json +++ b/packages/compiler/tsconfig.types.json @@ -23,6 +23,7 @@ "src/pool-node.js", "src/stage-worker.js", "src/toolchain.js", - "src/browser-assets.js" + "src/browser-assets.js", + "src/file-types.js" ] } diff --git a/packages/compiler/types-fixture/consumer.ts b/packages/compiler/types-fixture/consumer.ts index 2b533c26..52d622ae 100644 --- a/packages/compiler/types-fixture/consumer.ts +++ b/packages/compiler/types-fixture/consumer.ts @@ -4,9 +4,10 @@ // degrades to `any`), the deliberately wrong call stops erroring and tsc fails // this file with "unused '@ts-expect-error' directive". -import { collectOutputs, STAGE_NAMES } from '@dimina-kit/compiler' +import { collectOutputs, compileMiniApp, STAGE_NAMES } from '@dimina-kit/compiler' import { COMPILER_BROWSER_ASSETS, resolveBrowserAssets } from '@dimina-kit/compiler/browser-assets' import { initToolchain } from '@dimina-kit/compiler/browser' +import { QD_FILE_TYPES, hasExt, resolveFileTypes } from '@dimina-kit/compiler/file-types' import { createCompilerPool } from '@dimina-kit/compiler/pool' import { createNodeCompilerPool } from '@dimina-kit/compiler/pool-node' import '@dimina-kit/compiler/stage-worker' @@ -23,6 +24,16 @@ void assetDir resolveBrowserAssets(COMPILER_BROWSER_ASSETS) const outputs: Record = collectOutputs({ fs: {}, targetPath: '/dist' }) + +const { templateExts } = resolveFileTypes(QD_FILE_TYPES) +const isTemplate: boolean = hasExt('pages/index/index.qdml', templateExts) +void isTemplate +// The published dialect is frozen at runtime, so the declarations have to say so — +// otherwise "just add one more extension" type-checks and throws in the browser. +// @ts-expect-error QD_FILE_TYPES' lists are readonly +QD_FILE_TYPES.template?.push('qdx') +// @ts-expect-error hasExt takes the extension list, not a single extension +hasExt('pages/index/index.qdml', '.qdml') void outputs // @ts-expect-error products are text OR bytes; a downstream must narrow before treating one as a string const textOnly: Record = collectOutputs({ fs: {}, targetPath: '/dist' }) @@ -70,5 +81,13 @@ export async function compileOnce(): Promise { const nodePool = createNodeCompilerPool({ stages: ['logic'] }) void nodePool + +// The published dialect must go into the compile entries exactly as the README shows +// it. Its lists are frozen, so an entry still asking for mutable `string[]` rejects +// it with TS2322 — a downstream would have to copy the arrays to get past its own +// type-check, which is how each host ends up with its own drifting copy again. +void pool.compile({ files: { 'app.json': '{}' }, workPath: '/project', options: { fileTypes: QD_FILE_TYPES } }) +void compileMiniApp({ fs: {}, workPath: '/project', options: { fileTypes: QD_FILE_TYPES } }) +void nodePool.build('/out', '/project', true, { fileTypes: QD_FILE_TYPES }) // @ts-expect-error stages is a list of stage names createNodeCompilerPool({ stages: 'logic' }) From ba266906fb54c444dcaafe00501698188b3eac29 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 19:37:27 +0800 Subject: [PATCH 12/14] =?UTF-8?q?chore(ci):=20=E6=A0=B8=E5=AF=B9=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E4=BA=A7=E7=89=A9=EF=BC=8C=E5=B9=B6=E5=81=9C=E6=AD=A2?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E6=B5=8B=E8=AF=95=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 两类问题本仓的构建和测试都看不见，只在包发布出去之后才暴露： exports/main/types/bin 指向的文件没进 tarball（`files` 少写一个目录 就会这样，workspace 里读源码目录所以一直是好的），以及 tarball 里混 进测试文件。 check-publish-contract.js 拿每个可发布包真实的 `npm pack --dry-run` 清单来核对这两件事，而不是读 `files` 字段猜。subpath pattern 只要求 至少命中一个文件。 跑下来 5 个包在发测试文件：devkit 72 个（编译后的 dist/*.test.js 及其 .d.ts/.map）、inspect 19 个、fs-core 18 个、view-anchor 7 个、 design 3 个。用 files 的 "!" 规则排掉，devkit 的 tarball 从 131 个 文件降到 59 个，`watch-rebuild.testutil.*` 这类不是测试的文件仍然 保留。 --- .github/scripts/check-publish-contract.js | 124 ++++++++++++++++++ .../scripts/check-publish-contract.test.js | 73 +++++++++++ .github/workflows/ci.yml | 6 + package.json | 1 + packages/design/package.json | 4 +- packages/devkit/package.json | 4 +- packages/fs-core/package.json | 4 +- packages/inspect/package.json | 4 +- packages/view-anchor/package.json | 4 +- 9 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/check-publish-contract.js create mode 100644 .github/scripts/check-publish-contract.test.js diff --git a/.github/scripts/check-publish-contract.js b/.github/scripts/check-publish-contract.js new file mode 100644 index 00000000..db02f0e8 --- /dev/null +++ b/.github/scripts/check-publish-contract.js @@ -0,0 +1,124 @@ +#!/usr/bin/env node + +// 两类问题只在包发布出去之后才暴露，本仓自己的构建和测试都看不见： +// +// 1. exports/main/types/bin 指向的文件没进 tarball（`files` 少写一个目录就会 +// 这样）。安装方一 import 就是 ERR_MODULE_NOT_FOUND，而本仓 workspace 里 +// 同一个 import 一直是好的——它读的是源码目录，不是 tarball。 +// 2. tarball 里混进测试文件。它们不是 API 的一部分，却出现在安装方的 +// node_modules 里，占体积、也让人以为可以 import。 +// +// 所以这里拿 `npm pack --dry-run` 的真实打包清单来核对，而不是读 `files` 字段 +// 猜。--ignore-scripts：只要清单，不重跑各包的 prepack 构建。 + +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { NPM_PACKAGES } from './npm-packages.js' + +// exports 的值可以是字符串，也可以嵌套条件对象（types/require/default…）， +// 两种都要收集到叶子上的相对路径。 +function collectTargets(node, out) { + if (typeof node === 'string') { + if (node.startsWith('./')) out.push(node) + return out + } + if (node && typeof node === 'object') { + for (const value of Object.values(node)) collectTargets(value, out) + } + return out +} + +const normalize = (target) => (target.startsWith('./') ? target : `./${target}`) + +/** + * 一个包声明的、安装方能直接解析到的所有文件路径。 + * + * @param {Record} pkgJson + * @returns {string[]} 形如 './dist/index.js'，去重 + */ +export function entryTargets(pkgJson) { + const targets = collectTargets(pkgJson.exports, []) + if (typeof pkgJson.main === 'string') targets.push(normalize(pkgJson.main)) + if (typeof pkgJson.types === 'string') targets.push(normalize(pkgJson.types)) + const bin = typeof pkgJson.bin === 'string' ? { [String(pkgJson.name)]: pkgJson.bin } : pkgJson.bin + for (const value of Object.values(bin || {})) { + if (typeof value === 'string') targets.push(normalize(value)) + } + return [...new Set(targets)] +} + +// subpath pattern（'./dist/shared/*.js'）匹配的是一组文件，只要求至少命中一个。 +function patternToRegExp(target) { + const literals = target.split('*').map((part) => part.replace(/[.+?^${}()|[\]\\]/g, '\\$&')) + return new RegExp(`^${literals.join('.+')}$`) +} + +const TEST_FILE = /(^|\/)(__tests__|__mocks__|fixtures|test-fixtures|types-fixture)\/|\.(test|spec)\.[^/]+$/ + +/** + * 核对一个包的 package.json 与它真实的打包清单。 + * + * @param {Record} pkgJson + * @param {string[]} packedPaths npm pack 报告的 tarball 内相对路径 + * @returns {string[]} 每行一个问题；契约成立时为空 + */ +export function checkPackedFiles(pkgJson, packedPaths) { + const packed = packedPaths.map(normalize) + const packedSet = new Set(packed) + const problems = [] + + for (const target of entryTargets(pkgJson)) { + if (target.includes('*')) { + const pattern = patternToRegExp(target) + if (!packed.some((file) => pattern.test(file))) { + problems.push(`${target} 是 exports 里的 subpath pattern，但 tarball 里没有任何文件匹配它`) + } + continue + } + if (!packedSet.has(target)) { + problems.push(`${target} 被 package.json 声明为入口，但不在 tarball 里（检查 files 字段）`) + } + } + + const tests = packedPaths.filter((file) => TEST_FILE.test(file)) + if (tests.length > 0) { + const shown = tests.slice(0, 5).join(', ') + problems.push(`tarball 里有 ${tests.length} 个测试文件，用 files 的 "!" 规则排除掉：${shown}${tests.length > 5 ? ' …' : ''}`) + } + + return problems +} + +function packedPathsOf(dir) { + // prepack 之类的脚本会往 stdout 写构建日志，混在 --json 前面。 + const raw = execFileSync('npm', ['pack', '--dry-run', '--json', '--ignore-scripts'], { + cwd: dir, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'inherit'], + maxBuffer: 256 * 1024 * 1024, + }) + const start = raw.indexOf('[') + if (start < 0) throw new Error(`npm pack --json 没有输出 JSON:\n${raw.slice(0, 500)}`) + return JSON.parse(raw.slice(start))[0].files.map((file) => file.path) +} + +if (import.meta.url === `file://${process.argv[1]}`) { + let failed = false + for (const { name, dir } of NPM_PACKAGES) { + const pkgJson = JSON.parse(readFileSync(join(process.cwd(), dir, 'package.json'), 'utf8')) + const problems = checkPackedFiles(pkgJson, packedPathsOf(dir)) + if (problems.length === 0) { + console.log(`✅ ${name}`) + continue + } + failed = true + console.error(`❌ ${name}`) + for (const problem of problems) console.error(` ${problem}`) + } + if (failed) { + console.error('\n发布产物契约不成立。上面每一条都会在安装方那里才炸，本仓的构建和测试看不见。') + process.exit(1) + } + console.log('\n发布产物契约成立：入口都在 tarball 里，没有发出测试文件。') +} diff --git a/.github/scripts/check-publish-contract.test.js b/.github/scripts/check-publish-contract.test.js new file mode 100644 index 00000000..2ec8c37f --- /dev/null +++ b/.github/scripts/check-publish-contract.test.js @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { checkPackedFiles, entryTargets } from './check-publish-contract.js' + +test('入口收集覆盖 exports 条件对象、main、types 与 bin', () => { + assert.deepEqual( + entryTargets({ + name: '@scope/pkg', + main: 'dist/index.js', + types: 'dist/index.d.ts', + bin: { pkg: './bin/cli.js' }, + exports: { + '.': { types: './dist/index.d.ts', default: './dist/index.js' }, + './sub': './dist/sub.js', + './package.json': './package.json', + }, + }), + ['./dist/index.d.ts', './dist/index.js', './dist/sub.js', './package.json', './bin/cli.js'], + ) +}) + +test('入口指向没被打包的文件时报出来', () => { + const problems = checkPackedFiles( + { exports: { '.': './dist/index.js', './sub': './dist/sub.js' } }, + ['dist/index.js', 'package.json'], + ) + assert.equal(problems.length, 1) + assert.match(problems[0], /\.\/dist\/sub\.js/) + assert.match(problems[0], /files/) +}) + +test('声明的入口都在 tarball 里就没有问题', () => { + assert.deepEqual( + checkPackedFiles( + { main: 'dist/index.js', exports: { '.': { types: './dist/index.d.ts', default: './dist/index.js' } } }, + ['dist/index.js', 'dist/index.d.ts'], + ), + [], + ) +}) + +test('subpath pattern 只要求至少命中一个文件', () => { + const exports = { './shared/*': './dist/shared/*.js' } + assert.deepEqual(checkPackedFiles({ exports }, ['dist/shared/bridge.js']), []) + + const problems = checkPackedFiles({ exports }, ['dist/index.js']) + assert.equal(problems.length, 1) + assert.match(problems[0], /subpath pattern/) +}) + +test('pattern 里的点号按字面匹配，不当通配符', () => { + const problems = checkPackedFiles({ exports: { './x/*': './dist/x/*.js' } }, ['dist/xAy-js']) + assert.equal(problems.length, 1) +}) + +test('打包出测试文件要报出来——源码目录和编译产物里的都算', () => { + const problems = checkPackedFiles({ exports: { '.': './dist/index.js' } }, [ + 'dist/index.js', + 'src/client.test.ts', + 'dist/compile-log.test.d.ts', + 'src/__tests__/helper.ts', + 'types-fixture/consumer.ts', + ]) + assert.equal(problems.length, 1) + assert.match(problems[0], /4 个测试文件/) +}) + +test('名字里带 test 但不是测试文件的正常发布', () => { + assert.deepEqual( + checkPackedFiles({ exports: { '.': './dist/index.js' } }, ['dist/index.js', 'dist/test-utils.js', 'src/latest.ts']), + [], + ) +}) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1790c3c4..317ce4af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,6 +174,12 @@ jobs: pnpm exec tsc -p tools/pawl node --test ".github/scripts/*.test.js" "tools/pawl/*.test.ts" + # Runs after the test job's builds so every package's dist exists: the + # check reads each package's real `npm pack` listing, and a missing dist + # would report entry points as unpublished. + - name: Check publish contract + run: node .github/scripts/check-publish-contract.js + # setup-pawl installs the binary and runs `check`; comments are disabled # because this workflow grants read-only repository permissions, so a # sticky PR comment would fail to upsert. The install leaves pawl on diff --git a/package.json b/package.json index 6553e966..6933820e 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "lint": "turbo run lint", "dev": "turbo run dev", "check:electron-peer-sync": "node scripts/check-electron-peer-sync.mjs", + "check:publish-contract": "node .github/scripts/check-publish-contract.js", "pawl:check": "pawl check", "pawl:record": "pawl record", "pawl:diff": "pawl diff" diff --git a/packages/design/package.json b/packages/design/package.json index 2725d1c0..924d571b 100644 --- a/packages/design/package.json +++ b/packages/design/package.json @@ -41,7 +41,9 @@ "css", "tailwind-preset.cjs", "README.md", - "LICENSE" + "LICENSE", + "!**/*.test.*", + "!**/*.spec.*" ], "scripts": { "build": "tsc -p tsconfig.build.json", diff --git a/packages/devkit/package.json b/packages/devkit/package.json index 584d5755..20f5f44a 100644 --- a/packages/devkit/package.json +++ b/packages/devkit/package.json @@ -49,7 +49,9 @@ "fe/dimina-fe-server/index.js", "fe/dimina-fe-server/THIRD_PARTY_NOTICES.md", "README.md", - "LICENSE" + "LICENSE", + "!**/*.test.*", + "!**/*.spec.*" ], "scripts": { "build": "tsc", diff --git a/packages/fs-core/package.json b/packages/fs-core/package.json index da33c700..307859a1 100644 --- a/packages/fs-core/package.json +++ b/packages/fs-core/package.json @@ -73,7 +73,9 @@ "files": [ "dist", "src", - "sync" + "sync", + "!**/*.test.*", + "!**/*.spec.*" ], "scripts": { "build": "tsc -p tsconfig.worker-lib.build.json && tsc -p tsconfig.build.json && tsc -p tsconfig.sync.build.json && node build-workers.js", diff --git a/packages/inspect/package.json b/packages/inspect/package.json index 6a59d609..3c9a22c1 100644 --- a/packages/inspect/package.json +++ b/packages/inspect/package.json @@ -37,7 +37,9 @@ }, "files": [ "dist", - "src" + "src", + "!**/*.test.*", + "!**/*.spec.*" ], "scripts": { "build": "tsc -p tsconfig.build.json", diff --git a/packages/view-anchor/package.json b/packages/view-anchor/package.json index ed02bded..0a78419d 100644 --- a/packages/view-anchor/package.json +++ b/packages/view-anchor/package.json @@ -35,7 +35,9 @@ "src", "docs", "README.md", - "LICENSE" + "LICENSE", + "!**/*.test.*", + "!**/*.spec.*" ], "scripts": { "build": "tsc -p tsconfig.build.json && pnpm run build:docs", From 9501fa570f5fe21a8d97c8bc88fbb503f71b3f07 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Wed, 2 Sep 2026 21:30:11 +0800 Subject: [PATCH 13/14] =?UTF-8?q?fix(ci):=20=E5=8F=91=E5=B8=83=E5=A5=91?= =?UTF-8?q?=E7=BA=A6=E6=A3=80=E6=9F=A5=E5=89=8D=E5=85=88=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E6=8C=89=E5=8F=91=E5=B8=83=E5=90=8E=E7=9A=84?= =?UTF-8?q?=E6=B8=85=E5=8D=95=E6=A0=B8=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI 里这一步是红的：它跑在 `turbo run test` 之后，而 test 只依赖 `^build`， 只构建被别人依赖的包。仓库里没人依赖 @dimina-kit/devtools，它的 dist 一直是 空的，检查就把它声明的 11 个入口全报成"没进 tarball"。这一步现在自己先跑一次 `turbo run build`（本地实测 1 分钟，绝大部分命中 turbo 缓存）。 同时修掉检查本身漏看的三处： - pnpm publish 会用 publishConfig 里的同名字段覆盖发出去的 package.json。 design 和 view-anchor 正是这么写的——源码里 main 指向 ./src/index.ts，发布出去 指向 ./dist/index.js。原来按源码字段核对，恰恰把这两个最需要检查的包放行了。 - imports、typings、typesVersions 指向的文件同样要在 tarball 里，之前没收集。 - test- 打头的脚本也算测试文件（packages/compiler 的 scripts/ 里有 21 个跟着 发出去了），除非它被声明成入口——故意当 API 发的测试辅助工具照旧放行。 另外把判断"是否被直接运行"的 `file://` 字符串拼接换成路径比较：Windows 的 `file:///C:/…` 和路径里的空格/中文都会让原来的写法对不上，检查会静默不跑。 --- .github/scripts/check-publish-contract.js | 60 ++++++++++++++++--- .../scripts/check-publish-contract.test.js | 60 ++++++++++++++++++- .github/workflows/ci.yml | 11 ++-- packages/compiler/package.json | 3 +- 4 files changed, 120 insertions(+), 14 deletions(-) diff --git a/.github/scripts/check-publish-contract.js b/.github/scripts/check-publish-contract.js index db02f0e8..f95aed0b 100644 --- a/.github/scripts/check-publish-contract.js +++ b/.github/scripts/check-publish-contract.js @@ -13,7 +13,8 @@ import { execFileSync } from 'node:child_process' import { readFileSync } from 'node:fs' -import { join } from 'node:path' +import { join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' import { NPM_PACKAGES } from './npm-packages.js' // exports 的值可以是字符串，也可以嵌套条件对象（types/require/default…）， @@ -31,16 +32,51 @@ function collectTargets(node, out) { const normalize = (target) => (target.startsWith('./') ? target : `./${target}`) +// typesVersions 的叶子是一个路径数组，写法上 './dist/x.d.ts' 和 'dist/x.d.ts' 都常见， +// 所以这里不像 exports 那样按 './' 前缀筛，而是把每个字符串都当路径。 +function collectTypesVersionTargets(node, out) { + if (typeof node === 'string') { + out.push(normalize(node)) + return out + } + if (node && typeof node === 'object') { + for (const value of Object.values(node)) collectTypesVersionTargets(value, out) + } + return out +} + +// pnpm publish 会用 publishConfig 里的同名字段覆盖发布出去的 package.json——design +// 和 view-anchor 就靠这个：源码里 main 指向 ./src/index.ts，发到 npm 上指向 +// ./dist/index.js。安装方看到的是覆盖之后的清单，所以核对也必须按覆盖后的来，否则 +// 恰恰是这两个最需要检查的包被按源码字段放行了。 +// （publishConfig 里也能覆盖 files，但本仓没有包这么做；真出现时打包清单来自 +// `npm pack`，它只认源码里的 files，这里会跟着错。） +const OVERLAID_FIELDS = ['main', 'types', 'typings', 'exports', 'bin', 'imports', 'typesVersions'] + +export function publishedManifest(pkgJson) { + const published = { ...pkgJson } + for (const field of OVERLAID_FIELDS) { + if (pkgJson.publishConfig && field in pkgJson.publishConfig) published[field] = pkgJson.publishConfig[field] + } + return published +} + /** * 一个包声明的、安装方能直接解析到的所有文件路径。 * - * @param {Record} pkgJson + * @param {Record} sourcePkgJson 仓库里的 package.json（publishConfig 覆盖在内部处理） * @returns {string[]} 形如 './dist/index.js'，去重 */ -export function entryTargets(pkgJson) { +export function entryTargets(sourcePkgJson) { + const pkgJson = publishedManifest(sourcePkgJson) const targets = collectTargets(pkgJson.exports, []) + // imports 里的 '#internal' 只有包自己 import 得到，但同样要求文件真的发出去； + // 值可能是外部包名（不带 './'），collectTargets 已经把这类滤掉了。 + collectTargets(pkgJson.imports, targets) if (typeof pkgJson.main === 'string') targets.push(normalize(pkgJson.main)) if (typeof pkgJson.types === 'string') targets.push(normalize(pkgJson.types)) + if (typeof pkgJson.typings === 'string') targets.push(normalize(pkgJson.typings)) + collectTypesVersionTargets(pkgJson.typesVersions, targets) const bin = typeof pkgJson.bin === 'string' ? { [String(pkgJson.name)]: pkgJson.bin } : pkgJson.bin for (const value of Object.values(bin || {})) { if (typeof value === 'string') targets.push(normalize(value)) @@ -54,7 +90,11 @@ function patternToRegExp(target) { return new RegExp(`^${literals.join('.+')}$`) } -const TEST_FILE = /(^|\/)(__tests__|__mocks__|fixtures|test-fixtures|types-fixture)\/|\.(test|spec)\.[^/]+$/ +// 三种写法都算测试文件：测试目录下的、`x.test.ts` / `x.spec.ts`、以及 `test-x.js` +// 这种以 test- 打头的脚本（packages/compiler 的 scripts/ 里有二十来个）。最后一种 +// 有例外——包可能故意把测试辅助工具当 API 发出去，所以下面对"被声明为入口"的文件 +// 放行。 +const TEST_FILE = /(^|\/)(__tests__|__mocks__|fixtures|test-fixtures|types-fixture)\/|(^|\/)test-[^/]*\.[cm]?[jt]sx?$|\.(test|spec)\.[^/]+$/ /** * 核对一个包的 package.json 与它真实的打包清单。 @@ -66,9 +106,11 @@ const TEST_FILE = /(^|\/)(__tests__|__mocks__|fixtures|test-fixtures|types-fixtu export function checkPackedFiles(pkgJson, packedPaths) { const packed = packedPaths.map(normalize) const packedSet = new Set(packed) + const targets = entryTargets(pkgJson) + const declared = new Set(targets) const problems = [] - for (const target of entryTargets(pkgJson)) { + for (const target of targets) { if (target.includes('*')) { const pattern = patternToRegExp(target) if (!packed.some((file) => pattern.test(file))) { @@ -81,7 +123,7 @@ export function checkPackedFiles(pkgJson, packedPaths) { } } - const tests = packedPaths.filter((file) => TEST_FILE.test(file)) + const tests = packedPaths.filter((file) => TEST_FILE.test(file) && !declared.has(normalize(file))) if (tests.length > 0) { const shown = tests.slice(0, 5).join(', ') problems.push(`tarball 里有 ${tests.length} 个测试文件，用 files 的 "!" 规则排除掉：${shown}${tests.length > 5 ? ' …' : ''}`) @@ -103,7 +145,11 @@ function packedPathsOf(dir) { return JSON.parse(raw.slice(start))[0].files.map((file) => file.path) } -if (import.meta.url === `file://${process.argv[1]}`) { +// 直接跑这个文件时才执行检查；被 test 文件 import 时不执行。字符串拼 `file://` +// 在 Windows（`file:///C:/…`）和路径里有空格/中文（URL 转义）时都对不上，所以两边 +// 都换算成本地路径再比。 +const entryPath = process.argv[1] ? resolve(process.argv[1]) : '' +if (fileURLToPath(import.meta.url) === entryPath) { let failed = false for (const { name, dir } of NPM_PACKAGES) { const pkgJson = JSON.parse(readFileSync(join(process.cwd(), dir, 'package.json'), 'utf8')) diff --git a/.github/scripts/check-publish-contract.test.js b/.github/scripts/check-publish-contract.test.js index 2ec8c37f..7f1df7fb 100644 --- a/.github/scripts/check-publish-contract.test.js +++ b/.github/scripts/check-publish-contract.test.js @@ -65,9 +65,65 @@ test('打包出测试文件要报出来——源码目录和编译产物里的 assert.match(problems[0], /4 个测试文件/) }) -test('名字里带 test 但不是测试文件的正常发布', () => { +test('test- 打头的脚本算测试文件，被声明成入口的除外', () => { + const problems = checkPackedFiles({ exports: { '.': './dist/index.js' } }, [ + 'dist/index.js', + 'scripts/test-node.js', + 'scripts/build-compiler.js', + ]) + assert.equal(problems.length, 1) + assert.match(problems[0], /1 个测试文件/) + assert.match(problems[0], /scripts\/test-node\.js/) + + // 故意当 API 发出去的测试辅助工具（声明在 exports 里）不算。 + assert.deepEqual( + checkPackedFiles( + { exports: { '.': './dist/index.js', './test-utils': './dist/test-utils.js' } }, + ['dist/index.js', 'dist/test-utils.js'], + ), + [], + ) +}) + +test('名字里只是含 test 的文件正常发布', () => { assert.deepEqual( - checkPackedFiles({ exports: { '.': './dist/index.js' } }, ['dist/index.js', 'dist/test-utils.js', 'src/latest.ts']), + checkPackedFiles({ exports: { '.': './dist/index.js' } }, ['dist/index.js', 'dist/latest.js', 'src/attest.ts']), [], ) }) + +test('按 publishConfig 覆盖后的清单核对入口', () => { + // design/view-anchor 的真实形状：源码里 main 指向 ./src/index.ts，发布出去指向 + // ./dist/index.js。只有按发布后的字段核对，才会发现 dist 没进 tarball。 + const pkgJson = { + main: './src/index.ts', + types: './src/index.ts', + exports: { '.': './src/index.ts' }, + publishConfig: { + access: 'public', + main: './dist/index.js', + types: './dist/index.d.ts', + exports: { '.': { types: './dist/index.d.ts', default: './dist/index.js' } }, + }, + } + assert.deepEqual(entryTargets(pkgJson), ['./dist/index.d.ts', './dist/index.js']) + + const problems = checkPackedFiles(pkgJson, ['src/index.ts', 'package.json']) + assert.equal(problems.length, 2) + assert.match(problems.join('\n'), /\.\/dist\/index\.js/) +}) + +test('imports、typings 和 typesVersions 指向的文件也要在 tarball 里', () => { + assert.deepEqual( + entryTargets({ + typings: 'dist/index.d.ts', + imports: { '#internal': './dist/internal.js', '#dep': 'some-package' }, + typesVersions: { '*': { 'sub': ['dist/sub.d.ts'] } }, + }), + ['./dist/internal.js', './dist/index.d.ts', './dist/sub.d.ts'], + ) + + const problems = checkPackedFiles({ imports: { '#internal': './dist/internal.js' } }, ['dist/index.js']) + assert.equal(problems.length, 1) + assert.match(problems[0], /\.\/dist\/internal\.js/) +}) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 317ce4af..7dff7201 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,11 +174,14 @@ jobs: pnpm exec tsc -p tools/pawl node --test ".github/scripts/*.test.js" "tools/pawl/*.test.ts" - # Runs after the test job's builds so every package's dist exists: the - # check reads each package's real `npm pack` listing, and a missing dist - # would report entry points as unpublished. + # `npm pack` 只看磁盘上现在有什么，所以这一步得自己保证每个被检查的包都构建 + # 过。上面的 `turbo run test` 不够：test 只依赖 `^build`，也就是只构建被别人 + # 依赖的包。仓库里没有任何包依赖 @dimina-kit/devtools，它的 dist 于是始终是 + # 空的，检查会把它声明的每一个入口都报成"没进 tarball"。 - name: Check publish contract - run: node .github/scripts/check-publish-contract.js + run: | + pnpm turbo run build --cache-dir=.turbo + node .github/scripts/check-publish-contract.js # setup-pawl installs the binary and runs `check`; comments are disabled # because this workflow grants read-only repository permissions, so a diff --git a/packages/compiler/package.json b/packages/compiler/package.json index c0f4485a..41e9fab6 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -57,7 +57,8 @@ "files": [ "dist", "src", - "scripts" + "scripts", + "!scripts/test-*.js" ], "scripts": { "check:wasm-alignment": "node scripts/check-wasm-alignment.js", From d09b5c4540cd32908e56f8f73220c6f27d695ed4 Mon Sep 17 00:00:00 2001 From: lbb00 Date: Thu, 3 Sep 2026 00:19:45 +0800 Subject: [PATCH 14/14] =?UTF-8?q?fix(ci):=20=E5=8D=95=E7=8B=AC=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=20devtools=20=E7=9A=84=E5=8F=91=E5=B8=83=E5=85=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E5=88=AB=E8=AE=A9=E5=A5=91=E7=BA=A6=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E5=9C=A8=20CI=20=E4=B8=8A=E5=BF=85=E6=8C=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI 的 test-and-pawl job 装依赖只装了仓库根。@dimina-kit/devtools 的整包 build 第一步 是 build:container，要构建 dimina/fe 里的容器，而 dimina/fe 是另一个独立的 pnpm workspace，没装依赖，构建直接报 Cannot find package '@vitejs/plugin-vue'——这一步于是 在每个 PR 上都挂。 devtools 声明出去的 22 个入口全部出自 build:main 和 build:preload，都不碰 dimina/fe。 所以把 devtools 从 turbo 那一轮排除，单独跑这两步；容器和 native-host 不在入口里， 少了它们不影响这个检查要核对的东西。每个包仍然是按真实打包清单严格核对，没有降级。 顺带补上三处漏检： - browser 字段（字符串形式和替换表）也是入口，之前完全没看。 - test/、tests/、__snapshots__/、fixture/ 目录下的普通文件也算测试文件；之前只认 __tests__ 和 .test./.spec.，`test/helper.js` 会被发出去。 - publishConfig 里的 files 和 directory 只有 pnpm 认、`npm pack` 不认，出现了就明确 报错，而不是给出一个悄悄失真的结论。 入口文件不在磁盘上时单独报"没构建"，不再刷一串"没进 tarball" 把原因埋掉。 Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/check-publish-contract.js | 59 +++++++++++++++-- .../scripts/check-publish-contract.test.js | 66 ++++++++++++++++++- .github/workflows/ci.yml | 11 +++- 3 files changed, 128 insertions(+), 8 deletions(-) diff --git a/.github/scripts/check-publish-contract.js b/.github/scripts/check-publish-contract.js index f95aed0b..fb79b596 100644 --- a/.github/scripts/check-publish-contract.js +++ b/.github/scripts/check-publish-contract.js @@ -12,7 +12,7 @@ // 猜。--ignore-scripts：只要清单，不重跑各包的 prepack 构建。 import { execFileSync } from 'node:child_process' -import { readFileSync } from 'node:fs' +import { existsSync, readFileSync } from 'node:fs' import { join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { NPM_PACKAGES } from './npm-packages.js' @@ -49,9 +49,12 @@ function collectTypesVersionTargets(node, out) { // 和 view-anchor 就靠这个：源码里 main 指向 ./src/index.ts，发到 npm 上指向 // ./dist/index.js。安装方看到的是覆盖之后的清单，所以核对也必须按覆盖后的来，否则 // 恰恰是这两个最需要检查的包被按源码字段放行了。 -// （publishConfig 里也能覆盖 files，但本仓没有包这么做；真出现时打包清单来自 -// `npm pack`，它只认源码里的 files，这里会跟着错。） -const OVERLAID_FIELDS = ['main', 'types', 'typings', 'exports', 'bin', 'imports', 'typesVersions'] +const OVERLAID_FIELDS = ['main', 'browser', 'types', 'typings', 'exports', 'bin', 'imports', 'typesVersions'] + +// publishConfig 还能覆盖 files、改发布根目录（directory）。这两个 pnpm 认、`npm pack` +// 不认，打包清单会跟真正发出去的 tarball 对不上——而这个检查全靠打包清单。本仓现在 +// 没有包这么写；真有人加了，宁可在这里明确报错，也不要给出一个悄悄失真的结论。 +const UNMODELLED_PUBLISH_CONFIG = ['files', 'directory'] export function publishedManifest(pkgJson) { const published = { ...pkgJson } @@ -61,6 +64,15 @@ export function publishedManifest(pkgJson) { return published } +/** + * 这个包的 publishConfig 里有没有本脚本模型不了的字段。 + * @param {Record} pkgJson + * @returns {string[]} + */ +export function unmodelledPublishConfig(pkgJson) { + return UNMODELLED_PUBLISH_CONFIG.filter((field) => pkgJson.publishConfig && field in pkgJson.publishConfig) +} + /** * 一个包声明的、安装方能直接解析到的所有文件路径。 * @@ -74,6 +86,10 @@ export function entryTargets(sourcePkgJson) { // 值可能是外部包名（不带 './'），collectTargets 已经把这类滤掉了。 collectTargets(pkgJson.imports, targets) if (typeof pkgJson.main === 'string') targets.push(normalize(pkgJson.main)) + // browser 的字符串形式是入口；对象形式是"把 A 换成 B"的替换表，只有本地相对路径的 + // 那一侧需要真的发出去，值写成 false（禁用某个模块）或包名的都不是本包的文件。 + if (typeof pkgJson.browser === 'string') targets.push(normalize(pkgJson.browser)) + else if (pkgJson.browser && typeof pkgJson.browser === 'object') collectTargets(pkgJson.browser, targets) if (typeof pkgJson.types === 'string') targets.push(normalize(pkgJson.types)) if (typeof pkgJson.typings === 'string') targets.push(normalize(pkgJson.typings)) collectTypesVersionTargets(pkgJson.typesVersions, targets) @@ -94,7 +110,7 @@ function patternToRegExp(target) { // 这种以 test- 打头的脚本（packages/compiler 的 scripts/ 里有二十来个）。最后一种 // 有例外——包可能故意把测试辅助工具当 API 发出去，所以下面对"被声明为入口"的文件 // 放行。 -const TEST_FILE = /(^|\/)(__tests__|__mocks__|fixtures|test-fixtures|types-fixture)\/|(^|\/)test-[^/]*\.[cm]?[jt]sx?$|\.(test|spec)\.[^/]+$/ +const TEST_FILE = /(^|\/)(test|tests|__tests__|__mocks__|__snapshots__|fixture|fixtures|test-fixtures|types-fixture)\/|(^|\/)test-[^/]*\.[cm]?[jt]sx?$|\.(test|spec)\.[^/]+$/ /** * 核对一个包的 package.json 与它真实的打包清单。 @@ -132,6 +148,22 @@ export function checkPackedFiles(pkgJson, packedPaths) { return problems } +/** + * 声明为入口、但磁盘上找不到的文件。空数组就是"构建产物齐了"，可以按 tarball 核对； + * 非空说明这个包没构建（或构建没产出全），此时 tarball 里当然什么都没有，直接按 tarball + * 报会刷出一长串"没进 tarball"，把"你忘了构建"埋在噪音里。subpath pattern 匹配的是一组 + * 文件，不在这里判断。 + * + * @param {Record} pkgJson + * @param {string} pkgDir 包目录的绝对路径 + * @returns {string[]} + */ +export function missingEntryFiles(pkgJson, pkgDir) { + return entryTargets(pkgJson) + .filter((target) => !target.includes('*')) + .filter((target) => !existsSync(join(pkgDir, target.replace(/^\.\//, '')))) +} + function packedPathsOf(dir) { // prepack 之类的脚本会往 stdout 写构建日志，混在 --json 前面。 const raw = execFileSync('npm', ['pack', '--dry-run', '--json', '--ignore-scripts'], { @@ -152,7 +184,22 @@ const entryPath = process.argv[1] ? resolve(process.argv[1]) : '' if (fileURLToPath(import.meta.url) === entryPath) { let failed = false for (const { name, dir } of NPM_PACKAGES) { - const pkgJson = JSON.parse(readFileSync(join(process.cwd(), dir, 'package.json'), 'utf8')) + const abs = join(process.cwd(), dir) + const pkgJson = JSON.parse(readFileSync(join(abs, 'package.json'), 'utf8')) + const unmodelled = unmodelledPublishConfig(pkgJson) + if (unmodelled.length > 0) { + failed = true + console.error(`❌ ${name}`) + console.error(` publishConfig 里的 ${unmodelled.join('、')} 只有 pnpm 认，\`npm pack\` 不认，本检查的打包清单会跟真正发出去的 tarball 对不上。要么别这么写，要么先把本脚本改成读 pnpm 打出来的真 tarball。`) + continue + } + const missing = missingEntryFiles(pkgJson, abs) + if (missing.length > 0) { + failed = true + console.error(`❌ ${name}`) + console.error(` 这些入口文件不在磁盘上，说明这个包没构建或构建产物不全，先构建再跑本检查：${missing.join(', ')}`) + continue + } const problems = checkPackedFiles(pkgJson, packedPathsOf(dir)) if (problems.length === 0) { console.log(`✅ ${name}`) diff --git a/.github/scripts/check-publish-contract.test.js b/.github/scripts/check-publish-contract.test.js index 7f1df7fb..f5fdcff3 100644 --- a/.github/scripts/check-publish-contract.test.js +++ b/.github/scripts/check-publish-contract.test.js @@ -1,6 +1,9 @@ import assert from 'node:assert/strict' +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' import { test } from 'node:test' -import { checkPackedFiles, entryTargets } from './check-publish-contract.js' +import { checkPackedFiles, entryTargets, missingEntryFiles, unmodelledPublishConfig } from './check-publish-contract.js' test('入口收集覆盖 exports 条件对象、main、types 与 bin', () => { assert.deepEqual( @@ -127,3 +130,64 @@ test('imports、typings 和 typesVersions 指向的文件也要在 tarball 里', assert.equal(problems.length, 1) assert.match(problems[0], /\.\/dist\/internal\.js/) }) + +test('browser 入口也要在 tarball 里', () => { + // 字符串形式就是一个入口 + assert.deepEqual(entryTargets({ browser: './dist/browser.js' }), ['./dist/browser.js']) + + // 对象形式是替换表：只有指向本包文件的那一侧要发出去，false（禁用某个模块）和包名不是 + assert.deepEqual( + entryTargets({ browser: { './dist/node.js': './dist/browser.js', 'node:fs': false, path: 'path-browserify' } }), + ['./dist/browser.js'], + ) + + // publishConfig 同样能覆盖它 + assert.deepEqual( + entryTargets({ browser: './src/browser.ts', publishConfig: { browser: './dist/browser.js' } }), + ['./dist/browser.js'], + ) + + const problems = checkPackedFiles({ browser: './dist/browser.js' }, ['dist/index.js']) + assert.equal(problems.length, 1) + assert.match(problems[0], /\.\/dist\/browser\.js/) +}) + +test('test 和 tests 目录下的普通文件也算测试文件', () => { + const problems = checkPackedFiles({}, ['package.json', 'test/helper.js', 'tests/integration.js', 'dist/__snapshots__/a.snap']) + assert.equal(problems.length, 1) + assert.match(problems[0], /3 个测试文件/) + + // 名字里含 test 的正常源文件不受影响 + assert.deepEqual(checkPackedFiles({}, ['dist/latest.js', 'dist/contest/index.js']), []) +}) + +test('publishConfig 里出现 npm pack 不认的字段时明确报错', () => { + assert.deepEqual(unmodelledPublishConfig({ publishConfig: { access: 'public' } }), []) + assert.deepEqual(unmodelledPublishConfig({ publishConfig: { files: ['dist'] } }), ['files']) + assert.deepEqual(unmodelledPublishConfig({ publishConfig: { files: ['dist'], directory: 'dist' } }), ['files', 'directory']) +}) + +test('入口文件不在磁盘上时，报的是"没构建"而不是一串没进 tarball', () => { + const dir = mkdtempSync(join(tmpdir(), 'publish-contract-')) + const touch = (rel) => { + mkdirSync(join(dir, dirname(rel)), { recursive: true }) + writeFileSync(join(dir, rel), '') + } + const pkgJson = { + exports: { '.': './dist/index.js', './sub/*': './dist/sub/*.js' }, + types: 'dist/index.d.ts', + bin: { demo: './bin/cli.js' }, + } + + assert.deepEqual(missingEntryFiles(pkgJson, dir), ['./dist/index.js', './dist/index.d.ts', './bin/cli.js']) + + // 只产出了一部分仍然算没齐——`files` 少写目录不会让文件从磁盘上消失，所以这里剩下的 + // 就是构建自己的问题，不该被当成构建齐了去跟 tarball 比。 + touch('dist/index.js') + assert.deepEqual(missingEntryFiles(pkgJson, dir), ['./dist/index.d.ts', './bin/cli.js']) + + // subpath pattern 匹配一组文件，不参与这里的存在性判断 + touch('dist/index.d.ts') + touch('bin/cli.js') + assert.deepEqual(missingEntryFiles(pkgJson, dir), []) +}) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dff7201..08c657b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,9 +178,18 @@ jobs: # 过。上面的 `turbo run test` 不够：test 只依赖 `^build`，也就是只构建被别人 # 依赖的包。仓库里没有任何包依赖 @dimina-kit/devtools，它的 dist 于是始终是 # 空的，检查会把它声明的每一个入口都报成"没进 tarball"。 + # + # devtools 的整包 build 在这里跑不起来：第一步 build:container 要构建 + # dimina/fe 里的容器，而 dimina/fe 是另一个独立的 pnpm workspace，本 job 只装了 + # 仓库根，构建会直接报 `Cannot find package '@vitejs/plugin-vue'`。所以把它从 + # turbo 那一轮排除，单独跑 build:main 和 build:preload——devtools 声明出去的 + # 22 个入口全部出自这两步，都不碰 dimina/fe。容器和 native-host 不在入口里， + # 少了它们不影响这个检查要核对的东西。 - name: Check publish contract run: | - pnpm turbo run build --cache-dir=.turbo + pnpm turbo run build --filter='!@dimina-kit/devtools' --cache-dir=.turbo + pnpm --filter @dimina-kit/devtools run build:main + pnpm --filter @dimina-kit/devtools run build:preload node .github/scripts/check-publish-contract.js # setup-pawl installs the binary and runs `check`; comments are disabled