diff --git a/CHANGELOG.md b/CHANGELOG.md index a3ca0f8b51..ccb6f6175b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,15 @@ All notable changes to this project will be documented in this file. ### Fixed +- **@overeng/genie**: stage bare workspace-package imports in compiled-binary + mode. When genie runs as a `bun --compile` binary it stages the `.genie.ts` + import graph into `os.tmpdir()`, which stripped `node_modules` reachability so + bare `@overeng/*` / `effect` imports failed (the bundled copies are not + visible to externally-loaded staged files). The staging step now symlinks the + importer's real `node_modules` into the staged root, letting design-time + generators reference real workspace-package types/data instead of hand-copied + relative mirrors. Cold bootstrap-phase generators (no `node_modules`) are + unaffected. (#1316) - **devenv / ts:emit**: resolve project references with filesystem directory checks so dotted directory names map to `tsconfig.json`, and treat an all-`noEmit` reference graph as successful no-work instead of invoking the diff --git a/packages/@overeng/genie/src/core/generation.ts b/packages/@overeng/genie/src/core/generation.ts index 3d42fff64f..e550207a06 100644 --- a/packages/@overeng/genie/src/core/generation.ts +++ b/packages/@overeng/genie/src/core/generation.ts @@ -83,6 +83,33 @@ const resolveRelativeImportPath = async ({ return resolvedCandidates.find((candidate) => candidate !== undefined) } +/** + * Find the nearest `node_modules` directory by walking up from `fromPath`. + * + * Returns undefined when none exists — e.g. a cold bootstrap checkout before install. In that case + * the staged graph gets no `node_modules` symlink and bare imports stay unresolvable, which is + * correct: `bootstrap`-phase generators are statically guaranteed (see {@link checkBootstrapClosure}) + * never to reach a bare package, so only design-time generators (run post-install, node_modules + * present) rely on the symlink below. + */ +const findNearestNodeModules = async (fromPath: string): Promise => { + let dir = path.dirname(fromPath) + const { root } = path.parse(dir) + for (;;) { + const candidate = path.join(dir, 'node_modules') + try { + // Sequential walk-up: each level's check depends on the previous miss, so it cannot be parallelized. + // oxlint-disable-next-line eslint/no-await-in-loop -- inherent to walking up the directory tree + const stat = await nodeFs.stat(candidate) + if (stat.isDirectory() === true) return candidate + } catch { + // No node_modules at this level — keep walking up. + } + if (dir === root) return undefined + dir = path.dirname(dir) + } +} + const collectRelativeImportPaths = async ({ sourceCode, sourcePath, @@ -112,7 +139,13 @@ const collectRelativeImportPaths = async ({ ) } -const stageCompiledBinaryImportGraph = ({ +/** + * Copy a genie file's relative/`#` import closure into a fresh `os.tmpdir()` staging directory and + * return the staged entry path (used by the compiled-binary import path, which cannot register the + * Bun import-map plugin). The importer's real `node_modules` is symlinked into the staged root so + * bare workspace-package / runtime imports still resolve. Exported for {@link stageCompiledBinaryImportGraph} tests. + */ +export const stageCompiledBinaryImportGraph = ({ entryPath, }: { entryPath: string @@ -128,6 +161,37 @@ const stageCompiledBinaryImportGraph = ({ }), }) + // Bare specifiers (`effect`, `@overeng/otel-contract`, `@effect/platform`, …) survive staging + // unchanged — `resolveImportMapsInSource` only rewrites `#`/`#mr`/relative specifiers. Staged + // modules are read from `os.tmpdir()`, outside the repo, so those bare imports would have no + // reachable `node_modules` and fail (in a compiled binary the bundled copies are not visible to + // externally-loaded files). Symlinking the importer's real `node_modules` into the staged root + // lets Bun resolve every bare import against the real on-disk install — exactly as a non-compiled + // `bun` run does when it imports the genie file in place. The bare-imported package's own + // transitive closure (its `effect`, its relative files) resolves from that package's real + // location; only the entry and its relative/`#` closure are ever copied, so a bare-imported + // package is loaded exactly once (avoids duplicate-singleton hazards, see mk-pnpm-cli.nix). + const nearestNodeModules = yield* Effect.tryPromise({ + try: () => findNearestNodeModules(entryPath), + catch: (error) => + new GenieImportError({ + genieFilePath: entryPath, + message: `Failed to locate node_modules while staging compiled-binary imports for ${entryPath}: ${safeErrorString(error)}`, + cause: error, + }), + }) + if (nearestNodeModules !== undefined) { + yield* Effect.tryPromise({ + try: () => nodeFs.symlink(nearestNodeModules, path.join(tempRoot, 'node_modules'), 'dir'), + catch: (error) => + new GenieImportError({ + genieFilePath: entryPath, + message: `Failed to link node_modules into compiled-binary staging directory for ${entryPath}: ${safeErrorString(error)}`, + cause: error, + }), + }) + } + const stagedPaths = new Map() const relativeEntryPath = entryPath.replace(/^(?:[A-Za-z]:)?[\\/]+/, '') @@ -205,7 +269,8 @@ const stageCompiledBinaryImportGraph = ({ return { stagePath, tempRoot } }) -const removeStagedCompiledBinaryImportGraph = ({ +/** Recursively remove a staged import graph's temp root (unlinks the `node_modules` symlink without following it). */ +export const removeStagedCompiledBinaryImportGraph = ({ tempRoot, }: StagedCompiledBinaryImportGraph): Effect.Effect => Effect.sync(() => nodeFsSync.rmSync(tempRoot, { recursive: true, force: true })).pipe( diff --git a/packages/@overeng/genie/src/core/staging-compiled-binary.unit.test.ts b/packages/@overeng/genie/src/core/staging-compiled-binary.unit.test.ts new file mode 100644 index 0000000000..b63540bc1d --- /dev/null +++ b/packages/@overeng/genie/src/core/staging-compiled-binary.unit.test.ts @@ -0,0 +1,162 @@ +import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { pathToFileURL } from 'node:url' + +import { NodeContext } from '@effect/platform-node' +import { Effect } from 'effect' +import { afterEach, expect } from 'vitest' + +import { Vitest } from '@overeng/utils-dev/node-vitest' + +import { + removeStagedCompiledBinaryImportGraph, + stageCompiledBinaryImportGraph, +} from './generation.ts' + +/** + * Compiled-binary staging must let a `.genie.ts` reach a bare workspace-package subpath + * (`@scope/contract/registry`) whose own closure reaches a bare runtime package (`effect`). The fix + * symlinks the importer's real `node_modules` into the staged tmp root so those bare imports resolve + * against the real on-disk install — the bundled copies in a `bun --compile` binary are not visible + * to externally-loaded staged files, and staging to `os.tmpdir()` otherwise strips node_modules reach. + * + * These fixtures are built with `node:fs` (no install state) so the test is hermetic and does not + * depend on effect-utils' own live node_modules. + */ + +const TestLayer = NodeContext.layer + +const createdDirs: string[] = [] + +/** Symlink-resolved temp dir so staged mirror paths match realpath'd on-disk names. */ +const makeDir = (prefix: string): string => { + const dir = realpathSync(mkdtempSync(path.join(os.tmpdir(), prefix))) + createdDirs.push(dir) + return dir +} + +const write = (dir: string, relativePath: string, content: string): string => { + const filePath = path.join(dir, relativePath) + mkdirSync(path.dirname(filePath), { recursive: true }) + writeFileSync(filePath, content, 'utf8') + return filePath +} + +/** + * Build a hermetic repo fixture: + * node_modules/effect — a bare runtime package (the transitive edge) + * node_modules/@scope/contract — a bare workspace package exposing a `./registry` SUBPATH that + * itself imports `effect` + * packages/foo/.genie.ts — imports the bare subpath and re-exports it as a GenieOutput + */ +const makeRepoFixture = ({ importSpecifier }: { importSpecifier: string }): string => { + const repoRoot = makeDir('genie-staging-repo-') + + write( + repoRoot, + 'node_modules/effect/package.json', + JSON.stringify({ name: 'effect', type: 'module', exports: { '.': './index.js' } }), + ) + write( + repoRoot, + 'node_modules/effect/index.js', + `export const Effect = { succeed: (value) => value }\n`, + ) + + write( + repoRoot, + 'node_modules/@scope/contract/package.json', + JSON.stringify({ + name: '@scope/contract', + type: 'module', + exports: { './registry': './registry.ts' }, + }), + ) + write( + repoRoot, + 'node_modules/@scope/contract/registry.ts', + `import { Effect } from 'effect'\nexport const registry = { kind: Effect.succeed('semconv'), count: 2 }\n`, + ) + + write( + repoRoot, + 'packages/foo/data.json.genie.ts', + [ + `import { registry } from '${importSpecifier}'`, + `export default { data: registry, stringify: () => JSON.stringify(registry) }`, + ].join('\n'), + ) + + return path.join(repoRoot, 'packages/foo/data.json.genie.ts') +} + +/** Stage the genie graph as the compiled-binary path does, dynamically import the staged entry, and return its default export. */ +const stageAndImport = (entryPath: string) => + Effect.gen(function* () { + const staged = yield* stageCompiledBinaryImportGraph({ entryPath }) + const importUrl = `${pathToFileURL(staged.stagePath).href}?import=${createdDirs.length}` + const module = yield* Effect.tryPromise( + // oxlint-disable-next-line eslint-plugin-import/no-dynamic-require -- staged path is dynamic by design + () => import(importUrl) as Promise<{ default: { data: unknown } }>, + ).pipe(Effect.ensuring(removeStagedCompiledBinaryImportGraph(staged))) + return { module, tempRoot: staged.tempRoot } + }) + +afterEach(() => { + for (const dir of createdDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }) + } +}) + +Vitest.describe('stageCompiledBinaryImportGraph', () => { + Vitest.it.effect( + 'resolves a bare workspace-package subpath (and its transitive `effect`) via the staged node_modules symlink', + () => + Effect.gen(function* () { + const entryPath = makeRepoFixture({ importSpecifier: '@scope/contract/registry' }) + const { module } = yield* stageAndImport(entryPath) + // The registry data (built from the bare `effect` edge) flows through generation. + expect(module.default.data).toEqual({ kind: 'semconv', count: 2 }) + }).pipe(Effect.provide(TestLayer)), + ) + + Vitest.it.effect( + 'does NOT delete the real node_modules when the staged graph is cleaned up', + () => + Effect.gen(function* () { + const entryPath = makeRepoFixture({ importSpecifier: '@scope/contract/registry' }) + const realNodeModules = path.join(path.dirname(entryPath), '../../node_modules') + const sentinel = path.join(realNodeModules, 'effect/index.js') + yield* stageAndImport(entryPath) // stages, imports, then removes the temp root + expect(existsSync(realNodeModules)).toBe(true) + expect(existsSync(sentinel)).toBe(true) + }).pipe(Effect.provide(TestLayer)), + ) + + Vitest.it.effect( + 'still stages a pure-relative graph when no node_modules exists (cold bootstrap unaffected)', + () => + Effect.gen(function* () { + const repoRoot = makeDir('genie-staging-cold-') + write(repoRoot, 'packages/foo/helper.ts', `export const value = { ok: true }\n`) + const entryPath = write( + repoRoot, + 'packages/foo/data.json.genie.ts', + [ + `import { value } from './helper.ts'`, + `export default { data: value, stringify: () => JSON.stringify(value) }`, + ].join('\n'), + ) + // No node_modules anywhere up the tree — staging must still succeed and not create a symlink. + const staged = yield* stageCompiledBinaryImportGraph({ entryPath }) + expect(existsSync(path.join(staged.tempRoot, 'node_modules'))).toBe(false) + const importUrl = `${pathToFileURL(staged.stagePath).href}?import=cold` + const module = yield* Effect.tryPromise( + // oxlint-disable-next-line eslint-plugin-import/no-dynamic-require -- staged path is dynamic by design + () => import(importUrl) as Promise<{ default: { data: unknown } }>, + ).pipe(Effect.ensuring(removeStagedCompiledBinaryImportGraph(staged))) + expect(module.default.data).toEqual({ ok: true }) + }).pipe(Effect.provide(TestLayer)), + ) +})