feat(genie): stage bare workspace-package imports in compiled-binary mode (#1316)

CHANGELOG.md

@@ -24,6 +24,15 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+- **@overeng/genie**: stage bare workspace-package imports in compiled-binary
+  mode. When genie runs as a `bun --compile` binary it stages the `.genie.ts`
+  import graph into `os.tmpdir()`, which stripped `node_modules` reachability so
+  bare `@overeng/*` / `effect` imports failed (the bundled copies are not
+  visible to externally-loaded staged files). The staging step now symlinks the
+  importer's real `node_modules` into the staged root, letting design-time
+  generators reference real workspace-package types/data instead of hand-copied
+  relative mirrors. Cold bootstrap-phase generators (no `node_modules`) are
+  unaffected. (#1316)
 - **devenv / ts:emit**: resolve project references with filesystem directory
   checks so dotted directory names map to `tsconfig.json`, and treat an
   all-`noEmit` reference graph as successful no-work instead of invoking the

packages/@overeng/genie/src/core/generation.ts

@@ -83,6 +83,33 @@ const resolveRelativeImportPath = async ({
   return resolvedCandidates.find((candidate) => candidate !== undefined)
 }
 
+/**
+ * Find the nearest `node_modules` directory by walking up from `fromPath`.
+ *
+ * Returns undefined when none exists — e.g. a cold bootstrap checkout before install. In that case
+ * the staged graph gets no `node_modules` symlink and bare imports stay unresolvable, which is
+ * correct: `bootstrap`-phase generators are statically guaranteed (see {@link checkBootstrapClosure})
+ * never to reach a bare package, so only design-time generators (run post-install, node_modules
+ * present) rely on the symlink below.
+ */
+const findNearestNodeModules = async (fromPath: string): Promise<string | undefined> => {
+  let dir = path.dirname(fromPath)
+  const { root } = path.parse(dir)
+  for (;;) {
+    const candidate = path.join(dir, 'node_modules')
+    try {
+      // Sequential walk-up: each level's check depends on the previous miss, so it cannot be parallelized.
+      // oxlint-disable-next-line eslint/no-await-in-loop -- inherent to walking up the directory tree
+      const stat = await nodeFs.stat(candidate)
+      if (stat.isDirectory() === true) return candidate
+    } catch {
+      // No node_modules at this level — keep walking up.
+    }
+    if (dir === root) return undefined
+    dir = path.dirname(dir)
+  }
+}
+
 const collectRelativeImportPaths = async ({
   sourceCode,
   sourcePath,
@@ -112,7 +139,13 @@ const collectRelativeImportPaths = async ({
   )
 }
 
-const stageCompiledBinaryImportGraph = ({
+/**
+ * Copy a genie file's relative/`#` import closure into a fresh `os.tmpdir()` staging directory and
+ * return the staged entry path (used by the compiled-binary import path, which cannot register the
+ * Bun import-map plugin). The importer's real `node_modules` is symlinked into the staged root so
+ * bare workspace-package / runtime imports still resolve. Exported for {@link stageCompiledBinaryImportGraph} tests.
+ */
+export const stageCompiledBinaryImportGraph = ({
   entryPath,
 }: {
   entryPath: string
@@ -128,6 +161,37 @@ const stageCompiledBinaryImportGraph = ({
         }),
     })
 
+    // Bare specifiers (`effect`, `@overeng/otel-contract`, `@effect/platform`, …) survive staging
+    // unchanged — `resolveImportMapsInSource` only rewrites `#`/`#mr`/relative specifiers. Staged
+    // modules are read from `os.tmpdir()`, outside the repo, so those bare imports would have no
+    // reachable `node_modules` and fail (in a compiled binary the bundled copies are not visible to
+    // externally-loaded files). Symlinking the importer's real `node_modules` into the staged root
+    // lets Bun resolve every bare import against the real on-disk install — exactly as a non-compiled
+    // `bun` run does when it imports the genie file in place. The bare-imported package's own
+    // transitive closure (its `effect`, its relative files) resolves from that package's real
+    // location; only the entry and its relative/`#` closure are ever copied, so a bare-imported
+    // package is loaded exactly once (avoids duplicate-singleton hazards, see mk-pnpm-cli.nix).
+    const nearestNodeModules = yield* Effect.tryPromise({
+      try: () => findNearestNodeModules(entryPath),
+      catch: (error) =>
+        new GenieImportError({
+          genieFilePath: entryPath,
+          message: `Failed to locate node_modules while staging compiled-binary imports for ${entryPath}: ${safeErrorString(error)}`,
+          cause: error,
+        }),
+    })
+    if (nearestNodeModules !== undefined) {
+      yield* Effect.tryPromise({
+        try: () => nodeFs.symlink(nearestNodeModules, path.join(tempRoot, 'node_modules'), 'dir'),
+        catch: (error) =>
+          new GenieImportError({
+            genieFilePath: entryPath,
+            message: `Failed to link node_modules into compiled-binary staging directory for ${entryPath}: ${safeErrorString(error)}`,
+            cause: error,
+          }),
+      })
+    }
+
     const stagedPaths = new Map<string, string>()
     const relativeEntryPath = entryPath.replace(/^(?:[A-Za-z]:)?[\\/]+/, '')
 
@@ -205,7 +269,8 @@ const stageCompiledBinaryImportGraph = ({
     return { stagePath, tempRoot }
   })
 
-const removeStagedCompiledBinaryImportGraph = ({
+/** Recursively remove a staged import graph's temp root (unlinks the `node_modules` symlink without following it). */
+export const removeStagedCompiledBinaryImportGraph = ({
   tempRoot,
 }: StagedCompiledBinaryImportGraph): Effect.Effect<void> =>
   Effect.sync(() => nodeFsSync.rmSync(tempRoot, { recursive: true, force: true })).pipe(

packages/@overeng/genie/src/core/staging-compiled-binary.unit.test.ts

@@ -0,0 +1,162 @@
+import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { pathToFileURL } from 'node:url'
+
+import { NodeContext } from '@effect/platform-node'
+import { Effect } from 'effect'
+import { afterEach, expect } from 'vitest'
+
+import { Vitest } from '@overeng/utils-dev/node-vitest'
+
+import {
+  removeStagedCompiledBinaryImportGraph,
+  stageCompiledBinaryImportGraph,
+} from './generation.ts'
+
+/**
+ * Compiled-binary staging must let a `.genie.ts` reach a bare workspace-package subpath
+ * (`@scope/contract/registry`) whose own closure reaches a bare runtime package (`effect`). The fix
+ * symlinks the importer's real `node_modules` into the staged tmp root so those bare imports resolve
+ * against the real on-disk install — the bundled copies in a `bun --compile` binary are not visible
+ * to externally-loaded staged files, and staging to `os.tmpdir()` otherwise strips node_modules reach.
+ *
+ * These fixtures are built with `node:fs` (no install state) so the test is hermetic and does not
+ * depend on effect-utils' own live node_modules.
+ */
+
+const TestLayer = NodeContext.layer
+
+const createdDirs: string[] = []
+
+/** Symlink-resolved temp dir so staged mirror paths match realpath'd on-disk names. */
+const makeDir = (prefix: string): string => {
+  const dir = realpathSync(mkdtempSync(path.join(os.tmpdir(), prefix)))
+  createdDirs.push(dir)
+  return dir
+}
+
+const write = (dir: string, relativePath: string, content: string): string => {
+  const filePath = path.join(dir, relativePath)
+  mkdirSync(path.dirname(filePath), { recursive: true })
+  writeFileSync(filePath, content, 'utf8')
+  return filePath
+}
+
+/**
+ * Build a hermetic repo fixture:
+ *   node_modules/effect            — a bare runtime package (the transitive edge)
+ *   node_modules/@scope/contract   — a bare workspace package exposing a `./registry` SUBPATH that
+ *                                    itself imports `effect`
+ *   packages/foo/<entry>.genie.ts  — imports the bare subpath and re-exports it as a GenieOutput
+ */
+const makeRepoFixture = ({ importSpecifier }: { importSpecifier: string }): string => {
+  const repoRoot = makeDir('genie-staging-repo-')
+
+  write(
+    repoRoot,
+    'node_modules/effect/package.json',
+    JSON.stringify({ name: 'effect', type: 'module', exports: { '.': './index.js' } }),
+  )
+  write(
+    repoRoot,
+    'node_modules/effect/index.js',
+    `export const Effect = { succeed: (value) => value }\n`,
+  )
+
+  write(
+    repoRoot,
+    'node_modules/@scope/contract/package.json',
+    JSON.stringify({
+      name: '@scope/contract',
+      type: 'module',
+      exports: { './registry': './registry.ts' },
+    }),
+  )
+  write(
+    repoRoot,
+    'node_modules/@scope/contract/registry.ts',
+    `import { Effect } from 'effect'\nexport const registry = { kind: Effect.succeed('semconv'), count: 2 }\n`,
+  )
+
+  write(
+    repoRoot,
+    'packages/foo/data.json.genie.ts',
+    [
+      `import { registry } from '${importSpecifier}'`,
+      `export default { data: registry, stringify: () => JSON.stringify(registry) }`,
+    ].join('\n'),
+  )
+
+  return path.join(repoRoot, 'packages/foo/data.json.genie.ts')
+}
+
+/** Stage the genie graph as the compiled-binary path does, dynamically import the staged entry, and return its default export. */
+const stageAndImport = (entryPath: string) =>
+  Effect.gen(function* () {
+    const staged = yield* stageCompiledBinaryImportGraph({ entryPath })
+    const importUrl = `${pathToFileURL(staged.stagePath).href}?import=${createdDirs.length}`
+    const module = yield* Effect.tryPromise(
+      // oxlint-disable-next-line eslint-plugin-import/no-dynamic-require -- staged path is dynamic by design
+      () => import(importUrl) as Promise<{ default: { data: unknown } }>,
+    ).pipe(Effect.ensuring(removeStagedCompiledBinaryImportGraph(staged)))
+    return { module, tempRoot: staged.tempRoot }
+  })
+
+afterEach(() => {
+  for (const dir of createdDirs.splice(0)) {
+    rmSync(dir, { recursive: true, force: true })
+  }
+})
+
+Vitest.describe('stageCompiledBinaryImportGraph', () => {
+  Vitest.it.effect(
+    'resolves a bare workspace-package subpath (and its transitive `effect`) via the staged node_modules symlink',
+    () =>
+      Effect.gen(function* () {
+        const entryPath = makeRepoFixture({ importSpecifier: '@scope/contract/registry' })
+        const { module } = yield* stageAndImport(entryPath)
+        // The registry data (built from the bare `effect` edge) flows through generation.
+        expect(module.default.data).toEqual({ kind: 'semconv', count: 2 })
+      }).pipe(Effect.provide(TestLayer)),
+  )
+
+  Vitest.it.effect(
+    'does NOT delete the real node_modules when the staged graph is cleaned up',
+    () =>
+      Effect.gen(function* () {
+        const entryPath = makeRepoFixture({ importSpecifier: '@scope/contract/registry' })
+        const realNodeModules = path.join(path.dirname(entryPath), '../../node_modules')
+        const sentinel = path.join(realNodeModules, 'effect/index.js')
+        yield* stageAndImport(entryPath) // stages, imports, then removes the temp root
+        expect(existsSync(realNodeModules)).toBe(true)
+        expect(existsSync(sentinel)).toBe(true)
+      }).pipe(Effect.provide(TestLayer)),
+  )
+
+  Vitest.it.effect(
+    'still stages a pure-relative graph when no node_modules exists (cold bootstrap unaffected)',
+    () =>
+      Effect.gen(function* () {
+        const repoRoot = makeDir('genie-staging-cold-')
+        write(repoRoot, 'packages/foo/helper.ts', `export const value = { ok: true }\n`)
+        const entryPath = write(
+          repoRoot,
+          'packages/foo/data.json.genie.ts',
+          [
+            `import { value } from './helper.ts'`,
+            `export default { data: value, stringify: () => JSON.stringify(value) }`,
+          ].join('\n'),
+        )
+        // No node_modules anywhere up the tree — staging must still succeed and not create a symlink.
+        const staged = yield* stageCompiledBinaryImportGraph({ entryPath })
+        expect(existsSync(path.join(staged.tempRoot, 'node_modules'))).toBe(false)
+        const importUrl = `${pathToFileURL(staged.stagePath).href}?import=cold`
+        const module = yield* Effect.tryPromise(
+          // oxlint-disable-next-line eslint-plugin-import/no-dynamic-require -- staged path is dynamic by design
+          () => import(importUrl) as Promise<{ default: { data: unknown } }>,
+        ).pipe(Effect.ensuring(removeStagedCompiledBinaryImportGraph(staged)))
+        expect(module.default.data).toEqual({ ok: true })
+      }).pipe(Effect.provide(TestLayer)),
+  )
+})