Cache emitted library translation units

packages/cli/test/library-output.test.ts

@@ -20,6 +20,12 @@ test("library identity source stays private and cannot overwrite a sidecar", asy
   const profilePath = join(dir, "profile.json");
   try {
     await mkdir(cacheRoot, { mode: 0o700 });
+    await writeFile(join(dir, "helper.ts"), [
+      "export function initialValue(): number {",
+      "  return 1;",
+      "}",
+      "",
+    ].join("\n"));
     await writeFile(join(dir, "lib.ts"), [
       "export interface Model { value: number; }",
       "export type Msg = { kind: \"noop\" } | { kind: \"set\"; value: number };",
@@ -58,10 +64,13 @@ test("library identity source stays private and cannot overwrite a sidecar", asy
         },
       }, null, 2)}\n`,
     );
-    const runBuild = async (): Promise<void> => {
+    const runBuild = async (keepC = false): Promise<void> => {
       await execFileAsync(
         process.execPath,
-        ["--import", tsxLoader, cliEntry, "build", "--lib", "--profile", profilePath, "--no-keep-c"],
+        [
+          "--import", tsxLoader, cliEntry, "build", "--lib", "--profile", profilePath,
+          ...(keepC ? [] : ["--no-keep-c"]),
+        ],
         {
           env: {
             ...process.env,
@@ -79,6 +88,54 @@ test("library identity source stays private and cannot overwrite a sidecar", asy
     await runBuild();
     expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]);
 
+    // A single-source comment-only edit takes the semantic cache path. Its
+    // restored public TU must match a forced miss before --no-keep-c removes
+    // it again.
+    await writeFile(
+      join(dir, "lib.ts"),
+      `/* harmless rebuild comment */ ${await readFile(join(dir, "lib.ts"), "utf8")}`,
+    );
+    await runBuild(true);
+    const semanticHitC = await readFile(join(outDir, "lib.lib.c"), "utf8");
+    await rm(join(cacheRoot, "early-lib"), { recursive: true, force: true });
+    await runBuild(true);
+    expect(await readFile(join(outDir, "lib.lib.c"), "utf8")).toBe(semanticHitC);
+    await runBuild();
+    expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]);
+
+    // A line-shifting edit cannot safely reuse line-only annotations (not even
+    // synthetic byte-zero locations). It must match a forced frontend miss.
+    await writeFile(join(dir, "lib.ts"), [
+      "// line-shifting rebuild comment",
+      await readFile(join(dir, "lib.ts"), "utf8"),
+    ].join("\n"));
+    await runBuild(true);
+    const shiftedC = await readFile(join(outDir, "lib.lib.c"), "utf8");
+    await rm(join(cacheRoot, "early-lib"), { recursive: true, force: true });
+    await runBuild(true);
+    expect(await readFile(join(outDir, "lib.lib.c"), "utf8")).toBe(shiftedC);
+
+    // Move to a multi-source graph and seed its cache. Imported trivia is
+    // semantically unchanged too, but cached C annotations cannot be rebased
+    // through the entry-only line table. That shape must take the normal
+    // frontend path and match a forced cache miss.
+    await writeFile(join(dir, "lib.ts"), (await readFile(join(dir, "lib.ts"), "utf8"))
+      .replace(
+        "export interface Model",
+        "import { initialValue } from \"./helper.js\";\nexport interface Model",
+      )
+      .replace("value: 1", "value: initialValue()"));
+    await runBuild(true);
+    await writeFile(join(dir, "helper.ts"), [
+      "// harmless helper comment",
+      await readFile(join(dir, "helper.ts"), "utf8"),
+    ].join("\n"));
+    await runBuild(true);
+    const fallbackC = await readFile(join(outDir, "lib.lib.c"), "utf8");
+    await rm(join(cacheRoot, "early-lib"), { recursive: true, force: true });
+    await runBuild(true);
+    expect(await readFile(join(outDir, "lib.lib.c"), "utf8")).toBe(fallbackC);
+
     // This name collided with the former fixed `<stem>.lib.identity.c`
     // output. Repeat to exercise the exact early-cache-hit ordering that used
     // to restore JSON and then overwrite it with generated C.

packages/compiler/src/backend/emission/emitter.ts

@@ -62,7 +62,7 @@ import { emitAsyncScaffolding, childDataThunkFor, childExitThunkFor, childExitTh
 import { emitNpmEmbedding, islandAdapter, islandTypedAdapter } from "./emit-island.js";
 import { emitFunction, emitBlock, emitStmts, emitStmt, emitTryCatch, emitSwitch, mergeBrace, emitBranchInto, emitCondition } from "./emit-stmts.js";
 import { emitExpr } from "./emit-exprs.js";
-import { C_LIBRARY_IDENTITY_BEGIN, C_LIBRARY_IDENTITY_END } from "../library-identity.js";
+import { emitLibraryIdentityLines } from "../library-identity.js";
 
 export interface CEmitOptions {
   /** Library archive assembly may move the volatile identity getters into a
@@ -1158,18 +1158,7 @@ export class CEmitter {
       // pairing fence): pure data returns with NO entry prologue — exempt
       // from the poisoned guard and every runtime touch (ratified), so a
       // host can read them before init and after a trap.
-      out.push(
-        C_LIBRARY_IDENTITY_BEGIN,
-        `uint64_t ${lib.identity.buildIdSymbol}(void) {`,
-        `  return UINT64_C(0x${lib.identity.buildId});`,
-        `}`,
-        ``,
-        `uint32_t ${lib.identity.abiVersionSymbol}(void) {`,
-        `  return ${lib.identity.abiVersion}u;`,
-        `}`,
-        ``,
-        C_LIBRARY_IDENTITY_END,
-      );
+      out.push(...emitLibraryIdentityLines("c", lib.identity));
     }
     if (lib.resultResetSymbol !== null) {
       out.push(

packages/compiler/src/backend/library-identity.ts

@@ -5,29 +5,129 @@ export const C_LIBRARY_IDENTITY_END = "/* scriptc-library-identity: end */";
 export const LLVM_LIBRARY_IDENTITY_BEGIN = "; scriptc-library-identity: begin";
 export const LLVM_LIBRARY_IDENTITY_END = "; scriptc-library-identity: end";
 
-/** Remove the generated identity region from a complete caller-visible
- * library TU. Archive assembly compiles this stable projection beside the
- * small volatile identity C object, while the public TU remains complete. */
-export function stripLibraryIdentity(
+export interface LibraryIdentityValues {
+  buildIdSymbol: string;
+  abiVersionSymbol: string;
+  buildId: string;
+  abiVersion: number;
+}
+
+function identityMarkers(emission: LibraryEmission): { begin: string; end: string } {
+  return emission === "c"
+    ? { begin: C_LIBRARY_IDENTITY_BEGIN, end: C_LIBRARY_IDENTITY_END }
+    : { begin: LLVM_LIBRARY_IDENTITY_BEGIN, end: LLVM_LIBRARY_IDENTITY_END };
+}
+
+export function emitLibraryIdentityLines(
+  emission: LibraryEmission,
+  identity: LibraryIdentityValues,
+  llvmFunctionAttrs = "#0",
+): string[] {
+  if (!/^[0-9a-f]{16}$/.test(identity.buildId)) {
+    throw new Error("library build id must be exactly 16 lowercase hex digits");
+  }
+  if (emission === "c") {
+    return [
+      C_LIBRARY_IDENTITY_BEGIN,
+      `uint64_t ${identity.buildIdSymbol}(void) {`,
+      `  return UINT64_C(0x${identity.buildId});`,
+      `}`,
+      ``,
+      `uint32_t ${identity.abiVersionSymbol}(void) {`,
+      `  return ${identity.abiVersion}u;`,
+      `}`,
+      ``,
+      C_LIBRARY_IDENTITY_END,
+    ];
+  }
+  const signedBuildId = BigInt.asIntN(64, BigInt(`0x${identity.buildId}`)).toString();
+  return [
+    LLVM_LIBRARY_IDENTITY_BEGIN,
+    `define i64 @${identity.buildIdSymbol}() ${llvmFunctionAttrs} { ; identity getter build_id 0x${identity.buildId}`,
+    `entry:`,
+    `  ret i64 ${signedBuildId}`,
+    `}`,
+    ``,
+    `define i32 @${identity.abiVersionSymbol}() ${llvmFunctionAttrs} { ; identity getter abi_version`,
+    `entry:`,
+    `  ret i32 ${identity.abiVersion}`,
+    `}`,
+    ``,
+    LLVM_LIBRARY_IDENTITY_END,
+  ];
+}
+
+function identityOffsets(
   source: string,
   emission: LibraryEmission,
-): string {
-  const begin = emission === "c"
-    ? C_LIBRARY_IDENTITY_BEGIN
-    : LLVM_LIBRARY_IDENTITY_BEGIN;
-  const end = emission === "c"
-    ? C_LIBRARY_IDENTITY_END
-    : LLVM_LIBRARY_IDENTITY_END;
+): { start: number; end: number } | null {
+  const { begin, end } = identityMarkers(emission);
   const start = source.indexOf(`${begin}\n`);
-  if (start < 0) return source;
+  if (start < 0) return null;
   if (source.indexOf(begin, start + begin.length) >= 0) {
     throw new Error("generated library TU contains multiple identity regions");
   }
   const endStart = source.indexOf(end, start + begin.length);
   if (endStart < 0) throw new Error("generated library TU has an unterminated identity region");
-  const endOffset = endStart + end.length;
+  return { start, end: endStart + end.length };
+}
+
+/** Refresh only the volatile identity block in a cached public TU. */
+export function replaceLibraryIdentity(
+  source: string,
+  emission: LibraryEmission,
+  identity: LibraryIdentityValues,
+): string {
+  const offsets = identityOffsets(source, emission);
+  if (offsets === null) throw new Error("generated public library TU has no identity region");
+  const replacement = emitLibraryIdentityLines(emission, identity).join("\n");
+  return source.slice(0, offsets.start) + replacement + source.slice(offsets.end);
+}
+
+function escapedRegExp(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function sourceCommentPattern(sourceFile: string): RegExp {
+  // Emission appends these comments at physical line ends. Requiring that
+  // boundary prevents source-looking text inside a generated C string from
+  // being mistaken for an annotation and changing program semantics. An
+  // uninitialized reference declaration appends its own `/* let ... */`
+  // explanation after the location, so admit that one generated suffix too.
+  return new RegExp(
+    ` /\\* ${escapedRegExp(sourceFile)}:(\\d+) \\*/` +
+      `(?= /\\* let [^\\r\\n]*; \\*/(?=\\r?\\n|$)|\\r?\\n|$)`,
+    "g",
+  );
+}
+
+/** Refresh the source-line annotations retained in a caller-visible C TU. */
+export function rebaseLibrarySourceComments(
+  source: string,
+  sourceFile: string,
+  rebaseLine: (line: number) => number,
+): string {
+  return source.replace(sourceCommentPattern(sourceFile), (_match, line: string) =>
+    ` /* ${sourceFile}:${rebaseLine(Number(line))} */`);
+}
+
+/** Remove C-only debugging annotations from the private native-cache input. */
+export function stripLibrarySourceComments(source: string, sourceFile: string): string {
+  return source.replace(sourceCommentPattern(sourceFile), "");
+}
+
+/** Remove the generated identity region from a complete caller-visible
+ * library TU. Archive assembly compiles this stable projection beside the
+ * small volatile identity C object, while the public TU remains complete. */
+export function stripLibraryIdentity(
+  source: string,
+  emission: LibraryEmission,
+): string {
+  const offsets = identityOffsets(source, emission);
+  if (offsets === null) return source;
+  const endOffset = offsets.end;
   const suffix = source[endOffset] === "\n" ? endOffset + 1 : endOffset;
-  let prefix = source.slice(0, start);
+  let prefix = source.slice(0, offsets.start);
   // An omitted final region leaves the emitter's preceding empty array entry
   // as one trailing newline; a marked final region has material after that
   // entry and therefore renders it as two. Restore the omitted form exactly.

packages/compiler/src/backend/llvm/emitter.ts

@@ -61,7 +61,7 @@
  * lazy-inflate representation as the C debugging backend.
  */
 import { deflateRawSync } from "node:zlib";
-import { LLVM_LIBRARY_IDENTITY_BEGIN, LLVM_LIBRARY_IDENTITY_END } from "../library-identity.js";
+import { emitLibraryIdentityLines } from "../library-identity.js";
 import type {
   IrBytesElem,
   IrExpr,
@@ -2334,21 +2334,7 @@ class LlEmitter {
       // from the poisoned guard and every runtime touch (ratified), so a
       // host can read them before init and after a trap. The u64 rides
       // i64 two's-complement (LLVM integer constants are signed).
-      const buildId = BigInt.asIntN(64, BigInt(`0x${lib.identity.buildId}`)).toString();
-      out.push(
-        LLVM_LIBRARY_IDENTITY_BEGIN,
-        `define i64 @${lib.identity.buildIdSymbol}() ${FN_ATTRS} { ; identity getter build_id 0x${lib.identity.buildId}`,
-        `entry:`,
-        `  ret i64 ${buildId}`,
-        `}`,
-        ``,
-        `define i32 @${lib.identity.abiVersionSymbol}() ${FN_ATTRS} { ; identity getter abi_version`,
-        `entry:`,
-        `  ret i32 ${lib.identity.abiVersion}`,
-        `}`,
-        ``,
-        LLVM_LIBRARY_IDENTITY_END,
-      );
+      out.push(...emitLibraryIdentityLines("llvm", lib.identity, FN_ATTRS));
     }
     if (lib.resultResetSymbol !== null) {
       out.push(

packages/compiler/src/index.ts

@@ -3,7 +3,7 @@ import { basename, dirname, join, resolve } from "node:path";
 import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js";
 import { emitModule } from "./backend/emission/emitter.js";
 import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js";
-import { stripLibraryIdentity } from "./backend/library-identity.js";
+import { rebaseLibrarySourceComments, replaceLibraryIdentity, stripLibraryIdentity, stripLibrarySourceComments } from "./backend/library-identity.js";
 import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "./diagnostics/diagnostic.js";
 import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./library/int-infer.js";
 import { loadLibraryProfile, profileRemediation, profileTeaching, type LibraryProfile } from "./library/profile.js";
@@ -37,6 +37,7 @@ import { loadFfiProfile, type FfiProfile } from "./ffi/profile.js";
 import { hasForeignFfiCallback } from "./backend/ffi-callbacks.js";
 import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js";
 import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, readEarlyLibraryCache, readSemanticLibraryCache, type EarlyLibraryCacheOptions, type EarlyLibraryCachePublish, type EarlyLibraryNativeFeatures, type SemanticLibraryCacheHit } from "./library/early-cache.js";
+import { createSourceLineRebaser } from "./library/semantic-source.js";
 
 export const VERSION = "0.0.1";
 
@@ -1553,13 +1554,17 @@ async function compileLibraryNative(
   const localizeSymbols = libraryLocalizeSymbols(profile);
   let identityCSource: string | undefined;
   let programSource: string | undefined;
+  if (profile.sidecar !== null || profile.emission === "c") {
+    const publicSource = await readFile(cPath, "utf8");
+    programSource = publicSource;
+  }
   if (profile.sidecar !== null) {
     if (features.buildId === undefined) throw new Error("library identity TU has no build id");
-    const publicSource = await readFile(cPath, "utf8");
-    programSource = stripLibraryIdentity(publicSource, profile.emission);
-    if (programSource === publicSource) {
+    const withoutIdentity = stripLibraryIdentity(programSource!, profile.emission);
+    if (withoutIdentity === programSource) {
       throw new Error("generated public library TU has no identity region");
     }
+    programSource = withoutIdentity;
     identityCSource = [
       "#include <stdint.h>",
       "#include <inttypes.h>",
@@ -1568,6 +1573,9 @@ async function compileLibraryNative(
       "",
     ].join("\n");
   }
+  if (profile.emission === "c") {
+    programSource = stripLibrarySourceComments(programSource!, profile.entry);
+  }
   await compileLibArchive({
     cPath,
     ...(programSource !== undefined ? { programSource } : {}),
@@ -1626,20 +1634,27 @@ async function emitSemanticLibraryHit(
   }
   await mkdir(opts.outDir, { recursive: true });
   const stem = basename(profile.entry).replace(/\.(ts|js|mjs|cjs)$/, "");
-  let cPath: string;
+  const cPath = join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "ll" : "c"}`);
+  let translationUnit = hit.translationUnit;
+  if (profile.sidecar !== null) {
+    translationUnit = replaceLibraryIdentity(translationUnit, profile.emission, mod.lib!.identity!);
+  }
   if (profile.emission === "llvm") {
-    const ll = emitLlvmModule(mod);
-    cPath = join(opts.outDir, `${stem}.lib.ll`);
-    await writeFile(cPath, ll);
-    timing("semantic-llvm-emit", { output_bytes: Buffer.byteLength(ll) });
+    await writeFile(cPath, translationUnit);
   } else {
-    cPath = join(opts.outDir, `${stem}.lib.c`);
-    await writeFile(
-      cPath,
-      emitModule(mod, hit.sourceTexts.get(profile.entry)),
+    const previous = hit.previousSources.get(mod.sourceFile);
+    const current = hit.sourceTexts.get(mod.sourceFile);
+    if (previous === undefined || current === undefined) {
+      throw new Error("semantic library cache lost the entry source text");
+    }
+    translationUnit = rebaseLibrarySourceComments(
+      translationUnit,
+      mod.sourceFile,
+      createSourceLineRebaser(mod.sourceFile, previous, current),
     );
-    timing("semantic-c-emit");
+    await writeFile(cPath, translationUnit);
   }
+  timing("semantic-tu-restore", { output_bytes: Buffer.byteLength(translationUnit) });
   await rm(join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "c" : "ll"}`), { force: true });
   let irPath: string | undefined;
   if (opts.emitIr) {