Split volatile library identity code

packages/cli/README.md

@@ -24,7 +24,7 @@ $ npm install -g scriptc
 
 Requires clang on the PATH (Xcode Command Line Tools on macOS, `clang` package on Linux).
 
-Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend. TypeScript comment-only edits can restore validated lowered IR instead, rebasing source locations and regenerating exact-source build identity before emission; directives, JSDoc-bearing JavaScript, token edits, configuration, package resolution, and newly appearing candidates still invalidate it. The native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions.
+Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend. TypeScript comment-only edits can restore validated lowered IR instead, rebasing source locations and regenerating exact-source build identity before emission; directives, JSDoc-bearing JavaScript, token edits, configuration, package resolution, and newly appearing candidates still invalidate it. Library identity getters live in a tiny C translation unit, so build-id-only changes reuse the large compiled program object and compile only that small member before rearchiving. The native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions.
 
 ## Commands
 

packages/cli/test/library-output.test.ts

@@ -0,0 +1,97 @@
+import { execFile } from "node:child_process";
+import { createRequire } from "node:module";
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { promisify } from "node:util";
+import { expect, test } from "vitest";
+
+const execFileAsync = promisify(execFile);
+const require = createRequire(import.meta.url);
+const repoRoot = join(import.meta.dirname, "../../..");
+const cliEntry = join(repoRoot, "packages/cli/src/main.ts");
+const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs");
+
+test("library identity source stays private and cannot overwrite a sidecar", async () => {
+  const tempRoot = process.platform === "win32" ? tmpdir() : "/tmp";
+  const dir = await mkdtemp(join(tempRoot, "scriptc-cli-library-output-"));
+  const cacheRoot = join(dir, "cache");
+  const outDir = join(dir, ".scriptc");
+  const profilePath = join(dir, "profile.json");
+  try {
+    await mkdir(cacheRoot, { mode: 0o700 });
+    await writeFile(join(dir, "lib.ts"), [
+      "export interface Model { value: number; }",
+      "export type Msg = { kind: \"noop\" } | { kind: \"set\"; value: number };",
+      "export function init(): Model { return { value: 1 }; }",
+      "export function update(model: Model, msg: Msg): Model {",
+      "  return msg.kind === \"set\" ? { value: msg.value } : model;",
+      "}",
+      "export function boot(): number { return init().value; }",
+      "",
+    ].join("\n"));
+
+    const writeProfile = (sidecarPath: string): Promise<void> => writeFile(
+      profilePath,
+      `${JSON.stringify({
+        profile_format: 1,
+        name: "cli-library-output",
+        entry: "lib.ts",
+        emission: "c",
+        abi: {
+          prefix: "clo_",
+          init_symbol: "clo_init",
+          sink_register_symbol: "clo_set_panic_sink",
+          collect_symbol: null,
+          result_reset_symbol: null,
+        },
+        exports: [{ export: "boot", symbol: "clo_boot", params: [], returns: "f64" }],
+        sidecar: {
+          path: sidecarPath,
+          wire_version: 1,
+          abi_version: 1,
+          snapshot_format: 1,
+          build_id_symbol: "clo_build_id",
+          abi_version_symbol: "clo_abi_version",
+          model: "Model",
+          msg: "Msg",
+        },
+      }, null, 2)}\n`,
+    );
+    const runBuild = async (): Promise<void> => {
+      await execFileAsync(
+        process.execPath,
+        ["--import", tsxLoader, cliEntry, "build", "--lib", "--profile", profilePath, "--no-keep-c"],
+        {
+          env: {
+            ...process.env,
+            TMPDIR: tempRoot,
+            SCRIPTC_CACHE_DIR: cacheRoot,
+          },
+          maxBuffer: 1024 * 1024,
+        },
+      );
+    };
+
+    // --no-keep-c removes the public program TU, and the identity source is
+    // invocation-private rather than a second caller-visible C artifact.
+    await writeProfile("contract.json");
+    await runBuild();
+    expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]);
+
+    // This name collided with the former fixed `<stem>.lib.identity.c`
+    // output. Repeat to exercise the exact early-cache-hit ordering that used
+    // to restore JSON and then overwrite it with generated C.
+    await rm(outDir, { recursive: true, force: true });
+    await writeProfile("lib.lib.identity.c");
+    await runBuild();
+    await runBuild();
+    const sidecar = JSON.parse(await readFile(join(outDir, "lib.lib.identity.c"), "utf8")) as {
+      build_id?: unknown;
+    };
+    expect(sidecar.build_id).toMatch(/^[0-9a-f]{16}$/);
+    expect((await readdir(outDir)).sort()).toEqual(["lib.lib.a", "lib.lib.identity.c"]);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});

packages/compiler/src/backend/cc-cache.test.ts

@@ -2936,3 +2936,126 @@ test("library archives hit by content, invalidate on edits, and reuse runtime ob
     else process.env["PATH"] = oldPath;
   }
 });
+
+test("library identity edits reuse the cached large program object", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-object-"));
+  scratch.push(dir);
+  const cacheRoot = join(dir, "cache");
+  const cPath = join(dir, "program.c");
+  const outPath = join(dir, "program.lib.a");
+  const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
+  const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
+  try {
+    process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
+    delete process.env["SCRIPTC_NO_CACHE"];
+    await mkdir(cacheRoot, { mode: 0o700 });
+    const programSource = "int scriptc_large_program_value(void) { return 7; }\n";
+    await writeFile(
+      cPath,
+      `${programSource}unsigned long long scriptc_build_id(void) { return 1; }\n`,
+    );
+    await compileLibArchive({
+      cPath,
+      programSource,
+      identityCSource: "unsigned long long scriptc_build_id(void) { return 1; }\n",
+      outPath,
+      cacheIdentity: TEST_CACHE_IDENTITY,
+    });
+    const [objectName] = (await readdir(join(cacheRoot, "program-obj")))
+      .filter((name) => !name.endsWith(".sha256"));
+    expect(objectName).toBeDefined();
+    const objectPath = join(cacheRoot, "program-obj", objectName!);
+    const objectDigest = await readFile(`${objectPath}.sha256`, "utf8");
+    const old = new Date("2000-01-01T00:00:00.000Z");
+    await utimes(objectPath, old, old);
+
+    await writeFile(
+      cPath,
+      `${programSource}unsigned long long scriptc_build_id(void) { return 2; }\n`,
+    );
+    await compileLibArchive({
+      cPath,
+      programSource,
+      identityCSource: "unsigned long long scriptc_build_id(void) { return 2; }\n",
+      outPath,
+      cacheIdentity: TEST_CACHE_IDENTITY,
+    });
+    expect(await readFile(`${objectPath}.sha256`, "utf8")).toBe(objectDigest);
+    expect((await stat(objectPath)).mtimeMs).toBeGreaterThan(old.getTime());
+    expect((await readdir(join(cacheRoot, "program-obj"))).filter((name) => !name.endsWith(".sha256"))).toEqual([objectName]);
+    const probeSource = join(dir, "probe.c");
+    const probe = join(dir, "probe");
+    await writeFile(
+      probeSource,
+      "#include <stdio.h>\nint scriptc_large_program_value(void);\nunsigned long long scriptc_build_id(void);\nint main(void) { printf(\"%d %llu\\n\", scriptc_large_program_value(), scriptc_build_id()); }\n",
+    );
+    execFileSync("clang", [probeSource, outPath, "-lm", "-o", probe]);
+    expect(execFileSync(probe, { encoding: "utf8" })).toBe("7 2\n");
+  } finally {
+    if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
+    else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
+    if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
+    else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
+  }
+});
+
+test.skipIf(process.platform === "win32" || zigExecutable === undefined)(
+  "cross-ELF localized archives retain unreferenced identity roots",
+  async () => {
+    const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-localized-identity-"));
+    scratch.push(dir);
+    const cPath = join(dir, "program.c");
+    const probePath = join(dir, "probe.c");
+    const archivePath = join(dir, "program.lib.a");
+    const probeOutput = join(dir, "probe");
+    const cacheRoot = join(dir, "cache");
+    const target = "x86_64-linux-gnu.2.36";
+    const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"];
+    const oldNoCache = process.env["SCRIPTC_NO_CACHE"];
+    const oldCc = process.env["SCRIPTC_CC"];
+    const oldTarget = process.env["SCRIPTC_TARGET"];
+    try {
+      process.env["SCRIPTC_CACHE_DIR"] = cacheRoot;
+      delete process.env["SCRIPTC_NO_CACHE"];
+      process.env["SCRIPTC_CC"] = "zigcc";
+      process.env["SCRIPTC_TARGET"] = target;
+      await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n");
+      await compileLibArchive({
+        cPath,
+        identityCSource: [
+          "unsigned long long scriptc_build_id(void) { return 2; }",
+          "unsigned scriptc_abi_version(void) { return 1; }",
+          "",
+        ].join("\n"),
+        outPath: archivePath,
+        cacheIdentity: TEST_CACHE_IDENTITY,
+        localizeSymbols: [
+          "scriptc_large_program_value",
+          "scriptc_build_id",
+          "scriptc_abi_version",
+        ],
+      });
+      await writeFile(probePath, [
+        "int scriptc_large_program_value(void);",
+        "unsigned long long scriptc_build_id(void);",
+        "unsigned scriptc_abi_version(void);",
+        "int main(void) {",
+        "  return scriptc_large_program_value() != 7 || scriptc_build_id() != 2 || scriptc_abi_version() != 1;",
+        "}",
+        "",
+      ].join("\n"));
+      execFileSync(zigExecutable!, [
+        "cc", "-target", target, probePath, archivePath, "-lm", "-o", probeOutput,
+      ]);
+    } finally {
+      if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"];
+      else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir;
+      if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"];
+      else process.env["SCRIPTC_NO_CACHE"] = oldNoCache;
+      if (oldCc === undefined) delete process.env["SCRIPTC_CC"];
+      else process.env["SCRIPTC_CC"] = oldCc;
+      if (oldTarget === undefined) delete process.env["SCRIPTC_TARGET"];
+      else process.env["SCRIPTC_TARGET"] = oldTarget;
+    }
+  },
+);