From 94d6123c53fa9277f279a6b68df28d373f633054 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 19 Aug 2026 14:45:47 -0500 Subject: [PATCH 1/5] Split volatile library identity code - Emit build identity getters from a tiny standalone C translation unit - Cache generated library program objects independently of volatile identity and archives - Preserve toolchain, runtime, and translation-unit dependency invalidation coverage --- packages/cli/README.md | 2 +- .../compiler/src/backend/cc-cache.test.ts | 55 ++++++++ packages/compiler/src/backend/cc.ts | 126 +++++++++++++++--- .../compiler/src/backend/emission/emitter.ts | 16 --- packages/compiler/src/backend/llvm/emitter.ts | 20 --- packages/compiler/src/index.ts | 16 +++ packages/compiler/src/library/early-cache.ts | 6 +- tests/harness/README.md | 1 + 8 files changed, 184 insertions(+), 58 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 0abadbf5..657ff09a 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -24,7 +24,7 @@ $ npm install -g scriptc Requires clang on the PATH (Xcode Command Line Tools on macOS, `clang` package on Linux). -Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend. TypeScript comment-only edits can restore validated lowered IR instead, rebasing source locations and regenerating exact-source build identity before emission; directives, JSDoc-bearing JavaScript, token edits, configuration, package resolution, and newly appearing candidates still invalidate it. The native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. +Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend. TypeScript comment-only edits can restore validated lowered IR instead, rebasing source locations and regenerating exact-source build identity before emission; directives, JSDoc-bearing JavaScript, token edits, configuration, package resolution, and newly appearing candidates still invalidate it. Library identity getters live in a tiny C translation unit, so build-id-only changes reuse the large compiled program object and compile only that small member before rearchiving. The native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. ## Commands diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index 241a126e..1478bc91 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -2936,3 +2936,58 @@ test("library archives hit by content, invalidate on edits, and reuse runtime ob else process.env["PATH"] = oldPath; } }); + +test("library identity edits reuse the cached large program object", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-object-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const identityCPath = join(dir, "identity.c"); + const outPath = join(dir, "program.lib.a"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await mkdir(cacheRoot, { mode: 0o700 }); + await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n"); + await writeFile(identityCPath, "unsigned long long scriptc_build_id(void) { return 1; }\n"); + await compileLibArchive({ + cPath, + identityCPath, + outPath, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + const [objectName] = (await readdir(join(cacheRoot, "program-obj"))) + .filter((name) => !name.endsWith(".sha256")); + expect(objectName).toBeDefined(); + const objectPath = join(cacheRoot, "program-obj", objectName!); + const objectDigest = await readFile(`${objectPath}.sha256`, "utf8"); + const old = new Date("2000-01-01T00:00:00.000Z"); + await utimes(objectPath, old, old); + + await writeFile(identityCPath, "unsigned long long scriptc_build_id(void) { return 2; }\n"); + await compileLibArchive({ + cPath, + identityCPath, + outPath, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + expect(await readFile(`${objectPath}.sha256`, "utf8")).toBe(objectDigest); + expect((await stat(objectPath)).mtimeMs).toBeGreaterThan(old.getTime()); + expect((await readdir(join(cacheRoot, "program-obj"))).filter((name) => !name.endsWith(".sha256"))).toEqual([objectName]); + const probeSource = join(dir, "probe.c"); + const probe = join(dir, "probe"); + await writeFile( + probeSource, + "#include \nint scriptc_large_program_value(void);\nunsigned long long scriptc_build_id(void);\nint main(void) { printf(\"%d %llu\\n\", scriptc_large_program_value(), scriptc_build_id()); }\n", + ); + execFileSync("clang", [probeSource, outPath, "-lm", "-o", probe]); + expect(execFileSync(probe, { encoding: "utf8" })).toBe("7 2\n"); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } +}); diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 8af648ae..5c027dfa 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -1398,6 +1398,11 @@ const LIB_RUNTIME_SOURCES = [ export interface LibArchiveOptions { /** The program TU (.c or .ll — clang compiles either with -c). */ cPath: string; + /** Tiny generated C TU carrying volatile library identity getters. Its + * bytes join the complete archive key, but the large program-object cache + * is keyed independently so comment-only build-id changes compile only + * this file. */ + identityCPath?: string; /** The archive to produce (.lib.a). */ outPath: string; /** Caller-owned identity for the generated TU's complete non-system @@ -1568,29 +1573,29 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise let runtimeHash = ""; let programDependencyHash = ""; let cachedProgramBytes: Buffer | null = null; + let cachedIdentityBytes: Buffer | null = null; if (root !== null) { try { - const [cv, fingerprint, programBytes] = await Promise.all([ + const [cv, fingerprint, programBytes, identityBytes] = await Promise.all([ ccVersionOnce(driver.argv, toolchainEnv, true), runtimeFingerprint(rtDir), readFile(opts.cPath), + opts.identityCPath === undefined ? Promise.resolve(null) : readFile(opts.identityCPath), ]); compilerVersion = cv; runtimeHash = fingerprint; cachedProgramBytes = programBytes; + cachedIdentityBytes = identityBytes; + programDependencyHash = await translationUnitDependencyFingerprint( + driver, + cflags, + opts.cPath, + programBytes, + toolchainEnv, + ); if (cacheCompleteArchive) { - const [av, programDependencies] = await Promise.all([ - toolVersionOnce(arArgv, toolchainEnv, true), - translationUnitDependencyFingerprint( - driver, - cflags, - opts.cPath, - programBytes, - toolchainEnv, - ), - ]); + const av = await toolVersionOnce(arArgv, toolchainEnv, true); archiverVersion = av; - programDependencyHash = programDependencies; const key = createHash("sha256") // v7 adds effective compiler-wrapper invocations for the real runtime // and program compile flavors. @@ -1600,7 +1605,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise .update(implicitToolchain!).update("\0") .update(runtimeCompilerInvocation!).update("\0") .update(programCompilerInvocation!).update("\0") - .update(programDependencies).update("\0") + .update(programDependencyHash).update("\0") .update(opts.cacheIdentity!).update("\0") .update(driver.argv.join("\x1f")).update("\0") .update(cv).update("\0") @@ -1615,6 +1620,10 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise .update(opts.cPath).update("\0") .update(resolve(opts.cPath)).update("\0") .update(programBytes) + .update("\0identity\0") + .update(opts.identityCPath === undefined ? "" : resolve(opts.identityCPath)) + .update("\0") + .update(identityBytes ?? Buffer.alloc(0)) .digest("hex"); cachedArchive = join(root, "lib", key); const tmpOut = privateSiblingPath(opts.outPath, "lib-hit"); @@ -1693,11 +1702,88 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise ? opts.cPath : join(buildDir, `program${opts.cPath.endsWith(".ll") ? ".ll" : ".c"}`); if (cachedProgramBytes !== null) await writeFile(programSource, cachedProgramBytes); - const programObject = await compileOne( - programSource, - `${stem}.program.o`, - cachedProgramBytes === null ? undefined : opts.cPath, - ); + let cachedProgramObject: string | null = null; + if ( + root !== null && cachedProgramBytes !== null && compilerVersion !== "" && + implicitToolchain !== null && programCompilerInvocation !== null + ) { + const programKey = createHash("sha256") + .update("lib-program-obj-v1\0") + .update(cacheTargetIdentity(driver)).update("\0") + .update(toolchainEnv).update("\0") + .update(implicitToolchain).update("\0") + .update(programCompilerInvocation).update("\0") + .update(opts.cacheIdentity!).update("\0") + .update(driver.argv.join("\x1f")).update("\0") + .update(compilerVersion).update("\0") + .update(runtimeHash).update("\0") + .update(programDependencyHash).update("\0") + .update(programCompilerArgs.join("\x1f")).update("\0") + .update(opts.cPath).update("\0") + .update(resolve(opts.cPath)).update("\0") + .update(cachedProgramBytes) + .digest("hex"); + cachedProgramObject = join(root, "program-obj", programKey); + } + const stagedProgramObject = join(buildDir, `${stem}.program.o`); + let programObject: string; + if ( + cachedProgramObject !== null && + await copyValidCachedFile(cachedProgramObject, stagedProgramObject) + ) { + programObject = stagedProgramObject; + } else { + programObject = await compileOne( + programSource, + `${stem}.program.o`, + cachedProgramBytes === null ? undefined : opts.cPath, + ); + if (cachedProgramObject !== null) { + try { + const [currentRuntime, currentImplicit, currentInvocation, currentDependencies, currentCompiler] = + await Promise.all([ + runtimeFingerprint(rtDir), + implicitToolchainFingerprint(driver, toolchainEnv), + effectiveCompilerInvocationFingerprint( + driver, + toolchainEnv, + programCompilerArgs, + programSourceExtension, + ), + translationUnitDependencyFingerprint( + driver, + cflags, + opts.cPath, + cachedProgramBytes!, + toolchainEnv, + ), + ccVersionOnce(driver.argv, toolchainEnv, true), + ]); + if ( + currentRuntime === runtimeHash && + currentImplicit === implicitToolchain && + currentInvocation === programCompilerInvocation && + currentDependencies === programDependencyHash && + currentCompiler === compilerVersion + ) { + await publishCachedFile(programObject, cachedProgramObject); + } + } catch { + // Best-effort: the archive build already owns a valid object. + } + } + } + const identityObject = opts.identityCPath === undefined + ? null + : await compileOne( + cachedIdentityBytes === null ? opts.identityCPath : await (async () => { + const source = join(buildDir, "identity.c"); + await writeFile(source, cachedIdentityBytes); + return source; + })(), + `${stem}.identity.o`, + cachedIdentityBytes === null ? undefined : opts.identityCPath, + ); let runtimeObjects: string[] | null = null; let cacheInputsStable = true; let objectImplicitVerification: Promise | null = null; @@ -1746,7 +1832,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise ); } } - const objects = [programObject, ...runtimeObjects, ...lreObjects, ...zlibObjects]; + const objects = [programObject, ...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects]; // Multi-instance library mode: the archive's one member becomes the // combined, symbol-localized object (cached vendor/runtime objects // are read-only inputs here — the combine step never mutates them). @@ -1759,7 +1845,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise arArgv, buildDir, programObject, - [...runtimeObjects, ...lreObjects, ...zlibObjects], + [...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem, ), diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index 4403d160..d463c01e 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -1141,22 +1141,6 @@ export class CEmitter { } out.push(` return -1;`, `}`, ``); } - if (lib.identity !== undefined) { - // Profile-declared identity getters (the ask-2 sidecar's boot-time - // pairing fence): pure data returns with NO entry prologue — exempt - // from the poisoned guard and every runtime touch (ratified), so a - // host can read them before init and after a trap. - out.push( - `uint64_t ${lib.identity.buildIdSymbol}(void) {`, - ` return UINT64_C(0x${lib.identity.buildId});`, - `}`, - ``, - `uint32_t ${lib.identity.abiVersionSymbol}(void) {`, - ` return ${lib.identity.abiVersion}u;`, - `}`, - ``, - ); - } if (lib.resultResetSymbol !== null) { out.push( `void ${lib.resultResetSymbol}(void) {`, diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index ea05035f..2b7ab68c 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -2322,26 +2322,6 @@ class LlEmitter { }); out.push(`miss:`, ` ret i32 -1`, `}`, ``); } - if (lib.identity !== undefined) { - // Profile-declared identity getters (the ask-2 sidecar's boot-time - // pairing fence): pure data returns with NO entry prologue — exempt - // from the poisoned guard and every runtime touch (ratified), so a - // host can read them before init and after a trap. The u64 rides - // i64 two's-complement (LLVM integer constants are signed). - const buildId = BigInt.asIntN(64, BigInt(`0x${lib.identity.buildId}`)).toString(); - out.push( - `define i64 @${lib.identity.buildIdSymbol}() ${FN_ATTRS} { ; identity getter build_id 0x${lib.identity.buildId}`, - `entry:`, - ` ret i64 ${buildId}`, - `}`, - ``, - `define i32 @${lib.identity.abiVersionSymbol}() ${FN_ATTRS} { ; identity getter abi_version`, - `entry:`, - ` ret i32 ${lib.identity.abiVersion}`, - `}`, - ``, - ); - } if (lib.resultResetSymbol !== null) { out.push( `define void @${lib.resultResetSymbol}() ${FN_ATTRS} {`, diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 90e38ab7..bc1dada1 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1522,6 +1522,7 @@ function libraryNativeFeatures( zlib: moduleUsesZlib(mod), copying: moduleUsesCopying(mod), textDecoderLegacy: moduleUsesLegacyTextDecoder(mod), + ...(mod.lib?.identity !== undefined ? { buildId: mod.lib.identity.buildId } : {}), }; } @@ -1549,8 +1550,22 @@ async function compileLibraryNative( features: EarlyLibraryNativeFeatures, ): Promise { const localizeSymbols = libraryLocalizeSymbols(profile); + let identityCPath: string | undefined; + if (profile.sidecar !== null) { + if (features.buildId === undefined) throw new Error("library identity TU has no build id"); + const stem = basename(profile.entry).replace(/\.(ts|js|mjs|cjs)$/, ""); + identityCPath = join(dirname(cPath), `${stem}.lib.identity.c`); + await writeFile(identityCPath, [ + "#include ", + "#include ", + `uint64_t ${profile.sidecar.buildIdSymbol}(void) { return UINT64_C(0x${features.buildId}); }`, + `uint32_t ${profile.sidecar.abiVersionSymbol}(void) { return ${profile.sidecar.abiVersion}u; }`, + "", + ].join("\n")); + } await compileLibArchive({ cPath, + ...(identityCPath !== undefined ? { identityCPath } : {}), outPath: archivePath, cacheIdentity: "scriptc-generated-library-v1", sanitize, @@ -1592,6 +1607,7 @@ async function emitSemanticLibraryHit( modules, ); mod.lib.identity.buildId = buildId; + hit.native.buildId = buildId; sidecarJson = updateSidecarIdentity(sidecarJson, buildId, sourceHash); } const validation = validateModule(mod); diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index 0124be20..d05c8ac5 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -56,6 +56,8 @@ export interface EarlyLibraryNativeFeatures { zlib: boolean; copying: boolean; textDecoderLegacy: boolean; + /** Volatile exact-source build identity emitted from the tiny identity TU. */ + buildId?: string; } export interface EarlyLibraryCacheOptions { @@ -113,7 +115,8 @@ function validNativeFeatures(value: unknown): value is EarlyLibraryNativeFeature native.zlib, native.copying, native.textDecoderLegacy, - ].every((flag) => typeof flag === "boolean"); + ].every((flag) => typeof flag === "boolean") && + (native.buildId === undefined || /^[0-9a-f]{16}$/.test(native.buildId)); } function digest(bytes: Uint8Array): string { @@ -211,6 +214,7 @@ function frontendOutputExclusions( const outputArtifacts = [ paths.cPath, paths.staleCPath, + join(options.outDir, `${basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.identity.c`), ...(options.emitIr ? [paths.irPath] : []), archiveOutputPath(options), ...(sidecarPath === undefined ? [] : [sidecarPath]), diff --git a/tests/harness/README.md b/tests/harness/README.md index b1e845d7..5ff9ff76 100644 --- a/tests/harness/README.md +++ b/tests/harness/README.md @@ -73,6 +73,7 @@ Test runs are dominated by clang (~275 corpus programs × two lanes at -O2/-O1+A - **binaries** (`bin/`, cc.ts): key = resolved clang identity/version + target/compiler environment + implicit system-header dependency bytes + linker/assembler identities + runtime fingerprint (every runtime .c/.h + the vendor pin) + the full normalized command line + the emitted C bytes (byte-stable by project invariant). A hit skips native code generation and linking; the binary still RUNS live, so no comparison or sanitizer coverage is ever skipped. Each hit is checksum-verified. The sanitized lane's flags land in naturally distinct keys. FFI archive/object inputs and ambient system libraries always relink because their named files can hide mutable transitive dependencies. - **library archives** (`lib/`, cc.ts): key = resolved clang and archiver identities/versions + target/compiler environment and implicit dependencies + runtime fingerprint + target/flags + gated runtime-source set + emitted program-TU bytes. A checksum-verified hit skips native code generation and `ar`. +- **library program objects** (`program-obj/`, cc.ts): generated library TUs compile into checksum-verified objects keyed independently of the tiny exact-source identity TU. A build-id-only miss reuses the large program object, compiles the identity getters, and rearchives; runtime/header/toolchain inputs remain part of the key and are rechecked before publication. - **early library frontend** (`early-lib/`, library/early-cache.ts): exact library repeats validate content hashes for every file the TypeScript frontend read plus recorded failed-resolution and directory-enumeration probes, then restore the generated C/LLVM unit, optional IR, sidecar, and native feature gates without spawning TypeScript or lowering again. TypeScript comment-only misses may restore compressed lowered IR after token equivalence checks; source locations and exact-source sidecar/build identities regenerate from current bytes. Semantic comments/directives, JavaScript comments, token/config/package edits, and newly appearing resolution candidates miss. The native archive tier still performs its own toolchain/runtime checks. - **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Each object carries a verified digest; a damaged entry is rebuilt before it reaches the linker or archiver. Publication rechecks the runtime and implicit-toolchain fingerprints after compilation so a concurrent source/header edit cannot place new bytes under an old key. Compiles route through ccache when installed, silently falling back when not. - **oracle results** (`oracle/`, differential.test.ts): Node's stdout/exit per program, keyed by program bytes + the spawned node's version + shim contents + invocation shape. Only the spawn is skipped; the comparison never changes. Real-time programs (setTimeout/setInterval/Promise.race — 18 of 298) are excluded and always spawn Node live: their stdout is a timer interleave that Node and the native binary only agree on under the same instantaneous load, so a cached verdict from one run must never meet a live native run from another. From 6b2d31ce173618bcbeb5b6da386702cc2bc41b1e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 19 Aug 2026 14:59:25 -0500 Subject: [PATCH 2/5] Preserve library identity during localization --- .../compiler/src/backend/cc-cache.test.ts | 63 +++++++++++++++++ packages/compiler/src/backend/cc.ts | 28 ++++---- .../src/backend/object-localize.test.ts | 67 +++++++++++++++---- .../compiler/src/backend/object-localize.ts | 35 +++++----- 4 files changed, 150 insertions(+), 43 deletions(-) diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index 1478bc91..ca6701cc 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -2991,3 +2991,66 @@ test("library identity edits reuse the cached large program object", async () => else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; } }); + +test.skipIf(process.platform === "win32" || zigExecutable === undefined)( + "cross-ELF localized archives retain unreferenced identity roots", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-localized-identity-")); + scratch.push(dir); + const cPath = join(dir, "program.c"); + const identityCPath = join(dir, "identity.c"); + const probePath = join(dir, "probe.c"); + const archivePath = join(dir, "program.lib.a"); + const probeOutput = join(dir, "probe"); + const cacheRoot = join(dir, "cache"); + const target = "x86_64-linux-gnu.2.36"; + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldCc = process.env["SCRIPTC_CC"]; + const oldTarget = process.env["SCRIPTC_TARGET"]; + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + process.env["SCRIPTC_CC"] = "zigcc"; + process.env["SCRIPTC_TARGET"] = target; + await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n"); + await writeFile(identityCPath, [ + "unsigned long long scriptc_build_id(void) { return 2; }", + "unsigned scriptc_abi_version(void) { return 1; }", + "", + ].join("\n")); + await compileLibArchive({ + cPath, + identityCPath, + outPath: archivePath, + cacheIdentity: TEST_CACHE_IDENTITY, + localizeSymbols: [ + "scriptc_large_program_value", + "scriptc_build_id", + "scriptc_abi_version", + ], + }); + await writeFile(probePath, [ + "int scriptc_large_program_value(void);", + "unsigned long long scriptc_build_id(void);", + "unsigned scriptc_abi_version(void);", + "int main(void) {", + " return scriptc_large_program_value() != 7 || scriptc_build_id() != 2 || scriptc_abi_version() != 1;", + "}", + "", + ].join("\n")); + execFileSync(zigExecutable!, [ + "cc", "-target", target, probePath, archivePath, "-lm", "-o", probeOutput, + ]); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldCc === undefined) delete process.env["SCRIPTC_CC"]; + else process.env["SCRIPTC_CC"] = oldCc; + if (oldTarget === undefined) delete process.env["SCRIPTC_TARGET"]; + else process.env["SCRIPTC_TARGET"] = oldTarget; + } + }, +); diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 5c027dfa..c67d000e 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -1844,8 +1844,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise driver, arArgv, buildDir, - programObject, - [...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects], + [programObject, ...(identityObject === null ? [] : [identityObject])], + [...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem, ), @@ -1941,7 +1941,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise * shared by design. Windows embedders additionally link advapi32, iphlpapi, * and ws2_32. * - * Member selection matters: a classic archive's unused members (and their + * The generated program and optional identity objects are mandatory roots; + * support-member selection still matters. A classic archive's unused members (and their * undefined references to units library mode excludes, like the * fs-promises unit's fiber symbols) never reach an embedder's link. A * blind merge of every object would carry those references into the one @@ -1950,7 +1951,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise * members the program object transitively needs (the COFF arm implements * the same member semantics in process). * - * Mach-O — one host-ld64 invocation: -r merges program + needed members, + * Mach-O — one host-ld64 invocation: -r merges roots + needed members, * -exported_symbols_list demotes every unlisted global to * private extern, and -r without -keep_private_externs writes * private externs out as non-external symbols. Apple ASan's @@ -1985,7 +1986,7 @@ async function localizeLibraryObjects( driver: CcDriver, arArgv: readonly string[], buildDir: string, - programObject: string, + rootObjects: readonly string[], supportObjects: readonly string[], keepSymbols: readonly string[], stem: string, @@ -2008,14 +2009,15 @@ async function localizeLibraryObjects( if (platform === "win32") { // COFF has no relocatable-link tool to stage through; the member // selection and combine+demote happen in process over the object bytes. - const [program, ...support] = await Promise.all( - [programObject, ...supportObjects].map((path) => readFile(path)), - ); + const [roots, support] = await Promise.all([ + Promise.all(rootObjects.map((path) => readFile(path))), + Promise.all(supportObjects.map((path) => readFile(path))), + ]); try { await writeFile( combined, - mergeAndLocalizeCoffObjects(program!, support, new Set(keepSymbols), { - program: basename(programObject), + mergeAndLocalizeCoffObjects(roots, support, new Set(keepSymbols), { + roots: rootObjects.map((path) => basename(path)), support: supportObjects.map((path) => basename(path)), }), ); @@ -2029,10 +2031,10 @@ async function localizeLibraryObjects( await run([arArgv[0] ?? "ar", ...arArgv.slice(1), "rcs", staging, ...supportObjects]); if (platform === "darwin") { await writeFile(keepFile, keepSymbols.map((s) => `_${s}\n`).join("")); - await run(["ld", "-r", programObject, staging, "-o", combined, "-exported_symbols_list", keepFile]); + await run(["ld", "-r", ...rootObjects, staging, "-o", combined, "-exported_symbols_list", keepFile]); } else if (platform === "linux" && driver.target === null) { await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join("")); - await run(["ld", "-r", "--force-group-allocation", programObject, staging, "-o", combined]); + await run(["ld", "-r", "--force-group-allocation", ...rootObjects, staging, "-o", combined]); await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]); } else if (platform === "linux") { // Cross ELF: the cross driver's own lld performs the relocatable merge @@ -2046,7 +2048,7 @@ async function localizeLibraryObjects( ...driver.argv.slice(1), "-target", driver.zigTarget ?? driver.target!, "-nostdlib", - "-r", programObject, staging, + "-r", ...rootObjects, staging, "-o", combined, ]); try { diff --git a/packages/compiler/src/backend/object-localize.test.ts b/packages/compiler/src/backend/object-localize.test.ts index 7ec74fd8..e5321bee 100644 --- a/packages/compiler/src/backend/object-localize.test.ts +++ b/packages/compiler/src/backend/object-localize.test.ts @@ -431,7 +431,7 @@ describe("mergeAndLocalizeCoffObjects", () => { ], ); const merged = readCoff( - mergeAndLocalizeCoffObjects(program, [needed, unneeded], new Set(["keep_me"])), + mergeAndLocalizeCoffObjects([program], [needed, unneeded], new Set(["keep_me"])), ); const byName = new Map(merged.symbols.map((s) => [s.name, s])); expect(byName.get("keep_me")!.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL); @@ -446,6 +446,47 @@ describe("mergeAndLocalizeCoffObjects", () => { expect(merged.sections[0]!.relocs[0]!.sym).toBe(byName.get("helper")!.index); }); + test("includes every root even when no other object references its exports", () => { + const program = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "keep_me", section: 1 }, + ], + ); + const identity = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "build_id", section: 1 }, + { name: "abi_version", section: 1, value: 4 }, + ], + ); + const unneeded = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "lonely", section: 1 }, + { name: "excluded_unit_ref", section: 0 }, + ], + ); + + const merged = readCoff( + mergeAndLocalizeCoffObjects( + [program, identity], + [unneeded], + new Set(["keep_me", "build_id", "abi_version"]), + ), + ); + const byName = new Map(merged.symbols.map((symbol) => [symbol.name, symbol])); + for (const name of ["keep_me", "build_id", "abi_version"]) { + expect(byName.get(name)?.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL); + expect(byName.get(name)?.section).toBeGreaterThan(0); + } + expect(byName.has("lonely")).toBe(false); + expect(byName.has("excluded_unit_ref")).toBe(false); + }); + test("does not pull an alternate definition when a selected object already satisfies the reference", () => { // program defines foo and needs bar; the first support member defines // bar and calls back into foo. A real archive link stops there. The @@ -477,7 +518,7 @@ describe("mergeAndLocalizeCoffObjects", () => { ); const merged = readCoff( - mergeAndLocalizeCoffObjects(program, [needed, alternate], new Set()), + mergeAndLocalizeCoffObjects([program], [needed, alternate], new Set()), ); const byName = new Map(merged.symbols.map((s) => [s.name, s])); expect(byName.has("alternate_only")).toBe(false); @@ -528,7 +569,7 @@ describe("mergeAndLocalizeCoffObjects", () => { { name: ".refptr.shared", section: 2 }, ], ); - const merged = readCoff(mergeAndLocalizeCoffObjects(program, [support], new Set(["keep_me"]))); + const merged = readCoff(mergeAndLocalizeCoffObjects([program], [support], new Set(["keep_me"]))); // One survivor section carries the stub; no COMDAT flag remains anywhere. const rdata = merged.sections.filter((s) => s.name === ".rdata"); expect(rdata.length).toBe(1); @@ -578,7 +619,7 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT LARGEST retains the largest selected definition", () => { const pair = comdatPair(6, new Uint8Array([1, 2, 3, 4]), new Uint8Array([5, 6, 7, 8, 9])); - const merged = readCoff(mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(["entry"]))); + const merged = readCoff(mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(["entry"]))); const rdata = merged.sections.filter((section) => section.name === ".rdata"); expect(rdata).toHaveLength(1); expect([...rdata[0]!.data]).toEqual([5, 6, 7, 8, 9]); @@ -620,7 +661,7 @@ describe("mergeAndLocalizeCoffObjects", () => { }; const merged = readCoff( mergeAndLocalizeCoffObjects( - pair.program, + [pair.program], [withAssociate(1), withAssociate(2)], new Set(["entry"]), ), @@ -633,8 +674,8 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT SAME_SIZE refuses definitions with different sizes", () => { const pair = comdatPair(3, new Uint8Array(4), new Uint8Array(8)); expect(() => - mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), { - program: "program.o", + mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), { + roots: ["program.o"], support: ["one.o", "two.o"], }), ).toThrow(/SAME_SIZE mismatch.*one\.o.*two\.o/); @@ -643,8 +684,8 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT EXACT_MATCH refuses equal-size definitions with different contents", () => { const pair = comdatPair(4, new Uint8Array([1, 2, 3, 4]), new Uint8Array([1, 2, 3, 5])); expect(() => - mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), { - program: "program.o", + mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), { + roots: ["program.o"], support: ["one.o", "two.o"], }), ).toThrow(/EXACT_MATCH mismatch.*one\.o.*two\.o/); @@ -653,7 +694,7 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT duplicates refuse conflicting selection kinds", () => { const one = comdatPair(2, new Uint8Array(4), new Uint8Array(4)); const two = comdatPair(3, new Uint8Array(4), new Uint8Array(4)); - expect(() => mergeAndLocalizeCoffObjects(one.program, [one.support[0]!, two.support[1]!], new Set())) + expect(() => mergeAndLocalizeCoffObjects([one.program], [one.support[0]!, two.support[1]!], new Set())) .toThrow(/conflicting COMDAT selections/); }); @@ -676,8 +717,8 @@ describe("mergeAndLocalizeCoffObjects", () => { ], ); expect(() => - mergeAndLocalizeCoffObjects(one, [two], new Set(["keep_me"]), { - program: "one.o", + mergeAndLocalizeCoffObjects([one], [two], new Set(["keep_me"]), { + roots: ["one.o"], support: ["two.o"], }), ).toThrow(/duplicate external symbol dup.*one\.o.*two\.o/); @@ -686,6 +727,6 @@ describe("mergeAndLocalizeCoffObjects", () => { test("refuses non-AMD64 machines", () => { const object = buildCoff([text()], [{ name: "x", section: 1 }]); new DataView(object.buffer, object.byteOffset).setUint16(0, 0xaa64, true); - expect(() => mergeAndLocalizeCoffObjects(object, [], new Set())).toThrow(/machine/); + expect(() => mergeAndLocalizeCoffObjects([object], [], new Set())).toThrow(/machine/); }); }); diff --git a/packages/compiler/src/backend/object-localize.ts b/packages/compiler/src/backend/object-localize.ts index 979bac00..5c8bde8b 100644 --- a/packages/compiler/src/backend/object-localize.ts +++ b/packages/compiler/src/backend/object-localize.ts @@ -19,13 +19,13 @@ * another archive's copy at the embedder's link. * * mergeAndLocalizeCoffObjects - * the COFF combine+demote in one pass: pull - * support objects on undefined-symbol demand (the - * staging-archive member semantics `ld -r` gives - * the other formats), concatenate the selected - * objects' sections, resolve cross-object symbol - * references by index, then demote every defined - * external outside the keep set to a static + * the COFF combine+demote in one pass: include every + * root object, pull support objects on undefined- + * symbol demand (the staging-archive member semantics + * `ld -r` gives the other formats), concatenate the + * selected objects' sections, resolve cross-object + * symbol references by index, then demote every + * defined external outside the keep set to a static * symbol. * * Shared demotion rule (GNU objcopy --keep-global-symbols semantics): @@ -544,20 +544,21 @@ function coffSectionContentsEqual(a: CoffSection, b: CoffSection): boolean { return true; } -/** Combine a program object with the support objects it (transitively) - * needs into ONE COFF object, then demote every defined external outside - * `keep` to a static symbol. Support objects join on undefined-symbol +/** Combine mandatory root objects with the support objects they + * (transitively) need into ONE COFF object, then demote every defined external + * outside `keep` to a static symbol. Support objects join on undefined-symbol * demand — the staging-archive member semantics the other formats get from * `ld -r` — so an unused member's undefined references never reach the * embedder's link. x86_64 only (the one COFF target scriptc produces). */ export function mergeAndLocalizeCoffObjects( - program: Uint8Array, + roots: readonly Uint8Array[], support: readonly Uint8Array[], keep: ReadonlySet, - labels?: { program?: string; support?: readonly string[] }, + labels?: { roots?: readonly string[]; support?: readonly string[] }, ): Uint8Array { + if (roots.length === 0) fail("COFF localization requires at least one root object"); const objects = [ - parseCoff(program, labels?.program ?? "program object"), + ...roots.map((bytes, i) => parseCoff(bytes, labels?.roots?.[i] ?? `root object ${i}`)), ...support.map((bytes, i) => parseCoff(bytes, labels?.support?.[i] ?? `support object ${i}`)), ]; for (const object of objects) { @@ -571,7 +572,7 @@ export function mergeAndLocalizeCoffObjects( // list order wins the pull, matching `ar` member order. const definers = new Map(); objects.forEach((object, index) => { - if (index === 0) return; + if (index < roots.length) return; for (const sym of object.symbols) { if (!coffDefines(sym)) continue; const list = definers.get(sym.name); @@ -579,7 +580,7 @@ export function mergeAndLocalizeCoffObjects( else list.push(index); } }); - const included: boolean[] = objects.map((_, i) => i === 0); + const included: boolean[] = objects.map((_, i) => i < roots.length); // Archive extraction consults the linker's CURRENT symbol state: an // undefined in a newly pulled member is already satisfied when the // program object (or an earlier member) defines it. Record every @@ -593,8 +594,8 @@ export function mergeAndLocalizeCoffObjects( if (coffDefines(sym)) selectedDefinitions.add(sym.name); } }; - addDefinitions(objects[0]!); - const queue = [0]; + for (let i = 0; i < roots.length; i++) addDefinitions(objects[i]!); + const queue = roots.map((_, i) => i); while (queue.length > 0) { const object = objects[queue.shift()!]!; for (const sym of object.symbols) { From 8f9fcee6267f6ff0aede4de3bea01c65f4caa08e Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 19 Aug 2026 18:08:03 -0500 Subject: [PATCH 3/5] Keep library identity sources private --- packages/cli/test/library-output.test.ts | 97 +++++++++++++++++++ .../compiler/src/backend/cc-cache.test.ts | 19 ++-- packages/compiler/src/backend/cc.ts | 37 +++---- packages/compiler/src/index.ts | 16 ++- packages/compiler/src/ir/nodes.ts | 6 +- packages/compiler/src/library/early-cache.ts | 1 - 6 files changed, 130 insertions(+), 46 deletions(-) create mode 100644 packages/cli/test/library-output.test.ts diff --git a/packages/cli/test/library-output.test.ts b/packages/cli/test/library-output.test.ts new file mode 100644 index 00000000..8f1ad76b --- /dev/null +++ b/packages/cli/test/library-output.test.ts @@ -0,0 +1,97 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); +const repoRoot = join(import.meta.dirname, "../../.."); +const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); +const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); + +test("library identity source stays private and cannot overwrite a sidecar", async () => { + const tempRoot = process.platform === "win32" ? tmpdir() : "/tmp"; + const dir = await mkdtemp(join(tempRoot, "scriptc-cli-library-output-")); + const cacheRoot = join(dir, "cache"); + const outDir = join(dir, ".scriptc"); + const profilePath = join(dir, "profile.json"); + try { + await mkdir(cacheRoot, { mode: 0o700 }); + await writeFile(join(dir, "lib.ts"), [ + "export interface Model { value: number; }", + "export type Msg = { kind: \"noop\" } | { kind: \"set\"; value: number };", + "export function init(): Model { return { value: 1 }; }", + "export function update(model: Model, msg: Msg): Model {", + " return msg.kind === \"set\" ? { value: msg.value } : model;", + "}", + "export function boot(): number { return init().value; }", + "", + ].join("\n")); + + const writeProfile = (sidecarPath: string): Promise => writeFile( + profilePath, + `${JSON.stringify({ + profile_format: 1, + name: "cli-library-output", + entry: "lib.ts", + emission: "c", + abi: { + prefix: "clo_", + init_symbol: "clo_init", + sink_register_symbol: "clo_set_panic_sink", + collect_symbol: null, + result_reset_symbol: null, + }, + exports: [{ export: "boot", symbol: "clo_boot", params: [], returns: "f64" }], + sidecar: { + path: sidecarPath, + wire_version: 1, + abi_version: 1, + snapshot_format: 1, + build_id_symbol: "clo_build_id", + abi_version_symbol: "clo_abi_version", + model: "Model", + msg: "Msg", + }, + }, null, 2)}\n`, + ); + const runBuild = async (): Promise => { + await execFileAsync( + process.execPath, + ["--import", tsxLoader, cliEntry, "build", "--lib", "--profile", profilePath, "--no-keep-c"], + { + env: { + ...process.env, + TMPDIR: tempRoot, + SCRIPTC_CACHE_DIR: cacheRoot, + }, + maxBuffer: 1024 * 1024, + }, + ); + }; + + // --no-keep-c removes the public program TU, and the identity source is + // invocation-private rather than a second caller-visible C artifact. + await writeProfile("contract.json"); + await runBuild(); + expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); + + // This name collided with the former fixed `.lib.identity.c` + // output. Repeat to exercise the exact early-cache-hit ordering that used + // to restore JSON and then overwrite it with generated C. + await rm(outDir, { recursive: true, force: true }); + await writeProfile("lib.lib.identity.c"); + await runBuild(); + await runBuild(); + const sidecar = JSON.parse(await readFile(join(outDir, "lib.lib.identity.c"), "utf8")) as { + build_id?: unknown; + }; + expect(sidecar.build_id).toMatch(/^[0-9a-f]{16}$/); + expect((await readdir(outDir)).sort()).toEqual(["lib.lib.a", "lib.lib.identity.c"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index ca6701cc..6644539b 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -2942,7 +2942,6 @@ test("library identity edits reuse the cached large program object", async () => scratch.push(dir); const cacheRoot = join(dir, "cache"); const cPath = join(dir, "program.c"); - const identityCPath = join(dir, "identity.c"); const outPath = join(dir, "program.lib.a"); const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; @@ -2951,10 +2950,9 @@ test("library identity edits reuse the cached large program object", async () => delete process.env["SCRIPTC_NO_CACHE"]; await mkdir(cacheRoot, { mode: 0o700 }); await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n"); - await writeFile(identityCPath, "unsigned long long scriptc_build_id(void) { return 1; }\n"); await compileLibArchive({ cPath, - identityCPath, + identityCSource: "unsigned long long scriptc_build_id(void) { return 1; }\n", outPath, cacheIdentity: TEST_CACHE_IDENTITY, }); @@ -2966,10 +2964,9 @@ test("library identity edits reuse the cached large program object", async () => const old = new Date("2000-01-01T00:00:00.000Z"); await utimes(objectPath, old, old); - await writeFile(identityCPath, "unsigned long long scriptc_build_id(void) { return 2; }\n"); await compileLibArchive({ cPath, - identityCPath, + identityCSource: "unsigned long long scriptc_build_id(void) { return 2; }\n", outPath, cacheIdentity: TEST_CACHE_IDENTITY, }); @@ -2998,7 +2995,6 @@ test.skipIf(process.platform === "win32" || zigExecutable === undefined)( const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-localized-identity-")); scratch.push(dir); const cPath = join(dir, "program.c"); - const identityCPath = join(dir, "identity.c"); const probePath = join(dir, "probe.c"); const archivePath = join(dir, "program.lib.a"); const probeOutput = join(dir, "probe"); @@ -3014,14 +3010,13 @@ test.skipIf(process.platform === "win32" || zigExecutable === undefined)( process.env["SCRIPTC_CC"] = "zigcc"; process.env["SCRIPTC_TARGET"] = target; await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n"); - await writeFile(identityCPath, [ - "unsigned long long scriptc_build_id(void) { return 2; }", - "unsigned scriptc_abi_version(void) { return 1; }", - "", - ].join("\n")); await compileLibArchive({ cPath, - identityCPath, + identityCSource: [ + "unsigned long long scriptc_build_id(void) { return 2; }", + "unsigned scriptc_abi_version(void) { return 1; }", + "", + ].join("\n"), outPath: archivePath, cacheIdentity: TEST_CACHE_IDENTITY, localizeSymbols: [ diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index c67d000e..1d830cb7 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -1398,11 +1398,11 @@ const LIB_RUNTIME_SOURCES = [ export interface LibArchiveOptions { /** The program TU (.c or .ll — clang compiles either with -c). */ cPath: string; - /** Tiny generated C TU carrying volatile library identity getters. Its - * bytes join the complete archive key, but the large program-object cache - * is keyed independently so comment-only build-id changes compile only - * this file. */ - identityCPath?: string; + /** Tiny generated C source carrying volatile library identity getters. + * Its bytes join the complete archive key, but the source itself exists + * only in the invocation-private build directory and the large program- + * object cache is keyed independently. */ + identityCSource?: string; /** The archive to produce (.lib.a). */ outPath: string; /** Caller-owned identity for the generated TU's complete non-system @@ -1573,19 +1573,19 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise let runtimeHash = ""; let programDependencyHash = ""; let cachedProgramBytes: Buffer | null = null; - let cachedIdentityBytes: Buffer | null = null; + const identityBytes = opts.identityCSource === undefined + ? null + : Buffer.from(opts.identityCSource, "utf8"); if (root !== null) { try { - const [cv, fingerprint, programBytes, identityBytes] = await Promise.all([ + const [cv, fingerprint, programBytes] = await Promise.all([ ccVersionOnce(driver.argv, toolchainEnv, true), runtimeFingerprint(rtDir), readFile(opts.cPath), - opts.identityCPath === undefined ? Promise.resolve(null) : readFile(opts.identityCPath), ]); compilerVersion = cv; runtimeHash = fingerprint; cachedProgramBytes = programBytes; - cachedIdentityBytes = identityBytes; programDependencyHash = await translationUnitDependencyFingerprint( driver, cflags, @@ -1621,8 +1621,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise .update(resolve(opts.cPath)).update("\0") .update(programBytes) .update("\0identity\0") - .update(opts.identityCPath === undefined ? "" : resolve(opts.identityCPath)) - .update("\0") + .update(identityBytes === null ? "" : "").update("\0") .update(identityBytes ?? Buffer.alloc(0)) .digest("hex"); cachedArchive = join(root, "lib", key); @@ -1773,17 +1772,13 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise } } } - const identityObject = opts.identityCPath === undefined + const identityObject = identityBytes === null ? null - : await compileOne( - cachedIdentityBytes === null ? opts.identityCPath : await (async () => { - const source = join(buildDir, "identity.c"); - await writeFile(source, cachedIdentityBytes); - return source; - })(), - `${stem}.identity.o`, - cachedIdentityBytes === null ? undefined : opts.identityCPath, - ); + : await (async () => { + const source = join(buildDir, "identity.c"); + await writeFile(source, identityBytes); + return compileOne(source, `${stem}.identity.o`); + })(); let runtimeObjects: string[] | null = null; let cacheInputsStable = true; let objectImplicitVerification: Promise | null = null; diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index bc1dada1..c654eb06 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1550,22 +1550,20 @@ async function compileLibraryNative( features: EarlyLibraryNativeFeatures, ): Promise { const localizeSymbols = libraryLocalizeSymbols(profile); - let identityCPath: string | undefined; + let identityCSource: string | undefined; if (profile.sidecar !== null) { if (features.buildId === undefined) throw new Error("library identity TU has no build id"); - const stem = basename(profile.entry).replace(/\.(ts|js|mjs|cjs)$/, ""); - identityCPath = join(dirname(cPath), `${stem}.lib.identity.c`); - await writeFile(identityCPath, [ + identityCSource = [ "#include ", "#include ", `uint64_t ${profile.sidecar.buildIdSymbol}(void) { return UINT64_C(0x${features.buildId}); }`, `uint32_t ${profile.sidecar.abiVersionSymbol}(void) { return ${profile.sidecar.abiVersion}u; }`, "", - ].join("\n")); + ].join("\n"); } await compileLibArchive({ cPath, - ...(identityCPath !== undefined ? { identityCPath } : {}), + ...(identityCSource !== undefined ? { identityCSource } : {}), outPath: archivePath, cacheIdentity: "scriptc-generated-library-v1", sanitize, @@ -2021,9 +2019,9 @@ async function compileLibraryTracked( // The ask-2 contract sidecar rides the same invocation. Identity first // (schema §2's worked build_id definition over compiler version, profile // bytes, and the sorted canonical module graph; source_hash per the - // profile's "module-graph" contract) — the u64 lands on the IR so both - // backends emit the identity getters from the ONE value the sidecar - // records (V12's coherence by construction), then the projection into + // profile's "module-graph" contract) — the u64 lands on the IR so native + // archive assembly emits the identity getters from the ONE value the + // sidecar records (V12's coherence by construction), then the projection into // the schema (declaration orders from the AST) and the V1–V14 // self-check before anything is written. let sidecarJson: string | null = null; diff --git a/packages/compiler/src/ir/nodes.ts b/packages/compiler/src/ir/nodes.ts index 2b3a6794..ca48a7ac 100644 --- a/packages/compiler/src/ir/nodes.ts +++ b/packages/compiler/src/ir/nodes.ts @@ -984,9 +984,9 @@ export interface IrLibSection { identity?: IrLibIdentity; } -/** The profile-declared identity getters' facts, landed on the IR so both - * backends emit the same constants the sidecar records (V12's identity - * coherence is one-value-two-writes by construction). */ +/** The profile-declared identity getters' facts, landed on the IR so native + * archive assembly emits the same constants the sidecar records (V12's + * identity coherence is one-value-two-writes by construction). */ export interface IrLibIdentity { buildIdSymbol: string; abiVersionSymbol: string; diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index d05c8ac5..4a50e443 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -214,7 +214,6 @@ function frontendOutputExclusions( const outputArtifacts = [ paths.cPath, paths.staleCPath, - join(options.outDir, `${basename(options.entryPath).replace(/\.(ts|js|mjs|cjs)$/, "")}.lib.identity.c`), ...(options.emitIr ? [paths.irPath] : []), archiveOutputPath(options), ...(sidecarPath === undefined ? [] : [sidecarPath]), From f5e818e7a4b4163751702aa44b8e38c7f58307d0 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 19 Aug 2026 21:50:13 -0500 Subject: [PATCH 4/5] Preserve library identity in public emission --- .../compiler/src/backend/emission/emitter.ts | 31 ++++++++++- packages/compiler/src/backend/llvm/emitter.ts | 25 +++++++++ packages/compiler/src/index.ts | 17 ++++-- .../test/library-identity-emission.test.ts | 54 +++++++++++++++++++ 4 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 packages/compiler/test/library-identity-emission.test.ts diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index d463c01e..30006d92 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -63,8 +63,18 @@ import { emitNpmEmbedding, islandAdapter, islandTypedAdapter } from "./emit-isla import { emitFunction, emitBlock, emitStmts, emitStmt, emitTryCatch, emitSwitch, mergeBrace, emitBranchInto, emitCondition } from "./emit-stmts.js"; import { emitExpr } from "./emit-exprs.js"; -export function emitModule(mod: IrModule, sourceText?: string): string { - return new CEmitter(mod, sourceText).emit(); +export interface CEmitOptions { + /** Library archive assembly may move the volatile identity getters into a + * separate translation unit. Public/direct emission keeps them by default. */ + emitLibraryIdentity?: boolean; +} + +export function emitModule( + mod: IrModule, + sourceText?: string, + options: CEmitOptions = {}, +): string { + return new CEmitter(mod, sourceText, options).emit(); } // Box construction moved onto CEmitter (boxNewC method): obj-kind boxes now @@ -414,6 +424,7 @@ export class CEmitter { constructor( readonly mod: IrModule, sourceText?: string, + private readonly options: CEmitOptions = {}, ) { this.ffiCallbackAdapters = allocateFfiCallbackAdapters(mod.ffiImports ?? []); this.ffiHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []); @@ -1141,6 +1152,22 @@ export class CEmitter { } out.push(` return -1;`, `}`, ``); } + if (lib.identity !== undefined && this.options.emitLibraryIdentity !== false) { + // Profile-declared identity getters (the ask-2 sidecar's boot-time + // pairing fence): pure data returns with NO entry prologue — exempt + // from the poisoned guard and every runtime touch (ratified), so a + // host can read them before init and after a trap. + out.push( + `uint64_t ${lib.identity.buildIdSymbol}(void) {`, + ` return UINT64_C(0x${lib.identity.buildId});`, + `}`, + ``, + `uint32_t ${lib.identity.abiVersionSymbol}(void) {`, + ` return ${lib.identity.abiVersion}u;`, + `}`, + ``, + ); + } if (lib.resultResetSymbol !== null) { out.push( `void ${lib.resultResetSymbol}(void) {`, diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 2b7ab68c..25f8e726 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -190,6 +190,9 @@ export interface LlvmTargetOptions { pointerBits?: 32 | 64; /** Select the WASI libc entry-point convention. */ wasi?: boolean; + /** Library archive assembly may move the volatile identity getters into a + * separate translation unit. Public/direct emission keeps them by default. */ + emitLibraryIdentity?: boolean; } export function emitLlvmModule(mod: IrModule, options: LlvmTargetOptions = {}): string { @@ -974,6 +977,7 @@ class LlEmitter { readonly sizeType: "i32" | "i64"; readonly cycleColorOffset: number; private readonly wasi: boolean; + private readonly emitLibraryIdentity: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the C emitter's determinism discipline. */ private readonly literals = new Map(); @@ -1141,6 +1145,7 @@ class LlEmitter { constructor(private readonly mod: IrModule, options: LlvmTargetOptions) { this.sizeType = options.pointerBits === 32 ? "i32" : "i64"; this.wasi = options.wasi === true; + this.emitLibraryIdentity = options.emitLibraryIdentity !== false; // ScrCycHdr is { ptr trace; ptr free; i32 color; i16 buffered; // i16 gen; size_t buf_index }. The object follows it, so color is 12 // bytes behind a wasm32 object and 16 bytes behind a 64-bit object. @@ -2322,6 +2327,26 @@ class LlEmitter { }); out.push(`miss:`, ` ret i32 -1`, `}`, ``); } + if (lib.identity !== undefined && this.emitLibraryIdentity) { + // Profile-declared identity getters (the ask-2 sidecar's boot-time + // pairing fence): pure data returns with NO entry prologue — exempt + // from the poisoned guard and every runtime touch (ratified), so a + // host can read them before init and after a trap. The u64 rides + // i64 two's-complement (LLVM integer constants are signed). + const buildId = BigInt.asIntN(64, BigInt(`0x${lib.identity.buildId}`)).toString(); + out.push( + `define i64 @${lib.identity.buildIdSymbol}() ${FN_ATTRS} { ; identity getter build_id 0x${lib.identity.buildId}`, + `entry:`, + ` ret i64 ${buildId}`, + `}`, + ``, + `define i32 @${lib.identity.abiVersionSymbol}() ${FN_ATTRS} { ; identity getter abi_version`, + `entry:`, + ` ret i32 ${lib.identity.abiVersion}`, + `}`, + ``, + ); + } if (lib.resultResetSymbol !== null) { out.push( `define void @${lib.resultResetSymbol}() ${FN_ATTRS} {`, diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index c654eb06..9db9f033 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -41,7 +41,7 @@ export const VERSION = "0.0.1"; export { compileC, runtimeSrcDir, type CcOptions } from "./backend/cc.js"; export { ANDROID_MIN_API, IPHONEOS_MIN_VERSION, isAndroidTarget, isIosTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal } from "./backend/cc.js"; -export { emitModule } from "./backend/emission/emitter.js"; +export { emitModule, type CEmitOptions } from "./backend/emission/emitter.js"; export type { ScrDiagnostic } from "./diagnostics/diagnostic.js"; export { renderAll, renderDiagnostic } from "./diagnostics/render.js"; export { renderCoverage, type CoverageInput } from "./coverage/report.js"; @@ -1620,13 +1620,18 @@ async function emitSemanticLibraryHit( const stem = basename(profile.entry).replace(/\.(ts|js|mjs|cjs)$/, ""); let cPath: string; if (profile.emission === "llvm") { - const ll = emitLlvmModule(mod); + const ll = emitLlvmModule(mod, { emitLibraryIdentity: false }); cPath = join(opts.outDir, `${stem}.lib.ll`); await writeFile(cPath, ll); timing("semantic-llvm-emit", { output_bytes: Buffer.byteLength(ll) }); } else { cPath = join(opts.outDir, `${stem}.lib.c`); - await writeFile(cPath, emitModule(mod, hit.sourceTexts.get(profile.entry))); + await writeFile( + cPath, + emitModule(mod, hit.sourceTexts.get(profile.entry), { + emitLibraryIdentity: false, + }), + ); timing("semantic-c-emit"); } await rm(join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "c" : "ll"}`), { force: true }); @@ -2083,7 +2088,7 @@ async function compileLibraryTracked( let cPath: string; if (profile.emission === "llvm") { try { - const ll = emitLlvmModule(mod); + const ll = emitLlvmModule(mod, { emitLibraryIdentity: false }); timing("llvm-emit", { output_bytes: Buffer.byteLength(ll) }); cPath = join(opts.outDir, `${stem}.lib.ll`); await writeFile(cPath, ll); @@ -2095,7 +2100,9 @@ async function compileLibraryTracked( } } else { cPath = join(opts.outDir, `${stem}.lib.c`); - await writeFile(cPath, emitModule(mod, entryText)); + await writeFile(cPath, emitModule(mod, entryText, { + emitLibraryIdentity: false, + })); } await rm(join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "c" : "ll"}`), { force: true }); diff --git a/packages/compiler/test/library-identity-emission.test.ts b/packages/compiler/test/library-identity-emission.test.ts new file mode 100644 index 00000000..8e2e1317 --- /dev/null +++ b/packages/compiler/test/library-identity-emission.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "vitest"; +import { emitModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; +import type { IrModule } from "../src/ir/nodes.js"; +import { fibModule } from "./fixtures/fib-ir.js"; + +const libraryModule = (): IrModule => ({ + ...fibModule, + lib: { + profileName: "identity-emission", + prefix: "ie_", + initSymbol: "ie_init", + sinkRegisterSymbol: "ie_set_sink", + collectSymbol: null, + resultResetSymbol: null, + threadInstances: false, + exports: [], + trapOverlays: [], + identity: { + buildIdSymbol: "ie_build_id", + abiVersionSymbol: "ie_abi_version", + buildId: "fedcba9876543210", + abiVersion: 7, + }, + }, +}); + +describe("library identity emission", () => { + test("direct C emission retains the IR-declared identity getters", () => { + const emitted = emitModule(libraryModule()); + expect(emitted).toContain("uint64_t ie_build_id(void)"); + expect(emitted).toContain("return UINT64_C(0xfedcba9876543210);"); + expect(emitted).toContain("uint32_t ie_abi_version(void)"); + expect(emitted).toContain("return 7u;"); + }); + + test("direct LLVM emission retains the IR-declared identity getters", () => { + const emitted = emitLlvmModule(libraryModule()); + expect(emitted).toContain("define i64 @ie_build_id()"); + expect(emitted).toContain("identity getter build_id 0xfedcba9876543210"); + expect(emitted).toContain("define i32 @ie_abi_version()"); + expect(emitted).toContain("ret i32 7"); + }); + + test("archive program-TU mode suppresses identity definitions in both backends", () => { + const mod = libraryModule(); + const c = emitModule(mod, undefined, { emitLibraryIdentity: false }); + const llvm = emitLlvmModule(mod, { emitLibraryIdentity: false }); + expect(c).not.toContain("ie_build_id"); + expect(c).not.toContain("ie_abi_version"); + expect(llvm).not.toContain("@ie_build_id"); + expect(llvm).not.toContain("@ie_abi_version"); + }); +}); From a4c544ab76be3abd140435b8aeb95d41dd1d508f Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Wed, 19 Aug 2026 23:21:31 -0500 Subject: [PATCH 5/5] Preserve identity in kept library units --- .../compiler/src/backend/cc-cache.test.ts | 12 +++++- packages/compiler/src/backend/cc.ts | 10 ++++- .../compiler/src/backend/emission/emitter.ts | 3 ++ .../compiler/src/backend/library-identity.ts | 38 +++++++++++++++++++ packages/compiler/src/backend/llvm/emitter.ts | 3 ++ packages/compiler/src/index.ts | 22 ++++++----- .../test/library-identity-emission.test.ts | 11 ++++++ tests/harness/library-contract.test.ts | 20 +++++++++- 8 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 packages/compiler/src/backend/library-identity.ts diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index 6644539b..3d97d005 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -2949,9 +2949,14 @@ test("library identity edits reuse the cached large program object", async () => process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; delete process.env["SCRIPTC_NO_CACHE"]; await mkdir(cacheRoot, { mode: 0o700 }); - await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n"); + const programSource = "int scriptc_large_program_value(void) { return 7; }\n"; + await writeFile( + cPath, + `${programSource}unsigned long long scriptc_build_id(void) { return 1; }\n`, + ); await compileLibArchive({ cPath, + programSource, identityCSource: "unsigned long long scriptc_build_id(void) { return 1; }\n", outPath, cacheIdentity: TEST_CACHE_IDENTITY, @@ -2964,8 +2969,13 @@ test("library identity edits reuse the cached large program object", async () => const old = new Date("2000-01-01T00:00:00.000Z"); await utimes(objectPath, old, old); + await writeFile( + cPath, + `${programSource}unsigned long long scriptc_build_id(void) { return 2; }\n`, + ); await compileLibArchive({ cPath, + programSource, identityCSource: "unsigned long long scriptc_build_id(void) { return 2; }\n", outPath, cacheIdentity: TEST_CACHE_IDENTITY, diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 1d830cb7..58f76385 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -1398,6 +1398,10 @@ const LIB_RUNTIME_SOURCES = [ export interface LibArchiveOptions { /** The program TU (.c or .ll — clang compiles either with -c). */ cPath: string; + /** Invocation-owned program source to compile under `cPath`'s public + * spelling. Library assembly uses this for the identity-free projection of + * a complete caller-visible TU; its bytes drive every native cache key. */ + programSource?: string; /** Tiny generated C source carrying volatile library identity getters. * Its bytes join the complete archive key, but the source itself exists * only in the invocation-private build directory and the large program- @@ -1572,7 +1576,9 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise let archiverVersion = ""; let runtimeHash = ""; let programDependencyHash = ""; - let cachedProgramBytes: Buffer | null = null; + let cachedProgramBytes = opts.programSource === undefined + ? null + : Buffer.from(opts.programSource, "utf8"); const identityBytes = opts.identityCSource === undefined ? null : Buffer.from(opts.identityCSource, "utf8"); @@ -1581,7 +1587,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise const [cv, fingerprint, programBytes] = await Promise.all([ ccVersionOnce(driver.argv, toolchainEnv, true), runtimeFingerprint(rtDir), - readFile(opts.cPath), + cachedProgramBytes === null ? readFile(opts.cPath) : Promise.resolve(cachedProgramBytes), ]); compilerVersion = cv; runtimeHash = fingerprint; diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index 30006d92..a8b8e3d7 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -62,6 +62,7 @@ import { emitAsyncScaffolding, childDataThunkFor, childExitThunkFor, childExitTh import { emitNpmEmbedding, islandAdapter, islandTypedAdapter } from "./emit-island.js"; import { emitFunction, emitBlock, emitStmts, emitStmt, emitTryCatch, emitSwitch, mergeBrace, emitBranchInto, emitCondition } from "./emit-stmts.js"; import { emitExpr } from "./emit-exprs.js"; +import { C_LIBRARY_IDENTITY_BEGIN, C_LIBRARY_IDENTITY_END } from "../library-identity.js"; export interface CEmitOptions { /** Library archive assembly may move the volatile identity getters into a @@ -1158,6 +1159,7 @@ export class CEmitter { // from the poisoned guard and every runtime touch (ratified), so a // host can read them before init and after a trap. out.push( + C_LIBRARY_IDENTITY_BEGIN, `uint64_t ${lib.identity.buildIdSymbol}(void) {`, ` return UINT64_C(0x${lib.identity.buildId});`, `}`, @@ -1166,6 +1168,7 @@ export class CEmitter { ` return ${lib.identity.abiVersion}u;`, `}`, ``, + C_LIBRARY_IDENTITY_END, ); } if (lib.resultResetSymbol !== null) { diff --git a/packages/compiler/src/backend/library-identity.ts b/packages/compiler/src/backend/library-identity.ts new file mode 100644 index 00000000..7a27d63d --- /dev/null +++ b/packages/compiler/src/backend/library-identity.ts @@ -0,0 +1,38 @@ +export type LibraryEmission = "c" | "llvm"; + +export const C_LIBRARY_IDENTITY_BEGIN = "/* scriptc-library-identity: begin */"; +export const C_LIBRARY_IDENTITY_END = "/* scriptc-library-identity: end */"; +export const LLVM_LIBRARY_IDENTITY_BEGIN = "; scriptc-library-identity: begin"; +export const LLVM_LIBRARY_IDENTITY_END = "; scriptc-library-identity: end"; + +/** Remove the generated identity region from a complete caller-visible + * library TU. Archive assembly compiles this stable projection beside the + * small volatile identity C object, while the public TU remains complete. */ +export function stripLibraryIdentity( + source: string, + emission: LibraryEmission, +): string { + const begin = emission === "c" + ? C_LIBRARY_IDENTITY_BEGIN + : LLVM_LIBRARY_IDENTITY_BEGIN; + const end = emission === "c" + ? C_LIBRARY_IDENTITY_END + : LLVM_LIBRARY_IDENTITY_END; + const start = source.indexOf(`${begin}\n`); + if (start < 0) return source; + if (source.indexOf(begin, start + begin.length) >= 0) { + throw new Error("generated library TU contains multiple identity regions"); + } + const endStart = source.indexOf(end, start + begin.length); + if (endStart < 0) throw new Error("generated library TU has an unterminated identity region"); + const endOffset = endStart + end.length; + const suffix = source[endOffset] === "\n" ? endOffset + 1 : endOffset; + let prefix = source.slice(0, start); + // An omitted final region leaves the emitter's preceding empty array entry + // as one trailing newline; a marked final region has material after that + // entry and therefore renders it as two. Restore the omitted form exactly. + if (suffix === source.length && prefix.endsWith("\n\n")) { + prefix = prefix.slice(0, -1); + } + return prefix + source.slice(suffix); +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index 25f8e726..cd3b9ad7 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -61,6 +61,7 @@ * lazy-inflate representation as the C debugging backend. */ import { deflateRawSync } from "node:zlib"; +import { LLVM_LIBRARY_IDENTITY_BEGIN, LLVM_LIBRARY_IDENTITY_END } from "../library-identity.js"; import type { IrBytesElem, IrExpr, @@ -2335,6 +2336,7 @@ class LlEmitter { // i64 two's-complement (LLVM integer constants are signed). const buildId = BigInt.asIntN(64, BigInt(`0x${lib.identity.buildId}`)).toString(); out.push( + LLVM_LIBRARY_IDENTITY_BEGIN, `define i64 @${lib.identity.buildIdSymbol}() ${FN_ATTRS} { ; identity getter build_id 0x${lib.identity.buildId}`, `entry:`, ` ret i64 ${buildId}`, @@ -2345,6 +2347,7 @@ class LlEmitter { ` ret i32 ${lib.identity.abiVersion}`, `}`, ``, + LLVM_LIBRARY_IDENTITY_END, ); } if (lib.resultResetSymbol !== null) { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 9db9f033..bb3a1445 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,8 +1,9 @@ -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js"; import { emitModule } from "./backend/emission/emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; +import { stripLibraryIdentity } from "./backend/library-identity.js"; import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./library/int-infer.js"; import { loadLibraryProfile, profileRemediation, profileTeaching, type LibraryProfile } from "./library/profile.js"; @@ -1551,8 +1552,14 @@ async function compileLibraryNative( ): Promise { const localizeSymbols = libraryLocalizeSymbols(profile); let identityCSource: string | undefined; + let programSource: string | undefined; if (profile.sidecar !== null) { if (features.buildId === undefined) throw new Error("library identity TU has no build id"); + const publicSource = await readFile(cPath, "utf8"); + programSource = stripLibraryIdentity(publicSource, profile.emission); + if (programSource === publicSource) { + throw new Error("generated public library TU has no identity region"); + } identityCSource = [ "#include ", "#include ", @@ -1563,6 +1570,7 @@ async function compileLibraryNative( } await compileLibArchive({ cPath, + ...(programSource !== undefined ? { programSource } : {}), ...(identityCSource !== undefined ? { identityCSource } : {}), outPath: archivePath, cacheIdentity: "scriptc-generated-library-v1", @@ -1620,7 +1628,7 @@ async function emitSemanticLibraryHit( const stem = basename(profile.entry).replace(/\.(ts|js|mjs|cjs)$/, ""); let cPath: string; if (profile.emission === "llvm") { - const ll = emitLlvmModule(mod, { emitLibraryIdentity: false }); + const ll = emitLlvmModule(mod); cPath = join(opts.outDir, `${stem}.lib.ll`); await writeFile(cPath, ll); timing("semantic-llvm-emit", { output_bytes: Buffer.byteLength(ll) }); @@ -1628,9 +1636,7 @@ async function emitSemanticLibraryHit( cPath = join(opts.outDir, `${stem}.lib.c`); await writeFile( cPath, - emitModule(mod, hit.sourceTexts.get(profile.entry), { - emitLibraryIdentity: false, - }), + emitModule(mod, hit.sourceTexts.get(profile.entry)), ); timing("semantic-c-emit"); } @@ -2088,7 +2094,7 @@ async function compileLibraryTracked( let cPath: string; if (profile.emission === "llvm") { try { - const ll = emitLlvmModule(mod, { emitLibraryIdentity: false }); + const ll = emitLlvmModule(mod); timing("llvm-emit", { output_bytes: Buffer.byteLength(ll) }); cPath = join(opts.outDir, `${stem}.lib.ll`); await writeFile(cPath, ll); @@ -2100,9 +2106,7 @@ async function compileLibraryTracked( } } else { cPath = join(opts.outDir, `${stem}.lib.c`); - await writeFile(cPath, emitModule(mod, entryText, { - emitLibraryIdentity: false, - })); + await writeFile(cPath, emitModule(mod, entryText)); } await rm(join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "c" : "ll"}`), { force: true }); diff --git a/packages/compiler/test/library-identity-emission.test.ts b/packages/compiler/test/library-identity-emission.test.ts index 8e2e1317..67346f21 100644 --- a/packages/compiler/test/library-identity-emission.test.ts +++ b/packages/compiler/test/library-identity-emission.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "vitest"; import { emitModule } from "../src/index.js"; import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; +import { stripLibraryIdentity } from "../src/backend/library-identity.js"; import type { IrModule } from "../src/ir/nodes.js"; import { fibModule } from "./fixtures/fib-ir.js"; @@ -50,5 +51,15 @@ describe("library identity emission", () => { expect(c).not.toContain("ie_abi_version"); expect(llvm).not.toContain("@ie_build_id"); expect(llvm).not.toContain("@ie_abi_version"); + expect(stripLibraryIdentity(emitModule(mod), "c")).toBe(c); + expect(stripLibraryIdentity(emitLlvmModule(mod), "llvm")).toBe(llvm); + + mod.lib!.resultResetSymbol = "ie_reset"; + expect(stripLibraryIdentity(emitModule(mod), "c")).toBe( + emitModule(mod, undefined, { emitLibraryIdentity: false }), + ); + expect(stripLibraryIdentity(emitLlvmModule(mod), "llvm")).toBe( + emitLlvmModule(mod, { emitLibraryIdentity: false }), + ); }); }); diff --git a/tests/harness/library-contract.test.ts b/tests/harness/library-contract.test.ts index 2eef3e22..659b5d94 100644 --- a/tests/harness/library-contract.test.ts +++ b/tests/harness/library-contract.test.ts @@ -66,7 +66,7 @@ async function buildContract( * directories is exactly what the canonical root-relative paths * guarantee, and the V13-style assertions prove it). */ root = cacheDir, -): Promise<{ outDir: string; archive: string; sidecarPath: string; doc: SidecarDoc; bytes: Buffer }> { +): Promise<{ outDir: string; archive: string; cPath: string; sidecarPath: string; doc: SidecarDoc; bytes: Buffer }> { const dir = join(fixtureRoot, fixture); const outDir = join(root, `${fixture}-${emission}${tag}`); mkdirSync(outDir, { recursive: true }); @@ -92,6 +92,7 @@ async function buildContract( return { outDir, archive: result.archivePath, + cPath: result.cPath, sidecarPath: result.sidecarPath!, doc: JSON.parse(bytes.toString("utf8")) as SidecarDoc, bytes: bytes as Buffer, @@ -112,7 +113,7 @@ function nmDefined(archive: string, prefix: string): string[] { describe.each(EMISSIONS)("contract sidecar, %s emission", (emission) => { test("anti-alphabetical declaration order, schema shape, V11/V12 identity", async () => { - const { outDir, archive, doc, bytes } = await buildContract("contract", emission); + const { outDir, archive, cPath, doc, bytes } = await buildContract("contract", emission); // The emitter's own self-check ran before writing; the test-side // validator agrees the document conforms. @@ -236,6 +237,21 @@ describe.each(EMISSIONS)("contract sidecar, %s emission", (emission) => { }); expect(nmDefined(archive, "kc_")).toEqual(doc.abi.exports.map((s) => `kc_${s}`).sort()); + // The kept/returned program TU is a complete public artifact, even though + // archive assembly privately compiles an identity-free projection beside + // its tiny volatile identity object. + const keptObject = join(outDir, "kept-program.o"); + execFileSync("clang", [ + "-std=c11", + "-DSCR_LIB", + ...(emission === "llvm" + ? ["-Wno-override-module"] + : ["-Wno-comment", "-I", join(repoRoot, "packages/runtime/src")]), + "-c", cPath, + "-o", keptObject, + ]); + expect(nmDefined(keptObject, "kc_")).toEqual(doc.abi.exports.map((s) => `kc_${s}`).sort()); + // V12 + the poisoned-guard exemption, end to end: the probe reads the // getters before init and after a trap; both reads equal the // sidecar's build_id.