diff --git a/packages/cli/README.md b/packages/cli/README.md index 0abadbf50..657ff09a0 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -24,7 +24,7 @@ $ npm install -g scriptc Requires clang on the PATH (Xcode Command Line Tools on macOS, `clang` package on Linux). -Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend. TypeScript comment-only edits can restore validated lowered IR instead, rebasing source locations and regenerating exact-source build identity before emission; directives, JSDoc-bearing JavaScript, token edits, configuration, package resolution, and newly appearing candidates still invalidate it. The native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. +Builds use a bounded persistent cache by default. Exact unchanged library builds validate their recorded TypeScript/module-resolution inputs and restore the generated C/LLVM unit before starting the frontend. TypeScript comment-only edits can restore validated lowered IR instead, rebasing source locations and regenerating exact-source build identity before emission; directives, JSDoc-bearing JavaScript, token edits, configuration, package resolution, and newly appearing candidates still invalidate it. Library identity getters live in a tiny C translation unit, so build-id-only changes reuse the large compiled program object and compile only that small member before rearchiving. The native cache then applies its independent toolchain checks. Unchanged executables and library archives skip native code generation and linking after fresh compiler metadata probes, while edited builds reuse stable runtime objects. Experimental provenance-source builds bypass the early frontend tier because their fetched-source registry is process state. FFI builds with archive/object inputs or ambient `system_libraries` relink every time but still reuse runtime objects. Mutable compiler input paths such as `CPATH` and `SDKROOT`, and compiler wrappers, bypass persistent native artifacts and objects so same-path dependency edits cannot go stale. Opaque archiver wrappers rebuild library program members and archives while retaining runtime-object reuse. Direct Clang, Apple's system Clang shim, `zig cc`, trusted platform archivers, and `zig ar` retain their applicable persistent tiers. Set `SCRIPTC_NO_CACHE=1` to bypass every cache or `SCRIPTC_CACHE_DIR` to choose its location; an existing POSIX override must already be private, otherwise caching is bypassed without changing its permissions. ## Commands diff --git a/packages/cli/test/library-output.test.ts b/packages/cli/test/library-output.test.ts new file mode 100644 index 000000000..8f1ad76bc --- /dev/null +++ b/packages/cli/test/library-output.test.ts @@ -0,0 +1,97 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { promisify } from "node:util"; +import { expect, test } from "vitest"; + +const execFileAsync = promisify(execFile); +const require = createRequire(import.meta.url); +const repoRoot = join(import.meta.dirname, "../../.."); +const cliEntry = join(repoRoot, "packages/cli/src/main.ts"); +const tsxLoader = join(dirname(require.resolve("tsx/package.json")), "dist/loader.mjs"); + +test("library identity source stays private and cannot overwrite a sidecar", async () => { + const tempRoot = process.platform === "win32" ? tmpdir() : "/tmp"; + const dir = await mkdtemp(join(tempRoot, "scriptc-cli-library-output-")); + const cacheRoot = join(dir, "cache"); + const outDir = join(dir, ".scriptc"); + const profilePath = join(dir, "profile.json"); + try { + await mkdir(cacheRoot, { mode: 0o700 }); + await writeFile(join(dir, "lib.ts"), [ + "export interface Model { value: number; }", + "export type Msg = { kind: \"noop\" } | { kind: \"set\"; value: number };", + "export function init(): Model { return { value: 1 }; }", + "export function update(model: Model, msg: Msg): Model {", + " return msg.kind === \"set\" ? { value: msg.value } : model;", + "}", + "export function boot(): number { return init().value; }", + "", + ].join("\n")); + + const writeProfile = (sidecarPath: string): Promise => writeFile( + profilePath, + `${JSON.stringify({ + profile_format: 1, + name: "cli-library-output", + entry: "lib.ts", + emission: "c", + abi: { + prefix: "clo_", + init_symbol: "clo_init", + sink_register_symbol: "clo_set_panic_sink", + collect_symbol: null, + result_reset_symbol: null, + }, + exports: [{ export: "boot", symbol: "clo_boot", params: [], returns: "f64" }], + sidecar: { + path: sidecarPath, + wire_version: 1, + abi_version: 1, + snapshot_format: 1, + build_id_symbol: "clo_build_id", + abi_version_symbol: "clo_abi_version", + model: "Model", + msg: "Msg", + }, + }, null, 2)}\n`, + ); + const runBuild = async (): Promise => { + await execFileAsync( + process.execPath, + ["--import", tsxLoader, cliEntry, "build", "--lib", "--profile", profilePath, "--no-keep-c"], + { + env: { + ...process.env, + TMPDIR: tempRoot, + SCRIPTC_CACHE_DIR: cacheRoot, + }, + maxBuffer: 1024 * 1024, + }, + ); + }; + + // --no-keep-c removes the public program TU, and the identity source is + // invocation-private rather than a second caller-visible C artifact. + await writeProfile("contract.json"); + await runBuild(); + expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); + + // This name collided with the former fixed `.lib.identity.c` + // output. Repeat to exercise the exact early-cache-hit ordering that used + // to restore JSON and then overwrite it with generated C. + await rm(outDir, { recursive: true, force: true }); + await writeProfile("lib.lib.identity.c"); + await runBuild(); + await runBuild(); + const sidecar = JSON.parse(await readFile(join(outDir, "lib.lib.identity.c"), "utf8")) as { + build_id?: unknown; + }; + expect(sidecar.build_id).toMatch(/^[0-9a-f]{16}$/); + expect((await readdir(outDir)).sort()).toEqual(["lib.lib.a", "lib.lib.identity.c"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/compiler/src/backend/cc-cache.test.ts b/packages/compiler/src/backend/cc-cache.test.ts index 241a126ed..3d97d0057 100644 --- a/packages/compiler/src/backend/cc-cache.test.ts +++ b/packages/compiler/src/backend/cc-cache.test.ts @@ -2936,3 +2936,126 @@ test("library archives hit by content, invalidate on edits, and reuse runtime ob else process.env["PATH"] = oldPath; } }); + +test("library identity edits reuse the cached large program object", async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-program-object-")); + scratch.push(dir); + const cacheRoot = join(dir, "cache"); + const cPath = join(dir, "program.c"); + const outPath = join(dir, "program.lib.a"); + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + await mkdir(cacheRoot, { mode: 0o700 }); + const programSource = "int scriptc_large_program_value(void) { return 7; }\n"; + await writeFile( + cPath, + `${programSource}unsigned long long scriptc_build_id(void) { return 1; }\n`, + ); + await compileLibArchive({ + cPath, + programSource, + identityCSource: "unsigned long long scriptc_build_id(void) { return 1; }\n", + outPath, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + const [objectName] = (await readdir(join(cacheRoot, "program-obj"))) + .filter((name) => !name.endsWith(".sha256")); + expect(objectName).toBeDefined(); + const objectPath = join(cacheRoot, "program-obj", objectName!); + const objectDigest = await readFile(`${objectPath}.sha256`, "utf8"); + const old = new Date("2000-01-01T00:00:00.000Z"); + await utimes(objectPath, old, old); + + await writeFile( + cPath, + `${programSource}unsigned long long scriptc_build_id(void) { return 2; }\n`, + ); + await compileLibArchive({ + cPath, + programSource, + identityCSource: "unsigned long long scriptc_build_id(void) { return 2; }\n", + outPath, + cacheIdentity: TEST_CACHE_IDENTITY, + }); + expect(await readFile(`${objectPath}.sha256`, "utf8")).toBe(objectDigest); + expect((await stat(objectPath)).mtimeMs).toBeGreaterThan(old.getTime()); + expect((await readdir(join(cacheRoot, "program-obj"))).filter((name) => !name.endsWith(".sha256"))).toEqual([objectName]); + const probeSource = join(dir, "probe.c"); + const probe = join(dir, "probe"); + await writeFile( + probeSource, + "#include \nint scriptc_large_program_value(void);\nunsigned long long scriptc_build_id(void);\nint main(void) { printf(\"%d %llu\\n\", scriptc_large_program_value(), scriptc_build_id()); }\n", + ); + execFileSync("clang", [probeSource, outPath, "-lm", "-o", probe]); + expect(execFileSync(probe, { encoding: "utf8" })).toBe("7 2\n"); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + } +}); + +test.skipIf(process.platform === "win32" || zigExecutable === undefined)( + "cross-ELF localized archives retain unreferenced identity roots", + async () => { + const dir = await mkdtemp(join(tmpdir(), "scriptc-lib-localized-identity-")); + scratch.push(dir); + const cPath = join(dir, "program.c"); + const probePath = join(dir, "probe.c"); + const archivePath = join(dir, "program.lib.a"); + const probeOutput = join(dir, "probe"); + const cacheRoot = join(dir, "cache"); + const target = "x86_64-linux-gnu.2.36"; + const oldCacheDir = process.env["SCRIPTC_CACHE_DIR"]; + const oldNoCache = process.env["SCRIPTC_NO_CACHE"]; + const oldCc = process.env["SCRIPTC_CC"]; + const oldTarget = process.env["SCRIPTC_TARGET"]; + try { + process.env["SCRIPTC_CACHE_DIR"] = cacheRoot; + delete process.env["SCRIPTC_NO_CACHE"]; + process.env["SCRIPTC_CC"] = "zigcc"; + process.env["SCRIPTC_TARGET"] = target; + await writeFile(cPath, "int scriptc_large_program_value(void) { return 7; }\n"); + await compileLibArchive({ + cPath, + identityCSource: [ + "unsigned long long scriptc_build_id(void) { return 2; }", + "unsigned scriptc_abi_version(void) { return 1; }", + "", + ].join("\n"), + outPath: archivePath, + cacheIdentity: TEST_CACHE_IDENTITY, + localizeSymbols: [ + "scriptc_large_program_value", + "scriptc_build_id", + "scriptc_abi_version", + ], + }); + await writeFile(probePath, [ + "int scriptc_large_program_value(void);", + "unsigned long long scriptc_build_id(void);", + "unsigned scriptc_abi_version(void);", + "int main(void) {", + " return scriptc_large_program_value() != 7 || scriptc_build_id() != 2 || scriptc_abi_version() != 1;", + "}", + "", + ].join("\n")); + execFileSync(zigExecutable!, [ + "cc", "-target", target, probePath, archivePath, "-lm", "-o", probeOutput, + ]); + } finally { + if (oldCacheDir === undefined) delete process.env["SCRIPTC_CACHE_DIR"]; + else process.env["SCRIPTC_CACHE_DIR"] = oldCacheDir; + if (oldNoCache === undefined) delete process.env["SCRIPTC_NO_CACHE"]; + else process.env["SCRIPTC_NO_CACHE"] = oldNoCache; + if (oldCc === undefined) delete process.env["SCRIPTC_CC"]; + else process.env["SCRIPTC_CC"] = oldCc; + if (oldTarget === undefined) delete process.env["SCRIPTC_TARGET"]; + else process.env["SCRIPTC_TARGET"] = oldTarget; + } + }, +); diff --git a/packages/compiler/src/backend/cc.ts b/packages/compiler/src/backend/cc.ts index 8af648ae5..58f763859 100644 --- a/packages/compiler/src/backend/cc.ts +++ b/packages/compiler/src/backend/cc.ts @@ -1398,6 +1398,15 @@ const LIB_RUNTIME_SOURCES = [ export interface LibArchiveOptions { /** The program TU (.c or .ll — clang compiles either with -c). */ cPath: string; + /** Invocation-owned program source to compile under `cPath`'s public + * spelling. Library assembly uses this for the identity-free projection of + * a complete caller-visible TU; its bytes drive every native cache key. */ + programSource?: string; + /** Tiny generated C source carrying volatile library identity getters. + * Its bytes join the complete archive key, but the source itself exists + * only in the invocation-private build directory and the large program- + * object cache is keyed independently. */ + identityCSource?: string; /** The archive to produce (.lib.a). */ outPath: string; /** Caller-owned identity for the generated TU's complete non-system @@ -1567,30 +1576,32 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise let archiverVersion = ""; let runtimeHash = ""; let programDependencyHash = ""; - let cachedProgramBytes: Buffer | null = null; + let cachedProgramBytes = opts.programSource === undefined + ? null + : Buffer.from(opts.programSource, "utf8"); + const identityBytes = opts.identityCSource === undefined + ? null + : Buffer.from(opts.identityCSource, "utf8"); if (root !== null) { try { const [cv, fingerprint, programBytes] = await Promise.all([ ccVersionOnce(driver.argv, toolchainEnv, true), runtimeFingerprint(rtDir), - readFile(opts.cPath), + cachedProgramBytes === null ? readFile(opts.cPath) : Promise.resolve(cachedProgramBytes), ]); compilerVersion = cv; runtimeHash = fingerprint; cachedProgramBytes = programBytes; + programDependencyHash = await translationUnitDependencyFingerprint( + driver, + cflags, + opts.cPath, + programBytes, + toolchainEnv, + ); if (cacheCompleteArchive) { - const [av, programDependencies] = await Promise.all([ - toolVersionOnce(arArgv, toolchainEnv, true), - translationUnitDependencyFingerprint( - driver, - cflags, - opts.cPath, - programBytes, - toolchainEnv, - ), - ]); + const av = await toolVersionOnce(arArgv, toolchainEnv, true); archiverVersion = av; - programDependencyHash = programDependencies; const key = createHash("sha256") // v7 adds effective compiler-wrapper invocations for the real runtime // and program compile flavors. @@ -1600,7 +1611,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise .update(implicitToolchain!).update("\0") .update(runtimeCompilerInvocation!).update("\0") .update(programCompilerInvocation!).update("\0") - .update(programDependencies).update("\0") + .update(programDependencyHash).update("\0") .update(opts.cacheIdentity!).update("\0") .update(driver.argv.join("\x1f")).update("\0") .update(cv).update("\0") @@ -1615,6 +1626,9 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise .update(opts.cPath).update("\0") .update(resolve(opts.cPath)).update("\0") .update(programBytes) + .update("\0identity\0") + .update(identityBytes === null ? "" : "").update("\0") + .update(identityBytes ?? Buffer.alloc(0)) .digest("hex"); cachedArchive = join(root, "lib", key); const tmpOut = privateSiblingPath(opts.outPath, "lib-hit"); @@ -1693,11 +1707,84 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise ? opts.cPath : join(buildDir, `program${opts.cPath.endsWith(".ll") ? ".ll" : ".c"}`); if (cachedProgramBytes !== null) await writeFile(programSource, cachedProgramBytes); - const programObject = await compileOne( - programSource, - `${stem}.program.o`, - cachedProgramBytes === null ? undefined : opts.cPath, - ); + let cachedProgramObject: string | null = null; + if ( + root !== null && cachedProgramBytes !== null && compilerVersion !== "" && + implicitToolchain !== null && programCompilerInvocation !== null + ) { + const programKey = createHash("sha256") + .update("lib-program-obj-v1\0") + .update(cacheTargetIdentity(driver)).update("\0") + .update(toolchainEnv).update("\0") + .update(implicitToolchain).update("\0") + .update(programCompilerInvocation).update("\0") + .update(opts.cacheIdentity!).update("\0") + .update(driver.argv.join("\x1f")).update("\0") + .update(compilerVersion).update("\0") + .update(runtimeHash).update("\0") + .update(programDependencyHash).update("\0") + .update(programCompilerArgs.join("\x1f")).update("\0") + .update(opts.cPath).update("\0") + .update(resolve(opts.cPath)).update("\0") + .update(cachedProgramBytes) + .digest("hex"); + cachedProgramObject = join(root, "program-obj", programKey); + } + const stagedProgramObject = join(buildDir, `${stem}.program.o`); + let programObject: string; + if ( + cachedProgramObject !== null && + await copyValidCachedFile(cachedProgramObject, stagedProgramObject) + ) { + programObject = stagedProgramObject; + } else { + programObject = await compileOne( + programSource, + `${stem}.program.o`, + cachedProgramBytes === null ? undefined : opts.cPath, + ); + if (cachedProgramObject !== null) { + try { + const [currentRuntime, currentImplicit, currentInvocation, currentDependencies, currentCompiler] = + await Promise.all([ + runtimeFingerprint(rtDir), + implicitToolchainFingerprint(driver, toolchainEnv), + effectiveCompilerInvocationFingerprint( + driver, + toolchainEnv, + programCompilerArgs, + programSourceExtension, + ), + translationUnitDependencyFingerprint( + driver, + cflags, + opts.cPath, + cachedProgramBytes!, + toolchainEnv, + ), + ccVersionOnce(driver.argv, toolchainEnv, true), + ]); + if ( + currentRuntime === runtimeHash && + currentImplicit === implicitToolchain && + currentInvocation === programCompilerInvocation && + currentDependencies === programDependencyHash && + currentCompiler === compilerVersion + ) { + await publishCachedFile(programObject, cachedProgramObject); + } + } catch { + // Best-effort: the archive build already owns a valid object. + } + } + } + const identityObject = identityBytes === null + ? null + : await (async () => { + const source = join(buildDir, "identity.c"); + await writeFile(source, identityBytes); + return compileOne(source, `${stem}.identity.o`); + })(); let runtimeObjects: string[] | null = null; let cacheInputsStable = true; let objectImplicitVerification: Promise | null = null; @@ -1746,7 +1833,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise ); } } - const objects = [programObject, ...runtimeObjects, ...lreObjects, ...zlibObjects]; + const objects = [programObject, ...(identityObject === null ? [] : [identityObject]), ...runtimeObjects, ...lreObjects, ...zlibObjects]; // Multi-instance library mode: the archive's one member becomes the // combined, symbol-localized object (cached vendor/runtime objects // are read-only inputs here — the combine step never mutates them). @@ -1758,7 +1845,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise driver, arArgv, buildDir, - programObject, + [programObject, ...(identityObject === null ? [] : [identityObject])], [...runtimeObjects, ...lreObjects, ...zlibObjects], opts.localizeSymbols, stem, @@ -1855,7 +1942,8 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise * shared by design. Windows embedders additionally link advapi32, iphlpapi, * and ws2_32. * - * Member selection matters: a classic archive's unused members (and their + * The generated program and optional identity objects are mandatory roots; + * support-member selection still matters. A classic archive's unused members (and their * undefined references to units library mode excludes, like the * fs-promises unit's fiber symbols) never reach an embedder's link. A * blind merge of every object would carry those references into the one @@ -1864,7 +1952,7 @@ export async function compileLibArchive(opts: LibArchiveOptions): Promise * members the program object transitively needs (the COFF arm implements * the same member semantics in process). * - * Mach-O — one host-ld64 invocation: -r merges program + needed members, + * Mach-O — one host-ld64 invocation: -r merges roots + needed members, * -exported_symbols_list demotes every unlisted global to * private extern, and -r without -keep_private_externs writes * private externs out as non-external symbols. Apple ASan's @@ -1899,7 +1987,7 @@ async function localizeLibraryObjects( driver: CcDriver, arArgv: readonly string[], buildDir: string, - programObject: string, + rootObjects: readonly string[], supportObjects: readonly string[], keepSymbols: readonly string[], stem: string, @@ -1922,14 +2010,15 @@ async function localizeLibraryObjects( if (platform === "win32") { // COFF has no relocatable-link tool to stage through; the member // selection and combine+demote happen in process over the object bytes. - const [program, ...support] = await Promise.all( - [programObject, ...supportObjects].map((path) => readFile(path)), - ); + const [roots, support] = await Promise.all([ + Promise.all(rootObjects.map((path) => readFile(path))), + Promise.all(supportObjects.map((path) => readFile(path))), + ]); try { await writeFile( combined, - mergeAndLocalizeCoffObjects(program!, support, new Set(keepSymbols), { - program: basename(programObject), + mergeAndLocalizeCoffObjects(roots, support, new Set(keepSymbols), { + roots: rootObjects.map((path) => basename(path)), support: supportObjects.map((path) => basename(path)), }), ); @@ -1943,10 +2032,10 @@ async function localizeLibraryObjects( await run([arArgv[0] ?? "ar", ...arArgv.slice(1), "rcs", staging, ...supportObjects]); if (platform === "darwin") { await writeFile(keepFile, keepSymbols.map((s) => `_${s}\n`).join("")); - await run(["ld", "-r", programObject, staging, "-o", combined, "-exported_symbols_list", keepFile]); + await run(["ld", "-r", ...rootObjects, staging, "-o", combined, "-exported_symbols_list", keepFile]); } else if (platform === "linux" && driver.target === null) { await writeFile(keepFile, keepSymbols.map((s) => `${s}\n`).join("")); - await run(["ld", "-r", "--force-group-allocation", programObject, staging, "-o", combined]); + await run(["ld", "-r", "--force-group-allocation", ...rootObjects, staging, "-o", combined]); await run(["objcopy", `--keep-global-symbols=${keepFile}`, combined]); } else if (platform === "linux") { // Cross ELF: the cross driver's own lld performs the relocatable merge @@ -1960,7 +2049,7 @@ async function localizeLibraryObjects( ...driver.argv.slice(1), "-target", driver.zigTarget ?? driver.target!, "-nostdlib", - "-r", programObject, staging, + "-r", ...rootObjects, staging, "-o", combined, ]); try { diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index 4403d160c..a8b8e3d79 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -62,9 +62,20 @@ import { emitAsyncScaffolding, childDataThunkFor, childExitThunkFor, childExitTh import { emitNpmEmbedding, islandAdapter, islandTypedAdapter } from "./emit-island.js"; import { emitFunction, emitBlock, emitStmts, emitStmt, emitTryCatch, emitSwitch, mergeBrace, emitBranchInto, emitCondition } from "./emit-stmts.js"; import { emitExpr } from "./emit-exprs.js"; +import { C_LIBRARY_IDENTITY_BEGIN, C_LIBRARY_IDENTITY_END } from "../library-identity.js"; -export function emitModule(mod: IrModule, sourceText?: string): string { - return new CEmitter(mod, sourceText).emit(); +export interface CEmitOptions { + /** Library archive assembly may move the volatile identity getters into a + * separate translation unit. Public/direct emission keeps them by default. */ + emitLibraryIdentity?: boolean; +} + +export function emitModule( + mod: IrModule, + sourceText?: string, + options: CEmitOptions = {}, +): string { + return new CEmitter(mod, sourceText, options).emit(); } // Box construction moved onto CEmitter (boxNewC method): obj-kind boxes now @@ -414,6 +425,7 @@ export class CEmitter { constructor( readonly mod: IrModule, sourceText?: string, + private readonly options: CEmitOptions = {}, ) { this.ffiCallbackAdapters = allocateFfiCallbackAdapters(mod.ffiImports ?? []); this.ffiHasRetainedCallback = hasRetainedFfiCallback(mod.ffiImports ?? []); @@ -1141,12 +1153,13 @@ export class CEmitter { } out.push(` return -1;`, `}`, ``); } - if (lib.identity !== undefined) { + if (lib.identity !== undefined && this.options.emitLibraryIdentity !== false) { // Profile-declared identity getters (the ask-2 sidecar's boot-time // pairing fence): pure data returns with NO entry prologue — exempt // from the poisoned guard and every runtime touch (ratified), so a // host can read them before init and after a trap. out.push( + C_LIBRARY_IDENTITY_BEGIN, `uint64_t ${lib.identity.buildIdSymbol}(void) {`, ` return UINT64_C(0x${lib.identity.buildId});`, `}`, @@ -1155,6 +1168,7 @@ export class CEmitter { ` return ${lib.identity.abiVersion}u;`, `}`, ``, + C_LIBRARY_IDENTITY_END, ); } if (lib.resultResetSymbol !== null) { diff --git a/packages/compiler/src/backend/library-identity.ts b/packages/compiler/src/backend/library-identity.ts new file mode 100644 index 000000000..7a27d63d4 --- /dev/null +++ b/packages/compiler/src/backend/library-identity.ts @@ -0,0 +1,38 @@ +export type LibraryEmission = "c" | "llvm"; + +export const C_LIBRARY_IDENTITY_BEGIN = "/* scriptc-library-identity: begin */"; +export const C_LIBRARY_IDENTITY_END = "/* scriptc-library-identity: end */"; +export const LLVM_LIBRARY_IDENTITY_BEGIN = "; scriptc-library-identity: begin"; +export const LLVM_LIBRARY_IDENTITY_END = "; scriptc-library-identity: end"; + +/** Remove the generated identity region from a complete caller-visible + * library TU. Archive assembly compiles this stable projection beside the + * small volatile identity C object, while the public TU remains complete. */ +export function stripLibraryIdentity( + source: string, + emission: LibraryEmission, +): string { + const begin = emission === "c" + ? C_LIBRARY_IDENTITY_BEGIN + : LLVM_LIBRARY_IDENTITY_BEGIN; + const end = emission === "c" + ? C_LIBRARY_IDENTITY_END + : LLVM_LIBRARY_IDENTITY_END; + const start = source.indexOf(`${begin}\n`); + if (start < 0) return source; + if (source.indexOf(begin, start + begin.length) >= 0) { + throw new Error("generated library TU contains multiple identity regions"); + } + const endStart = source.indexOf(end, start + begin.length); + if (endStart < 0) throw new Error("generated library TU has an unterminated identity region"); + const endOffset = endStart + end.length; + const suffix = source[endOffset] === "\n" ? endOffset + 1 : endOffset; + let prefix = source.slice(0, start); + // An omitted final region leaves the emitter's preceding empty array entry + // as one trailing newline; a marked final region has material after that + // entry and therefore renders it as two. Restore the omitted form exactly. + if (suffix === source.length && prefix.endsWith("\n\n")) { + prefix = prefix.slice(0, -1); + } + return prefix + source.slice(suffix); +} diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index ea05035f8..cd3b9ad7d 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -61,6 +61,7 @@ * lazy-inflate representation as the C debugging backend. */ import { deflateRawSync } from "node:zlib"; +import { LLVM_LIBRARY_IDENTITY_BEGIN, LLVM_LIBRARY_IDENTITY_END } from "../library-identity.js"; import type { IrBytesElem, IrExpr, @@ -190,6 +191,9 @@ export interface LlvmTargetOptions { pointerBits?: 32 | 64; /** Select the WASI libc entry-point convention. */ wasi?: boolean; + /** Library archive assembly may move the volatile identity getters into a + * separate translation unit. Public/direct emission keeps them by default. */ + emitLibraryIdentity?: boolean; } export function emitLlvmModule(mod: IrModule, options: LlvmTargetOptions = {}): string { @@ -974,6 +978,7 @@ class LlEmitter { readonly sizeType: "i32" | "i64"; readonly cycleColorOffset: number; private readonly wasi: boolean; + private readonly emitLibraryIdentity: boolean; /** Interned string literals: UTF-8 text → { symbol, byte length } — * first-use order, the C emitter's determinism discipline. */ private readonly literals = new Map(); @@ -1141,6 +1146,7 @@ class LlEmitter { constructor(private readonly mod: IrModule, options: LlvmTargetOptions) { this.sizeType = options.pointerBits === 32 ? "i32" : "i64"; this.wasi = options.wasi === true; + this.emitLibraryIdentity = options.emitLibraryIdentity !== false; // ScrCycHdr is { ptr trace; ptr free; i32 color; i16 buffered; // i16 gen; size_t buf_index }. The object follows it, so color is 12 // bytes behind a wasm32 object and 16 bytes behind a 64-bit object. @@ -2322,7 +2328,7 @@ class LlEmitter { }); out.push(`miss:`, ` ret i32 -1`, `}`, ``); } - if (lib.identity !== undefined) { + if (lib.identity !== undefined && this.emitLibraryIdentity) { // Profile-declared identity getters (the ask-2 sidecar's boot-time // pairing fence): pure data returns with NO entry prologue — exempt // from the poisoned guard and every runtime touch (ratified), so a @@ -2330,6 +2336,7 @@ class LlEmitter { // i64 two's-complement (LLVM integer constants are signed). const buildId = BigInt.asIntN(64, BigInt(`0x${lib.identity.buildId}`)).toString(); out.push( + LLVM_LIBRARY_IDENTITY_BEGIN, `define i64 @${lib.identity.buildIdSymbol}() ${FN_ATTRS} { ; identity getter build_id 0x${lib.identity.buildId}`, `entry:`, ` ret i64 ${buildId}`, @@ -2340,6 +2347,7 @@ class LlEmitter { ` ret i32 ${lib.identity.abiVersion}`, `}`, ``, + LLVM_LIBRARY_IDENTITY_END, ); } if (lib.resultResetSymbol !== null) { diff --git a/packages/compiler/src/backend/object-localize.test.ts b/packages/compiler/src/backend/object-localize.test.ts index 7ec74fd8b..e5321bee0 100644 --- a/packages/compiler/src/backend/object-localize.test.ts +++ b/packages/compiler/src/backend/object-localize.test.ts @@ -431,7 +431,7 @@ describe("mergeAndLocalizeCoffObjects", () => { ], ); const merged = readCoff( - mergeAndLocalizeCoffObjects(program, [needed, unneeded], new Set(["keep_me"])), + mergeAndLocalizeCoffObjects([program], [needed, unneeded], new Set(["keep_me"])), ); const byName = new Map(merged.symbols.map((s) => [s.name, s])); expect(byName.get("keep_me")!.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL); @@ -446,6 +446,47 @@ describe("mergeAndLocalizeCoffObjects", () => { expect(merged.sections[0]!.relocs[0]!.sym).toBe(byName.get("helper")!.index); }); + test("includes every root even when no other object references its exports", () => { + const program = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "keep_me", section: 1 }, + ], + ); + const identity = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "build_id", section: 1 }, + { name: "abi_version", section: 1, value: 4 }, + ], + ); + const unneeded = buildCoff( + [text()], + [ + { name: ".text", section: 1, storageClass: IMAGE_SYM_CLASS_STATIC, sectionDef: true }, + { name: "lonely", section: 1 }, + { name: "excluded_unit_ref", section: 0 }, + ], + ); + + const merged = readCoff( + mergeAndLocalizeCoffObjects( + [program, identity], + [unneeded], + new Set(["keep_me", "build_id", "abi_version"]), + ), + ); + const byName = new Map(merged.symbols.map((symbol) => [symbol.name, symbol])); + for (const name of ["keep_me", "build_id", "abi_version"]) { + expect(byName.get(name)?.storageClass).toBe(IMAGE_SYM_CLASS_EXTERNAL); + expect(byName.get(name)?.section).toBeGreaterThan(0); + } + expect(byName.has("lonely")).toBe(false); + expect(byName.has("excluded_unit_ref")).toBe(false); + }); + test("does not pull an alternate definition when a selected object already satisfies the reference", () => { // program defines foo and needs bar; the first support member defines // bar and calls back into foo. A real archive link stops there. The @@ -477,7 +518,7 @@ describe("mergeAndLocalizeCoffObjects", () => { ); const merged = readCoff( - mergeAndLocalizeCoffObjects(program, [needed, alternate], new Set()), + mergeAndLocalizeCoffObjects([program], [needed, alternate], new Set()), ); const byName = new Map(merged.symbols.map((s) => [s.name, s])); expect(byName.has("alternate_only")).toBe(false); @@ -528,7 +569,7 @@ describe("mergeAndLocalizeCoffObjects", () => { { name: ".refptr.shared", section: 2 }, ], ); - const merged = readCoff(mergeAndLocalizeCoffObjects(program, [support], new Set(["keep_me"]))); + const merged = readCoff(mergeAndLocalizeCoffObjects([program], [support], new Set(["keep_me"]))); // One survivor section carries the stub; no COMDAT flag remains anywhere. const rdata = merged.sections.filter((s) => s.name === ".rdata"); expect(rdata.length).toBe(1); @@ -578,7 +619,7 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT LARGEST retains the largest selected definition", () => { const pair = comdatPair(6, new Uint8Array([1, 2, 3, 4]), new Uint8Array([5, 6, 7, 8, 9])); - const merged = readCoff(mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(["entry"]))); + const merged = readCoff(mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(["entry"]))); const rdata = merged.sections.filter((section) => section.name === ".rdata"); expect(rdata).toHaveLength(1); expect([...rdata[0]!.data]).toEqual([5, 6, 7, 8, 9]); @@ -620,7 +661,7 @@ describe("mergeAndLocalizeCoffObjects", () => { }; const merged = readCoff( mergeAndLocalizeCoffObjects( - pair.program, + [pair.program], [withAssociate(1), withAssociate(2)], new Set(["entry"]), ), @@ -633,8 +674,8 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT SAME_SIZE refuses definitions with different sizes", () => { const pair = comdatPair(3, new Uint8Array(4), new Uint8Array(8)); expect(() => - mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), { - program: "program.o", + mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), { + roots: ["program.o"], support: ["one.o", "two.o"], }), ).toThrow(/SAME_SIZE mismatch.*one\.o.*two\.o/); @@ -643,8 +684,8 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT EXACT_MATCH refuses equal-size definitions with different contents", () => { const pair = comdatPair(4, new Uint8Array([1, 2, 3, 4]), new Uint8Array([1, 2, 3, 5])); expect(() => - mergeAndLocalizeCoffObjects(pair.program, pair.support, new Set(), { - program: "program.o", + mergeAndLocalizeCoffObjects([pair.program], pair.support, new Set(), { + roots: ["program.o"], support: ["one.o", "two.o"], }), ).toThrow(/EXACT_MATCH mismatch.*one\.o.*two\.o/); @@ -653,7 +694,7 @@ describe("mergeAndLocalizeCoffObjects", () => { test("COMDAT duplicates refuse conflicting selection kinds", () => { const one = comdatPair(2, new Uint8Array(4), new Uint8Array(4)); const two = comdatPair(3, new Uint8Array(4), new Uint8Array(4)); - expect(() => mergeAndLocalizeCoffObjects(one.program, [one.support[0]!, two.support[1]!], new Set())) + expect(() => mergeAndLocalizeCoffObjects([one.program], [one.support[0]!, two.support[1]!], new Set())) .toThrow(/conflicting COMDAT selections/); }); @@ -676,8 +717,8 @@ describe("mergeAndLocalizeCoffObjects", () => { ], ); expect(() => - mergeAndLocalizeCoffObjects(one, [two], new Set(["keep_me"]), { - program: "one.o", + mergeAndLocalizeCoffObjects([one], [two], new Set(["keep_me"]), { + roots: ["one.o"], support: ["two.o"], }), ).toThrow(/duplicate external symbol dup.*one\.o.*two\.o/); @@ -686,6 +727,6 @@ describe("mergeAndLocalizeCoffObjects", () => { test("refuses non-AMD64 machines", () => { const object = buildCoff([text()], [{ name: "x", section: 1 }]); new DataView(object.buffer, object.byteOffset).setUint16(0, 0xaa64, true); - expect(() => mergeAndLocalizeCoffObjects(object, [], new Set())).toThrow(/machine/); + expect(() => mergeAndLocalizeCoffObjects([object], [], new Set())).toThrow(/machine/); }); }); diff --git a/packages/compiler/src/backend/object-localize.ts b/packages/compiler/src/backend/object-localize.ts index 979bac009..5c8bde8bb 100644 --- a/packages/compiler/src/backend/object-localize.ts +++ b/packages/compiler/src/backend/object-localize.ts @@ -19,13 +19,13 @@ * another archive's copy at the embedder's link. * * mergeAndLocalizeCoffObjects - * the COFF combine+demote in one pass: pull - * support objects on undefined-symbol demand (the - * staging-archive member semantics `ld -r` gives - * the other formats), concatenate the selected - * objects' sections, resolve cross-object symbol - * references by index, then demote every defined - * external outside the keep set to a static + * the COFF combine+demote in one pass: include every + * root object, pull support objects on undefined- + * symbol demand (the staging-archive member semantics + * `ld -r` gives the other formats), concatenate the + * selected objects' sections, resolve cross-object + * symbol references by index, then demote every + * defined external outside the keep set to a static * symbol. * * Shared demotion rule (GNU objcopy --keep-global-symbols semantics): @@ -544,20 +544,21 @@ function coffSectionContentsEqual(a: CoffSection, b: CoffSection): boolean { return true; } -/** Combine a program object with the support objects it (transitively) - * needs into ONE COFF object, then demote every defined external outside - * `keep` to a static symbol. Support objects join on undefined-symbol +/** Combine mandatory root objects with the support objects they + * (transitively) need into ONE COFF object, then demote every defined external + * outside `keep` to a static symbol. Support objects join on undefined-symbol * demand — the staging-archive member semantics the other formats get from * `ld -r` — so an unused member's undefined references never reach the * embedder's link. x86_64 only (the one COFF target scriptc produces). */ export function mergeAndLocalizeCoffObjects( - program: Uint8Array, + roots: readonly Uint8Array[], support: readonly Uint8Array[], keep: ReadonlySet, - labels?: { program?: string; support?: readonly string[] }, + labels?: { roots?: readonly string[]; support?: readonly string[] }, ): Uint8Array { + if (roots.length === 0) fail("COFF localization requires at least one root object"); const objects = [ - parseCoff(program, labels?.program ?? "program object"), + ...roots.map((bytes, i) => parseCoff(bytes, labels?.roots?.[i] ?? `root object ${i}`)), ...support.map((bytes, i) => parseCoff(bytes, labels?.support?.[i] ?? `support object ${i}`)), ]; for (const object of objects) { @@ -571,7 +572,7 @@ export function mergeAndLocalizeCoffObjects( // list order wins the pull, matching `ar` member order. const definers = new Map(); objects.forEach((object, index) => { - if (index === 0) return; + if (index < roots.length) return; for (const sym of object.symbols) { if (!coffDefines(sym)) continue; const list = definers.get(sym.name); @@ -579,7 +580,7 @@ export function mergeAndLocalizeCoffObjects( else list.push(index); } }); - const included: boolean[] = objects.map((_, i) => i === 0); + const included: boolean[] = objects.map((_, i) => i < roots.length); // Archive extraction consults the linker's CURRENT symbol state: an // undefined in a newly pulled member is already satisfied when the // program object (or an earlier member) defines it. Record every @@ -593,8 +594,8 @@ export function mergeAndLocalizeCoffObjects( if (coffDefines(sym)) selectedDefinitions.add(sym.name); } }; - addDefinitions(objects[0]!); - const queue = [0]; + for (let i = 0; i < roots.length; i++) addDefinitions(objects[i]!); + const queue = roots.map((_, i) => i); while (queue.length > 0) { const object = objects[queue.shift()!]!; for (const sym of object.symbols) { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index 90e38ab7d..bb3a14457 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,8 +1,9 @@ -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js"; import { emitModule } from "./backend/emission/emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; +import { stripLibraryIdentity } from "./backend/library-identity.js"; import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./library/int-infer.js"; import { loadLibraryProfile, profileRemediation, profileTeaching, type LibraryProfile } from "./library/profile.js"; @@ -41,7 +42,7 @@ export const VERSION = "0.0.1"; export { compileC, runtimeSrcDir, type CcOptions } from "./backend/cc.js"; export { ANDROID_MIN_API, IPHONEOS_MIN_VERSION, isAndroidTarget, isIosTarget, isMobileTarget, mobileLibraryTarget, mobileTargetRefusal } from "./backend/cc.js"; -export { emitModule } from "./backend/emission/emitter.js"; +export { emitModule, type CEmitOptions } from "./backend/emission/emitter.js"; export type { ScrDiagnostic } from "./diagnostics/diagnostic.js"; export { renderAll, renderDiagnostic } from "./diagnostics/render.js"; export { renderCoverage, type CoverageInput } from "./coverage/report.js"; @@ -1522,6 +1523,7 @@ function libraryNativeFeatures( zlib: moduleUsesZlib(mod), copying: moduleUsesCopying(mod), textDecoderLegacy: moduleUsesLegacyTextDecoder(mod), + ...(mod.lib?.identity !== undefined ? { buildId: mod.lib.identity.buildId } : {}), }; } @@ -1549,8 +1551,27 @@ async function compileLibraryNative( features: EarlyLibraryNativeFeatures, ): Promise { const localizeSymbols = libraryLocalizeSymbols(profile); + let identityCSource: string | undefined; + let programSource: string | undefined; + if (profile.sidecar !== null) { + if (features.buildId === undefined) throw new Error("library identity TU has no build id"); + const publicSource = await readFile(cPath, "utf8"); + programSource = stripLibraryIdentity(publicSource, profile.emission); + if (programSource === publicSource) { + throw new Error("generated public library TU has no identity region"); + } + identityCSource = [ + "#include ", + "#include ", + `uint64_t ${profile.sidecar.buildIdSymbol}(void) { return UINT64_C(0x${features.buildId}); }`, + `uint32_t ${profile.sidecar.abiVersionSymbol}(void) { return ${profile.sidecar.abiVersion}u; }`, + "", + ].join("\n"); + } await compileLibArchive({ cPath, + ...(programSource !== undefined ? { programSource } : {}), + ...(identityCSource !== undefined ? { identityCSource } : {}), outPath: archivePath, cacheIdentity: "scriptc-generated-library-v1", sanitize, @@ -1592,6 +1613,7 @@ async function emitSemanticLibraryHit( modules, ); mod.lib.identity.buildId = buildId; + hit.native.buildId = buildId; sidecarJson = updateSidecarIdentity(sidecarJson, buildId, sourceHash); } const validation = validateModule(mod); @@ -1612,7 +1634,10 @@ async function emitSemanticLibraryHit( timing("semantic-llvm-emit", { output_bytes: Buffer.byteLength(ll) }); } else { cPath = join(opts.outDir, `${stem}.lib.c`); - await writeFile(cPath, emitModule(mod, hit.sourceTexts.get(profile.entry))); + await writeFile( + cPath, + emitModule(mod, hit.sourceTexts.get(profile.entry)), + ); timing("semantic-c-emit"); } await rm(join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "c" : "ll"}`), { force: true }); @@ -2005,9 +2030,9 @@ async function compileLibraryTracked( // The ask-2 contract sidecar rides the same invocation. Identity first // (schema §2's worked build_id definition over compiler version, profile // bytes, and the sorted canonical module graph; source_hash per the - // profile's "module-graph" contract) — the u64 lands on the IR so both - // backends emit the identity getters from the ONE value the sidecar - // records (V12's coherence by construction), then the projection into + // profile's "module-graph" contract) — the u64 lands on the IR so native + // archive assembly emits the identity getters from the ONE value the + // sidecar records (V12's coherence by construction), then the projection into // the schema (declaration orders from the AST) and the V1–V14 // self-check before anything is written. let sidecarJson: string | null = null; diff --git a/packages/compiler/src/ir/nodes.ts b/packages/compiler/src/ir/nodes.ts index 2b3a67948..ca48a7ac0 100644 --- a/packages/compiler/src/ir/nodes.ts +++ b/packages/compiler/src/ir/nodes.ts @@ -984,9 +984,9 @@ export interface IrLibSection { identity?: IrLibIdentity; } -/** The profile-declared identity getters' facts, landed on the IR so both - * backends emit the same constants the sidecar records (V12's identity - * coherence is one-value-two-writes by construction). */ +/** The profile-declared identity getters' facts, landed on the IR so native + * archive assembly emits the same constants the sidecar records (V12's + * identity coherence is one-value-two-writes by construction). */ export interface IrLibIdentity { buildIdSymbol: string; abiVersionSymbol: string; diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index 0124be20a..4a50e4430 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -56,6 +56,8 @@ export interface EarlyLibraryNativeFeatures { zlib: boolean; copying: boolean; textDecoderLegacy: boolean; + /** Volatile exact-source build identity emitted from the tiny identity TU. */ + buildId?: string; } export interface EarlyLibraryCacheOptions { @@ -113,7 +115,8 @@ function validNativeFeatures(value: unknown): value is EarlyLibraryNativeFeature native.zlib, native.copying, native.textDecoderLegacy, - ].every((flag) => typeof flag === "boolean"); + ].every((flag) => typeof flag === "boolean") && + (native.buildId === undefined || /^[0-9a-f]{16}$/.test(native.buildId)); } function digest(bytes: Uint8Array): string { diff --git a/packages/compiler/test/library-identity-emission.test.ts b/packages/compiler/test/library-identity-emission.test.ts new file mode 100644 index 000000000..67346f211 --- /dev/null +++ b/packages/compiler/test/library-identity-emission.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test } from "vitest"; +import { emitModule } from "../src/index.js"; +import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; +import { stripLibraryIdentity } from "../src/backend/library-identity.js"; +import type { IrModule } from "../src/ir/nodes.js"; +import { fibModule } from "./fixtures/fib-ir.js"; + +const libraryModule = (): IrModule => ({ + ...fibModule, + lib: { + profileName: "identity-emission", + prefix: "ie_", + initSymbol: "ie_init", + sinkRegisterSymbol: "ie_set_sink", + collectSymbol: null, + resultResetSymbol: null, + threadInstances: false, + exports: [], + trapOverlays: [], + identity: { + buildIdSymbol: "ie_build_id", + abiVersionSymbol: "ie_abi_version", + buildId: "fedcba9876543210", + abiVersion: 7, + }, + }, +}); + +describe("library identity emission", () => { + test("direct C emission retains the IR-declared identity getters", () => { + const emitted = emitModule(libraryModule()); + expect(emitted).toContain("uint64_t ie_build_id(void)"); + expect(emitted).toContain("return UINT64_C(0xfedcba9876543210);"); + expect(emitted).toContain("uint32_t ie_abi_version(void)"); + expect(emitted).toContain("return 7u;"); + }); + + test("direct LLVM emission retains the IR-declared identity getters", () => { + const emitted = emitLlvmModule(libraryModule()); + expect(emitted).toContain("define i64 @ie_build_id()"); + expect(emitted).toContain("identity getter build_id 0xfedcba9876543210"); + expect(emitted).toContain("define i32 @ie_abi_version()"); + expect(emitted).toContain("ret i32 7"); + }); + + test("archive program-TU mode suppresses identity definitions in both backends", () => { + const mod = libraryModule(); + const c = emitModule(mod, undefined, { emitLibraryIdentity: false }); + const llvm = emitLlvmModule(mod, { emitLibraryIdentity: false }); + expect(c).not.toContain("ie_build_id"); + expect(c).not.toContain("ie_abi_version"); + expect(llvm).not.toContain("@ie_build_id"); + expect(llvm).not.toContain("@ie_abi_version"); + expect(stripLibraryIdentity(emitModule(mod), "c")).toBe(c); + expect(stripLibraryIdentity(emitLlvmModule(mod), "llvm")).toBe(llvm); + + mod.lib!.resultResetSymbol = "ie_reset"; + expect(stripLibraryIdentity(emitModule(mod), "c")).toBe( + emitModule(mod, undefined, { emitLibraryIdentity: false }), + ); + expect(stripLibraryIdentity(emitLlvmModule(mod), "llvm")).toBe( + emitLlvmModule(mod, { emitLibraryIdentity: false }), + ); + }); +}); diff --git a/tests/harness/README.md b/tests/harness/README.md index b1e845d70..5ff9ff766 100644 --- a/tests/harness/README.md +++ b/tests/harness/README.md @@ -73,6 +73,7 @@ Test runs are dominated by clang (~275 corpus programs × two lanes at -O2/-O1+A - **binaries** (`bin/`, cc.ts): key = resolved clang identity/version + target/compiler environment + implicit system-header dependency bytes + linker/assembler identities + runtime fingerprint (every runtime .c/.h + the vendor pin) + the full normalized command line + the emitted C bytes (byte-stable by project invariant). A hit skips native code generation and linking; the binary still RUNS live, so no comparison or sanitizer coverage is ever skipped. Each hit is checksum-verified. The sanitized lane's flags land in naturally distinct keys. FFI archive/object inputs and ambient system libraries always relink because their named files can hide mutable transitive dependencies. - **library archives** (`lib/`, cc.ts): key = resolved clang and archiver identities/versions + target/compiler environment and implicit dependencies + runtime fingerprint + target/flags + gated runtime-source set + emitted program-TU bytes. A checksum-verified hit skips native code generation and `ar`. +- **library program objects** (`program-obj/`, cc.ts): generated library TUs compile into checksum-verified objects keyed independently of the tiny exact-source identity TU. A build-id-only miss reuses the large program object, compiles the identity getters, and rearchives; runtime/header/toolchain inputs remain part of the key and are rechecked before publication. - **early library frontend** (`early-lib/`, library/early-cache.ts): exact library repeats validate content hashes for every file the TypeScript frontend read plus recorded failed-resolution and directory-enumeration probes, then restore the generated C/LLVM unit, optional IR, sidecar, and native feature gates without spawning TypeScript or lowering again. TypeScript comment-only misses may restore compressed lowered IR after token equivalence checks; source locations and exact-source sidecar/build identities regenerate from current bytes. Semantic comments/directives, JavaScript comments, token/config/package edits, and newly appearing resolution candidates miss. The native archive tier still performs its own toolchain/runtime checks. - **runtime objects** (`obj/`, cc.ts): per-flavor .o for the runtime sources, including a distinct `-DSCR_LIB` flavor, so an edited executable or library recompiles only the program's own translation unit before linking or archiving. Each object carries a verified digest; a damaged entry is rebuilt before it reaches the linker or archiver. Publication rechecks the runtime and implicit-toolchain fingerprints after compilation so a concurrent source/header edit cannot place new bytes under an old key. Compiles route through ccache when installed, silently falling back when not. - **oracle results** (`oracle/`, differential.test.ts): Node's stdout/exit per program, keyed by program bytes + the spawned node's version + shim contents + invocation shape. Only the spawn is skipped; the comparison never changes. Real-time programs (setTimeout/setInterval/Promise.race — 18 of 298) are excluded and always spawn Node live: their stdout is a timer interleave that Node and the native binary only agree on under the same instantaneous load, so a cached verdict from one run must never meet a live native run from another. diff --git a/tests/harness/library-contract.test.ts b/tests/harness/library-contract.test.ts index 2eef3e22d..659b5d94c 100644 --- a/tests/harness/library-contract.test.ts +++ b/tests/harness/library-contract.test.ts @@ -66,7 +66,7 @@ async function buildContract( * directories is exactly what the canonical root-relative paths * guarantee, and the V13-style assertions prove it). */ root = cacheDir, -): Promise<{ outDir: string; archive: string; sidecarPath: string; doc: SidecarDoc; bytes: Buffer }> { +): Promise<{ outDir: string; archive: string; cPath: string; sidecarPath: string; doc: SidecarDoc; bytes: Buffer }> { const dir = join(fixtureRoot, fixture); const outDir = join(root, `${fixture}-${emission}${tag}`); mkdirSync(outDir, { recursive: true }); @@ -92,6 +92,7 @@ async function buildContract( return { outDir, archive: result.archivePath, + cPath: result.cPath, sidecarPath: result.sidecarPath!, doc: JSON.parse(bytes.toString("utf8")) as SidecarDoc, bytes: bytes as Buffer, @@ -112,7 +113,7 @@ function nmDefined(archive: string, prefix: string): string[] { describe.each(EMISSIONS)("contract sidecar, %s emission", (emission) => { test("anti-alphabetical declaration order, schema shape, V11/V12 identity", async () => { - const { outDir, archive, doc, bytes } = await buildContract("contract", emission); + const { outDir, archive, cPath, doc, bytes } = await buildContract("contract", emission); // The emitter's own self-check ran before writing; the test-side // validator agrees the document conforms. @@ -236,6 +237,21 @@ describe.each(EMISSIONS)("contract sidecar, %s emission", (emission) => { }); expect(nmDefined(archive, "kc_")).toEqual(doc.abi.exports.map((s) => `kc_${s}`).sort()); + // The kept/returned program TU is a complete public artifact, even though + // archive assembly privately compiles an identity-free projection beside + // its tiny volatile identity object. + const keptObject = join(outDir, "kept-program.o"); + execFileSync("clang", [ + "-std=c11", + "-DSCR_LIB", + ...(emission === "llvm" + ? ["-Wno-override-module"] + : ["-Wno-comment", "-I", join(repoRoot, "packages/runtime/src")]), + "-c", cPath, + "-o", keptObject, + ]); + expect(nmDefined(keptObject, "kc_")).toEqual(doc.abi.exports.map((s) => `kc_${s}`).sort()); + // V12 + the poisoned-guard exemption, end to end: the probe reads the // getters before init and after a trap; both reads equal the // sidecar's build_id.