From 1515fd6091fa4cab2807f5a4cb2ece10d86b0e4c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 20 Aug 2026 00:06:11 -0500 Subject: [PATCH 1/4] Cache emitted library translation units - Restore cached C and LLVM translation units on semantic library cache hits while refreshing volatile build identity - Rebase public C source annotations and strip them from stable native-cache inputs --- packages/cli/test/library-output.test.ts | 9 ++ .../compiler/src/backend/emission/emitter.ts | 15 +-- .../compiler/src/backend/library-identity.ts | 122 ++++++++++++++++-- packages/compiler/src/backend/llvm/emitter.ts | 18 +-- packages/compiler/src/index.ts | 43 ++++-- .../compiler/src/library/early-cache.test.ts | 6 + packages/compiler/src/library/early-cache.ts | 13 +- .../src/library/semantic-source.test.ts | 11 +- .../compiler/src/library/semantic-source.ts | 50 +++++++ .../test/library-identity-emission.test.ts | 73 ++++++++++- 10 files changed, 299 insertions(+), 61 deletions(-) diff --git a/packages/cli/test/library-output.test.ts b/packages/cli/test/library-output.test.ts index 8f1ad76b..0a9fa0ef 100644 --- a/packages/cli/test/library-output.test.ts +++ b/packages/cli/test/library-output.test.ts @@ -79,6 +79,15 @@ test("library identity source stays private and cannot overwrite a sidecar", asy await runBuild(); expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); + // A comment-only edit takes the semantic cache path: restore the emitted + // TU, refresh its private identity object, and still honor --no-keep-c. + await writeFile(join(dir, "lib.ts"), [ + "// harmless rebuild comment", + await readFile(join(dir, "lib.ts"), "utf8"), + ].join("\n")); + await runBuild(); + expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); + // This name collided with the former fixed `.lib.identity.c` // output. Repeat to exercise the exact early-cache-hit ordering that used // to restore JSON and then overwrite it with generated C. diff --git a/packages/compiler/src/backend/emission/emitter.ts b/packages/compiler/src/backend/emission/emitter.ts index a8b8e3d7..9cd87659 100644 --- a/packages/compiler/src/backend/emission/emitter.ts +++ b/packages/compiler/src/backend/emission/emitter.ts @@ -62,7 +62,7 @@ import { emitAsyncScaffolding, childDataThunkFor, childExitThunkFor, childExitTh import { emitNpmEmbedding, islandAdapter, islandTypedAdapter } from "./emit-island.js"; import { emitFunction, emitBlock, emitStmts, emitStmt, emitTryCatch, emitSwitch, mergeBrace, emitBranchInto, emitCondition } from "./emit-stmts.js"; import { emitExpr } from "./emit-exprs.js"; -import { C_LIBRARY_IDENTITY_BEGIN, C_LIBRARY_IDENTITY_END } from "../library-identity.js"; +import { emitLibraryIdentityLines } from "../library-identity.js"; export interface CEmitOptions { /** Library archive assembly may move the volatile identity getters into a @@ -1158,18 +1158,7 @@ export class CEmitter { // pairing fence): pure data returns with NO entry prologue — exempt // from the poisoned guard and every runtime touch (ratified), so a // host can read them before init and after a trap. - out.push( - C_LIBRARY_IDENTITY_BEGIN, - `uint64_t ${lib.identity.buildIdSymbol}(void) {`, - ` return UINT64_C(0x${lib.identity.buildId});`, - `}`, - ``, - `uint32_t ${lib.identity.abiVersionSymbol}(void) {`, - ` return ${lib.identity.abiVersion}u;`, - `}`, - ``, - C_LIBRARY_IDENTITY_END, - ); + out.push(...emitLibraryIdentityLines("c", lib.identity)); } if (lib.resultResetSymbol !== null) { out.push( diff --git a/packages/compiler/src/backend/library-identity.ts b/packages/compiler/src/backend/library-identity.ts index 7a27d63d..7a74217b 100644 --- a/packages/compiler/src/backend/library-identity.ts +++ b/packages/compiler/src/backend/library-identity.ts @@ -5,29 +5,123 @@ export const C_LIBRARY_IDENTITY_END = "/* scriptc-library-identity: end */"; export const LLVM_LIBRARY_IDENTITY_BEGIN = "; scriptc-library-identity: begin"; export const LLVM_LIBRARY_IDENTITY_END = "; scriptc-library-identity: end"; -/** Remove the generated identity region from a complete caller-visible - * library TU. Archive assembly compiles this stable projection beside the - * small volatile identity C object, while the public TU remains complete. */ -export function stripLibraryIdentity( +export interface LibraryIdentityValues { + buildIdSymbol: string; + abiVersionSymbol: string; + buildId: string; + abiVersion: number; +} + +function identityMarkers(emission: LibraryEmission): { begin: string; end: string } { + return emission === "c" + ? { begin: C_LIBRARY_IDENTITY_BEGIN, end: C_LIBRARY_IDENTITY_END } + : { begin: LLVM_LIBRARY_IDENTITY_BEGIN, end: LLVM_LIBRARY_IDENTITY_END }; +} + +export function emitLibraryIdentityLines( + emission: LibraryEmission, + identity: LibraryIdentityValues, + llvmFunctionAttrs = "#0", +): string[] { + if (!/^[0-9a-f]{16}$/.test(identity.buildId)) { + throw new Error("library build id must be exactly 16 lowercase hex digits"); + } + if (emission === "c") { + return [ + C_LIBRARY_IDENTITY_BEGIN, + `uint64_t ${identity.buildIdSymbol}(void) {`, + ` return UINT64_C(0x${identity.buildId});`, + `}`, + ``, + `uint32_t ${identity.abiVersionSymbol}(void) {`, + ` return ${identity.abiVersion}u;`, + `}`, + ``, + C_LIBRARY_IDENTITY_END, + ]; + } + const signedBuildId = BigInt.asIntN(64, BigInt(`0x${identity.buildId}`)).toString(); + return [ + LLVM_LIBRARY_IDENTITY_BEGIN, + `define i64 @${identity.buildIdSymbol}() ${llvmFunctionAttrs} { ; identity getter build_id 0x${identity.buildId}`, + `entry:`, + ` ret i64 ${signedBuildId}`, + `}`, + ``, + `define i32 @${identity.abiVersionSymbol}() ${llvmFunctionAttrs} { ; identity getter abi_version`, + `entry:`, + ` ret i32 ${identity.abiVersion}`, + `}`, + ``, + LLVM_LIBRARY_IDENTITY_END, + ]; +} + +function identityOffsets( source: string, emission: LibraryEmission, -): string { - const begin = emission === "c" - ? C_LIBRARY_IDENTITY_BEGIN - : LLVM_LIBRARY_IDENTITY_BEGIN; - const end = emission === "c" - ? C_LIBRARY_IDENTITY_END - : LLVM_LIBRARY_IDENTITY_END; +): { start: number; end: number } | null { + const { begin, end } = identityMarkers(emission); const start = source.indexOf(`${begin}\n`); - if (start < 0) return source; + if (start < 0) return null; if (source.indexOf(begin, start + begin.length) >= 0) { throw new Error("generated library TU contains multiple identity regions"); } const endStart = source.indexOf(end, start + begin.length); if (endStart < 0) throw new Error("generated library TU has an unterminated identity region"); - const endOffset = endStart + end.length; + return { start, end: endStart + end.length }; +} + +/** Refresh only the volatile identity block in a cached public TU. */ +export function replaceLibraryIdentity( + source: string, + emission: LibraryEmission, + identity: LibraryIdentityValues, +): string { + const offsets = identityOffsets(source, emission); + if (offsets === null) throw new Error("generated public library TU has no identity region"); + const replacement = emitLibraryIdentityLines(emission, identity).join("\n"); + return source.slice(0, offsets.start) + replacement + source.slice(offsets.end); +} + +function escapedRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function sourceCommentPattern(sourceFile: string): RegExp { + // Emission appends these comments at physical line ends. Requiring that + // boundary prevents source-looking text inside a generated C string from + // being mistaken for an annotation and changing program semantics. + return new RegExp(` /\\* ${escapedRegExp(sourceFile)}:(\\d+) \\*/(?=\\r?\\n|$)`, "g"); +} + +/** Refresh the source-line annotations retained in a caller-visible C TU. */ +export function rebaseLibrarySourceComments( + source: string, + sourceFile: string, + rebaseLine: (line: number) => number, +): string { + return source.replace(sourceCommentPattern(sourceFile), (_match, line: string) => + ` /* ${sourceFile}:${rebaseLine(Number(line))} */`); +} + +/** Remove C-only debugging annotations from the private native-cache input. */ +export function stripLibrarySourceComments(source: string, sourceFile: string): string { + return source.replace(sourceCommentPattern(sourceFile), ""); +} + +/** Remove the generated identity region from a complete caller-visible + * library TU. Archive assembly compiles this stable projection beside the + * small volatile identity C object, while the public TU remains complete. */ +export function stripLibraryIdentity( + source: string, + emission: LibraryEmission, +): string { + const offsets = identityOffsets(source, emission); + if (offsets === null) return source; + const endOffset = offsets.end; const suffix = source[endOffset] === "\n" ? endOffset + 1 : endOffset; - let prefix = source.slice(0, start); + let prefix = source.slice(0, offsets.start); // An omitted final region leaves the emitter's preceding empty array entry // as one trailing newline; a marked final region has material after that // entry and therefore renders it as two. Restore the omitted form exactly. diff --git a/packages/compiler/src/backend/llvm/emitter.ts b/packages/compiler/src/backend/llvm/emitter.ts index cd3b9ad7..6f685bc2 100644 --- a/packages/compiler/src/backend/llvm/emitter.ts +++ b/packages/compiler/src/backend/llvm/emitter.ts @@ -61,7 +61,7 @@ * lazy-inflate representation as the C debugging backend. */ import { deflateRawSync } from "node:zlib"; -import { LLVM_LIBRARY_IDENTITY_BEGIN, LLVM_LIBRARY_IDENTITY_END } from "../library-identity.js"; +import { emitLibraryIdentityLines } from "../library-identity.js"; import type { IrBytesElem, IrExpr, @@ -2334,21 +2334,7 @@ class LlEmitter { // from the poisoned guard and every runtime touch (ratified), so a // host can read them before init and after a trap. The u64 rides // i64 two's-complement (LLVM integer constants are signed). - const buildId = BigInt.asIntN(64, BigInt(`0x${lib.identity.buildId}`)).toString(); - out.push( - LLVM_LIBRARY_IDENTITY_BEGIN, - `define i64 @${lib.identity.buildIdSymbol}() ${FN_ATTRS} { ; identity getter build_id 0x${lib.identity.buildId}`, - `entry:`, - ` ret i64 ${buildId}`, - `}`, - ``, - `define i32 @${lib.identity.abiVersionSymbol}() ${FN_ATTRS} { ; identity getter abi_version`, - `entry:`, - ` ret i32 ${lib.identity.abiVersion}`, - `}`, - ``, - LLVM_LIBRARY_IDENTITY_END, - ); + out.push(...emitLibraryIdentityLines("llvm", lib.identity, FN_ATTRS)); } if (lib.resultResetSymbol !== null) { out.push( diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index bb3a1445..282c5451 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -3,7 +3,7 @@ import { basename, dirname, join, resolve } from "node:path"; import { buildCacheRoot, CcCompileError, compileC, compileLibArchive, mobileLibraryTarget, mobileTargetRefusal, prepareBuildCacheRoot, pruneBuildCache, resolveCc, targetPlatform } from "./backend/cc.js"; import { emitModule } from "./backend/emission/emitter.js"; import { emitLlvmModule, LlvmUnsupportedError } from "./backend/llvm/emitter.js"; -import { stripLibraryIdentity } from "./backend/library-identity.js"; +import { rebaseLibrarySourceComments, replaceLibraryIdentity, stripLibraryIdentity, stripLibrarySourceComments } from "./backend/library-identity.js"; import { checkerPanicDiag, ffiNativeBuildDiag, libAsyncExportDiag, libAsyncSurfaceDiag, libExportUnresolvedDiag, libGenericExportDiag, libIntBoundaryDiag, libNpmIneligibleDiag, libSidecarDiag, libUnmappableSignatureDiag, iceDiag, isCheckerPanic, LIB_INBOUND_BYTES_TRAP_CODE, LIB_RUNTIME_TRAP_CODES, type ScrDiagnostic } from "./diagnostics/diagnostic.js"; import { checkLibraryIntegerSlots, classSeed, hasIntSlots, numberCarrierKind, type FnIntSlots, type IntSlotConfig } from "./library/int-infer.js"; import { loadLibraryProfile, profileRemediation, profileTeaching, type LibraryProfile } from "./library/profile.js"; @@ -37,6 +37,7 @@ import { loadFfiProfile, type FfiProfile } from "./ffi/profile.js"; import { hasForeignFfiCallback } from "./backend/ffi-callbacks.js"; import { FrontendInputTracker, trackedReadFile } from "./frontend/input-tracker.js"; import { libraryFrontendImplementationFingerprint, publishEarlyLibraryCache, readEarlyLibraryCache, readSemanticLibraryCache, type EarlyLibraryCacheOptions, type EarlyLibraryCachePublish, type EarlyLibraryNativeFeatures, type SemanticLibraryCacheHit } from "./library/early-cache.js"; +import { createSourceLineRebaser } from "./library/semantic-source.js"; export const VERSION = "0.0.1"; @@ -1553,13 +1554,17 @@ async function compileLibraryNative( const localizeSymbols = libraryLocalizeSymbols(profile); let identityCSource: string | undefined; let programSource: string | undefined; + if (profile.sidecar !== null || profile.emission === "c") { + const publicSource = await readFile(cPath, "utf8"); + programSource = publicSource; + } if (profile.sidecar !== null) { if (features.buildId === undefined) throw new Error("library identity TU has no build id"); - const publicSource = await readFile(cPath, "utf8"); - programSource = stripLibraryIdentity(publicSource, profile.emission); - if (programSource === publicSource) { + const withoutIdentity = stripLibraryIdentity(programSource!, profile.emission); + if (withoutIdentity === programSource) { throw new Error("generated public library TU has no identity region"); } + programSource = withoutIdentity; identityCSource = [ "#include ", "#include ", @@ -1568,6 +1573,9 @@ async function compileLibraryNative( "", ].join("\n"); } + if (profile.emission === "c") { + programSource = stripLibrarySourceComments(programSource!, profile.entry); + } await compileLibArchive({ cPath, ...(programSource !== undefined ? { programSource } : {}), @@ -1626,20 +1634,27 @@ async function emitSemanticLibraryHit( } await mkdir(opts.outDir, { recursive: true }); const stem = basename(profile.entry).replace(/\.(ts|js|mjs|cjs)$/, ""); - let cPath: string; + const cPath = join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "ll" : "c"}`); + let translationUnit = hit.translationUnit; + if (profile.sidecar !== null) { + translationUnit = replaceLibraryIdentity(translationUnit, profile.emission, mod.lib!.identity!); + } if (profile.emission === "llvm") { - const ll = emitLlvmModule(mod); - cPath = join(opts.outDir, `${stem}.lib.ll`); - await writeFile(cPath, ll); - timing("semantic-llvm-emit", { output_bytes: Buffer.byteLength(ll) }); + await writeFile(cPath, translationUnit); } else { - cPath = join(opts.outDir, `${stem}.lib.c`); - await writeFile( - cPath, - emitModule(mod, hit.sourceTexts.get(profile.entry)), + const previous = hit.previousSources.get(mod.sourceFile); + const current = hit.sourceTexts.get(mod.sourceFile); + if (previous === undefined || current === undefined) { + throw new Error("semantic library cache lost the entry source text"); + } + translationUnit = rebaseLibrarySourceComments( + translationUnit, + mod.sourceFile, + createSourceLineRebaser(mod.sourceFile, previous, current), ); - timing("semantic-c-emit"); + await writeFile(cPath, translationUnit); } + timing("semantic-tu-restore", { output_bytes: Buffer.byteLength(translationUnit) }); await rm(join(opts.outDir, `${stem}.lib.${profile.emission === "llvm" ? "c" : "ll"}`), { force: true }); let irPath: string | undefined; if (opts.emitIr) { diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts index 1161b933..a949f0c3 100644 --- a/packages/compiler/src/library/early-cache.test.ts +++ b/packages/compiler/src/library/early-cache.test.ts @@ -285,11 +285,17 @@ test("semantic library cache restores and rebases IR after a comment-only edit", const hit = await readSemanticLibraryCache(f.root, f.options, null); expect(hit).not.toBeNull(); expect(hit?.changedSources).toEqual([f.source]); + expect(hit?.translationUnit).toBe("; generated llvm\n"); expect(hit?.mod.functions[0]!.loc.start).toBe(sourceAfter.indexOf("return")); expect(hit?.frontend.probes).toContainEqual(expect.objectContaining({ op: "file", path: f.source, })); + + const earlyRoot = join(f.root, "early-lib"); + const [key] = await readdir(earlyRoot); + await writeFile(join(earlyRoot, key!, "program.tu"), "corrupt\n"); + expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); }); test("semantic library cache refuses token and directive edits", async () => { diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index 4a50e443..832db5e9 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -93,6 +93,7 @@ export interface EarlyLibraryCachePublish extends EarlyLibraryCacheHit { export interface SemanticLibraryCacheHit { mod: IrModule; + translationUnit: string; sourceTexts: Map; previousSources: Map; frontend: FrontendInputSnapshot; @@ -367,22 +368,28 @@ export async function readSemanticLibraryCache( if ( stamp.version !== 2 || stamp.key !== cacheKey(options) || !validFrontendInputSnapshot(stamp.frontend) || !validNativeFeatures(stamp.native) || + stamp.files?.translationUnit?.name !== "program.tu" || stamp.files?.semanticIr?.name !== "semantic.ir.json.gz" || stamp.files?.sources?.name !== "sources.json.gz" || + !/^[0-9a-f]{64}$/.test(stamp.files.translationUnit.digest) || !/^[0-9a-f]{64}$/.test(stamp.files.semanticIr.digest) || !/^[0-9a-f]{64}$/.test(stamp.files.sources.digest) || (stamp.files.sidecar !== null) !== (sidecarConfiguredPath !== undefined) || stampIntegrity(unsigned) !== integrity ) return null; const directory = dirname(path); - const [irCompressed, sourcesCompressed, sidecar] = await Promise.all([ + const [translationUnit, irCompressed, sourcesCompressed, sidecar] = await Promise.all([ + readCachedFile(join(directory, stamp.files.translationUnit.name), stamp.files.translationUnit.digest), readCachedFile(join(directory, stamp.files.semanticIr.name), stamp.files.semanticIr.digest), readCachedFile(join(directory, stamp.files.sources.name), stamp.files.sources.digest), stamp.files.sidecar === null ? Promise.resolve(null) : readCachedFile(join(directory, stamp.files.sidecar.name), stamp.files.sidecar.digest), ]); - if (irCompressed === null || sourcesCompressed === null || (stamp.files.sidecar !== null && sidecar === null)) { + if ( + translationUnit === null || irCompressed === null || sourcesCompressed === null || + (stamp.files.sidecar !== null && sidecar === null) + ) { return null; } const [irJson, sourcesJson] = await Promise.all([ @@ -409,12 +416,14 @@ export async function readSemanticLibraryCache( const now = new Date(); await Promise.all([ path, + join(directory, stamp.files.translationUnit.name), join(directory, stamp.files.semanticIr.name), join(directory, stamp.files.sources.name), ...(stamp.files.sidecar === null ? [] : [join(directory, stamp.files.sidecar.name)]), ].map((cachePath) => utimes(cachePath, now, now).catch(() => undefined))); return { mod, + translationUnit: translationUnit.toString("utf8"), sourceTexts: semantic.currentSources, previousSources, frontend: semantic.snapshot, diff --git a/packages/compiler/src/library/semantic-source.test.ts b/packages/compiler/src/library/semantic-source.test.ts index adfdd5d2..66316580 100644 --- a/packages/compiler/src/library/semantic-source.test.ts +++ b/packages/compiler/src/library/semantic-source.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "vitest"; -import { rebaseSourceLocations, semanticallyEqualSource, semanticSourceDigest } from "./semantic-source.js"; +import { createSourceLineRebaser, rebaseSourceLocations, semanticallyEqualSource, semanticSourceDigest } from "./semantic-source.js"; test("ordinary TypeScript comments and formatting preserve semantic identity", () => { const before = "// old note\nexport function value(): number { return 1; }\n"; @@ -78,6 +78,15 @@ test("source locations rebase across changed comment trivia", () => { expect(payload.loc.end).toBe(after.indexOf("return") + "return 1".length); }); +test("source lines rebase across changed comment trivia", () => { + const before = "// old\nexport function value() {\n return 1;\n}\n"; + const after = "/* much longer\n * note\n */\n\nexport function value() {\n return 1;\n}\n"; + const rebase = createSourceLineRebaser("/entry.ts", before, after); + expect(rebase(2)).toBe(5); + expect(rebase(3)).toBe(6); + expect(rebase(4)).toBe(7); +}); + test("source locations at adjacent token boundaries rebase past inserted comments", () => { const before = "export function value() { return left+right; }\n"; const after = "export function value() { return left+/* note */right; }\n"; diff --git a/packages/compiler/src/library/semantic-source.ts b/packages/compiler/src/library/semantic-source.ts index 52978a32..c8b0090d 100644 --- a/packages/compiler/src/library/semantic-source.ts +++ b/packages/compiler/src/library/semantic-source.ts @@ -154,6 +154,56 @@ function offsetMapper(path: string, previous: string, current: string): OffsetMa }; } +function lineStarts(source: string): number[] { + const starts = [0]; + for (let offset = 0; offset < source.length; offset++) { + if (source[offset] === "\n") starts.push(offset + 1); + } + return starts; +} + +function lineAt(starts: readonly number[], offset: number): number { + let lo = 0; + let hi = starts.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (starts[mid]! <= offset) lo = mid; + else hi = mid - 1; + } + return lo + 1; +} + +/** Build a cheap old-line to current-line mapper for semantically identical + * sources. C emission records only a location's line, not its byte offset, so + * representative semantic tokens anchor each old line across trivia edits. */ +export function createSourceLineRebaser( + path: string, + previous: string, + current: string, +): (line: number) => number { + const oldTokens = semanticTokens(path, previous); + const newTokens = semanticTokens(path, current); + if (oldTokens === null || newTokens === null || !tokensEqual(oldTokens, newTokens)) { + return (line) => line; + } + const oldStarts = lineStarts(previous); + const newStarts = lineStarts(current); + const firstTokenByLine = new Map(); + for (let index = 0; index < oldTokens.length; index++) { + const line = lineAt(oldStarts, oldTokens[index]!.start); + if (!firstTokenByLine.has(line)) firstTokenByLine.set(line, index); + } + const mapper = offsetMapper(path, previous, current); + return (line): number => { + if (!Number.isSafeInteger(line) || line < 1 || line > oldStarts.length) return line; + const tokenIndex = firstTokenByLine.get(line); + const offset = tokenIndex === undefined + ? mapper.start(oldStarts[line - 1]!) + : newTokens[tokenIndex]!.start; + return lineAt(newStarts, offset); + }; +} + /** Rebase every SrcLoc-shaped object in a deserialized cache payload. */ export function rebaseSourceLocations( value: T, diff --git a/packages/compiler/test/library-identity-emission.test.ts b/packages/compiler/test/library-identity-emission.test.ts index 67346f21..8e3aaa11 100644 --- a/packages/compiler/test/library-identity-emission.test.ts +++ b/packages/compiler/test/library-identity-emission.test.ts @@ -1,8 +1,9 @@ import { describe, expect, test } from "vitest"; import { emitModule } from "../src/index.js"; import { emitLlvmModule } from "../src/backend/llvm/emitter.js"; -import { stripLibraryIdentity } from "../src/backend/library-identity.js"; +import { rebaseLibrarySourceComments, replaceLibraryIdentity, stripLibraryIdentity, stripLibrarySourceComments } from "../src/backend/library-identity.js"; import type { IrModule } from "../src/ir/nodes.js"; +import { rebaseSourceLocations, createSourceLineRebaser } from "../src/library/semantic-source.js"; import { fibModule } from "./fixtures/fib-ir.js"; const libraryModule = (): IrModule => ({ @@ -62,4 +63,74 @@ describe("library identity emission", () => { emitLlvmModule(mod, { emitLibraryIdentity: false }), ); }); + + test("cached public TUs refresh only their identity region", () => { + const mod = libraryModule(); + const identity = { + ...mod.lib!.identity!, + buildId: "0123456789abcdef", + }; + const expected = libraryModule(); + expected.lib!.identity = identity; + expect(replaceLibraryIdentity(emitModule(mod), "c", identity)).toBe(emitModule(expected)); + expect(replaceLibraryIdentity(emitLlvmModule(mod), "llvm", identity)).toBe(emitLlvmModule(expected)); + }); + + test("C source-line annotations can be refreshed and removed", () => { + const sourceFile = "/tmp/entry[1].ts"; + const emitted = [ + `value(); /* ${sourceFile}:3 */`, + `const char *text = " /* ${sourceFile}:5 */";`, + `other(); /* ${sourceFile}:9 */`, + "", + ].join("\n"); + expect(rebaseLibrarySourceComments(emitted, sourceFile, (line) => line + 4)).toBe( + [ + `value(); /* ${sourceFile}:7 */`, + `const char *text = " /* ${sourceFile}:5 */";`, + `other(); /* ${sourceFile}:13 */`, + "", + ].join("\n"), + ); + expect(stripLibrarySourceComments(emitted, sourceFile)).toBe( + `value();\nconst char *text = " /* ${sourceFile}:5 */";\nother();\n`, + ); + }); + + test("cached C annotations match a fresh emission after a comment edit", () => { + const before = "// old note\nfunction fib() {\n return 1;\n}\n"; + const after = "/* longer\n * replacement note\n */\n\nfunction fib() {\n return 1;\n}\n"; + // JSON round-tripping gives every SrcLoc its own object, matching + // deserialized semantic-cache payloads rather than this fixture's shared + // hand-written `loc` constant. + const cachedMod = JSON.parse(JSON.stringify(libraryModule())) as IrModule; + const oldOffset = before.indexOf("return"); + const setLocations = (value: unknown): void => { + if (value === null || typeof value !== "object") return; + const record = value as Record; + if ( + record["file"] === "fib.ts" && + typeof record["start"] === "number" && + typeof record["end"] === "number" + ) { + record["start"] = oldOffset; + record["end"] = oldOffset + "return".length; + return; + } + Object.values(record).forEach(setLocations); + }; + setLocations(cachedMod); + const currentMod = structuredClone(cachedMod); + rebaseSourceLocations( + currentMod, + new Map([["fib.ts", before]]), + new Map([["fib.ts", after]]), + ); + const restored = rebaseLibrarySourceComments( + emitModule(cachedMod, before), + "fib.ts", + createSourceLineRebaser("fib.ts", before, after), + ); + expect(restored).toBe(emitModule(currentMod, after)); + }); }); From e346c71872baa61090925379c49eb97b97f3b27d Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 20 Aug 2026 00:35:35 -0500 Subject: [PATCH 2/4] Fix cached library source annotations --- packages/cli/test/library-output.test.ts | 29 ++++++++-- .../compiler/src/backend/library-identity.ts | 10 +++- .../compiler/src/library/early-cache.test.ts | 53 ++++++++++++++++++- packages/compiler/src/library/early-cache.ts | 8 +++ .../test/library-identity-emission.test.ts | 4 +- 5 files changed, 97 insertions(+), 7 deletions(-) diff --git a/packages/cli/test/library-output.test.ts b/packages/cli/test/library-output.test.ts index 0a9fa0ef..ab446784 100644 --- a/packages/cli/test/library-output.test.ts +++ b/packages/cli/test/library-output.test.ts @@ -20,10 +20,17 @@ test("library identity source stays private and cannot overwrite a sidecar", asy const profilePath = join(dir, "profile.json"); try { await mkdir(cacheRoot, { mode: 0o700 }); + await writeFile(join(dir, "helper.ts"), [ + "export function initialValue(): number {", + " return 1;", + "}", + "", + ].join("\n")); await writeFile(join(dir, "lib.ts"), [ + "import { initialValue } from \"./helper.js\";", "export interface Model { value: number; }", "export type Msg = { kind: \"noop\" } | { kind: \"set\"; value: number };", - "export function init(): Model { return { value: 1 }; }", + "export function init(): Model { return { value: initialValue() }; }", "export function update(model: Model, msg: Msg): Model {", " return msg.kind === \"set\" ? { value: msg.value } : model;", "}", @@ -58,10 +65,13 @@ test("library identity source stays private and cannot overwrite a sidecar", asy }, }, null, 2)}\n`, ); - const runBuild = async (): Promise => { + const runBuild = async (keepC = false): Promise => { await execFileAsync( process.execPath, - ["--import", tsxLoader, cliEntry, "build", "--lib", "--profile", profilePath, "--no-keep-c"], + [ + "--import", tsxLoader, cliEntry, "build", "--lib", "--profile", profilePath, + ...(keepC ? [] : ["--no-keep-c"]), + ], { env: { ...process.env, @@ -88,6 +98,19 @@ test("library identity source stays private and cannot overwrite a sidecar", asy await runBuild(); expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); + // Imported trivia is semantically unchanged too, but the cached C text + // cannot be line-rebased through the entry-only annotation table. That + // shape must take the normal frontend path and match a forced cache miss. + await writeFile(join(dir, "helper.ts"), [ + "// harmless helper comment", + await readFile(join(dir, "helper.ts"), "utf8"), + ].join("\n")); + await runBuild(true); + const fallbackC = await readFile(join(outDir, "lib.lib.c"), "utf8"); + await rm(join(cacheRoot, "early-lib"), { recursive: true, force: true }); + await runBuild(true); + expect(await readFile(join(outDir, "lib.lib.c"), "utf8")).toBe(fallbackC); + // This name collided with the former fixed `.lib.identity.c` // output. Repeat to exercise the exact early-cache-hit ordering that used // to restore JSON and then overwrite it with generated C. diff --git a/packages/compiler/src/backend/library-identity.ts b/packages/compiler/src/backend/library-identity.ts index 7a74217b..8872dc5d 100644 --- a/packages/compiler/src/backend/library-identity.ts +++ b/packages/compiler/src/backend/library-identity.ts @@ -91,8 +91,14 @@ function escapedRegExp(text: string): string { function sourceCommentPattern(sourceFile: string): RegExp { // Emission appends these comments at physical line ends. Requiring that // boundary prevents source-looking text inside a generated C string from - // being mistaken for an annotation and changing program semantics. - return new RegExp(` /\\* ${escapedRegExp(sourceFile)}:(\\d+) \\*/(?=\\r?\\n|$)`, "g"); + // being mistaken for an annotation and changing program semantics. An + // uninitialized reference declaration appends its own `/* let ... */` + // explanation after the location, so admit that one generated suffix too. + return new RegExp( + ` /\\* ${escapedRegExp(sourceFile)}:(\\d+) \\*/` + + `(?= /\\* let [^\\r\\n]*; \\*/(?=\\r?\\n|$)|\\r?\\n|$)`, + "g", + ); } /** Refresh the source-line annotations retained in a caller-visible C TU. */ diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts index a949f0c3..aa09a660 100644 --- a/packages/compiler/src/library/early-cache.test.ts +++ b/packages/compiler/src/library/early-cache.test.ts @@ -1,6 +1,6 @@ import { mkdir, mkdtemp, readFile, readdir, rm, stat, utimes, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { afterEach, expect, test } from "vitest"; import type { IrModule } from "../ir/nodes.js"; import { FrontendInputTracker, trackedAccessibleEntries, trackedDirectoryExists, trackedFileExists, trackedReadFile } from "../frontend/input-tracker.js"; @@ -342,6 +342,57 @@ test("semantic library cache refuses token and directive edits", async () => { expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); }); +test("semantic C cache refuses comment-only edits outside the entry", async () => { + const f = await fixture(); + const imported = join(dirname(f.source), "helper.ts"); + const entrySource = await readFile(f.source, "utf8"); + const importedSource = "export function helper(): number { return 1; }\n"; + await writeFile(imported, importedSource); + const semanticMod = { + irVersion: 6, + sourceFile: f.source, + functions: [{ + name: "__main", + params: [], + returnType: { kind: "void" }, + locals: [], + body: [], + loc: { file: f.source, start: 0, end: entrySource.length }, + }], + entry: "__main", + } satisfies IrModule; + const tracker = new FrontendInputTracker(); + tracker.run(() => { + trackedReadFile(f.source); + trackedReadFile(imported); + }); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "c", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + semantic: { + mod: semanticMod, + sources: new Map([[f.source, entrySource], [imported, importedSource]]), + }, + }); + + await writeFile(imported, `// inserted comment\n${importedSource}`); + expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); +}); + test("early library cache is separated by the host Node version", async () => { const f = await fixture(); const tracker = new FrontendInputTracker(); diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index 832db5e9..b1371f36 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -410,6 +410,14 @@ export async function readSemanticLibraryCache( ), ); if (semantic === null || semantic.changed.length === 0) return null; + // C source annotations are rendered through the entry source's line + // table, including locations originating in imported modules. Rebasing + // cached imported locations therefore cannot reproduce a fresh lowering + // exactly; take the normal frontend path for that uncommon edit shape. + if ( + stamp.native.backend === "c" && + semantic.changed.some((change) => change.path !== resolve(options.entryPath)) + ) return null; const mod = deserializeV8(irJson) as IrModule; if (mod.irVersion !== IR_VERSION) return null; rebaseSourceLocations(mod, previousSources, semantic.currentSources); diff --git a/packages/compiler/test/library-identity-emission.test.ts b/packages/compiler/test/library-identity-emission.test.ts index 8e3aaa11..b6f20a0e 100644 --- a/packages/compiler/test/library-identity-emission.test.ts +++ b/packages/compiler/test/library-identity-emission.test.ts @@ -81,6 +81,7 @@ describe("library identity emission", () => { const emitted = [ `value(); /* ${sourceFile}:3 */`, `const char *text = " /* ${sourceFile}:5 */";`, + `slot = NULL; /* ${sourceFile}:7 */ /* let slot; */`, `other(); /* ${sourceFile}:9 */`, "", ].join("\n"); @@ -88,12 +89,13 @@ describe("library identity emission", () => { [ `value(); /* ${sourceFile}:7 */`, `const char *text = " /* ${sourceFile}:5 */";`, + `slot = NULL; /* ${sourceFile}:11 */ /* let slot; */`, `other(); /* ${sourceFile}:13 */`, "", ].join("\n"), ); expect(stripLibrarySourceComments(emitted, sourceFile)).toBe( - `value();\nconst char *text = " /* ${sourceFile}:5 */";\nother();\n`, + `value();\nconst char *text = " /* ${sourceFile}:5 */";\nslot = NULL; /* let slot; */\nother();\n`, ); }); From bd85e5405fee7f70240d8d8132b222d17f53608c Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 20 Aug 2026 01:27:51 -0500 Subject: [PATCH 3/4] Fix cached C source annotations --- packages/cli/test/library-output.test.ts | 45 ++++++++++++---- .../compiler/src/library/early-cache.test.ts | 51 ++++++++++++++++++- packages/compiler/src/library/early-cache.ts | 24 +++++---- .../src/library/semantic-source.test.ts | 8 ++- .../compiler/src/library/semantic-source.ts | 19 +++++++ 5 files changed, 125 insertions(+), 22 deletions(-) diff --git a/packages/cli/test/library-output.test.ts b/packages/cli/test/library-output.test.ts index ab446784..2dd70f72 100644 --- a/packages/cli/test/library-output.test.ts +++ b/packages/cli/test/library-output.test.ts @@ -27,10 +27,9 @@ test("library identity source stays private and cannot overwrite a sidecar", asy "", ].join("\n")); await writeFile(join(dir, "lib.ts"), [ - "import { initialValue } from \"./helper.js\";", "export interface Model { value: number; }", "export type Msg = { kind: \"noop\" } | { kind: \"set\"; value: number };", - "export function init(): Model { return { value: initialValue() }; }", + "export function init(): Model { return { value: 1 }; }", "export function update(model: Model, msg: Msg): Model {", " return msg.kind === \"set\" ? { value: msg.value } : model;", "}", @@ -89,18 +88,44 @@ test("library identity source stays private and cannot overwrite a sidecar", asy await runBuild(); expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); - // A comment-only edit takes the semantic cache path: restore the emitted - // TU, refresh its private identity object, and still honor --no-keep-c. + // A single-source comment-only edit takes the semantic cache path. Its + // restored public TU must match a forced miss before --no-keep-c removes + // it again. + await writeFile( + join(dir, "lib.ts"), + `/* harmless rebuild comment */ ${await readFile(join(dir, "lib.ts"), "utf8")}`, + ); + await runBuild(true); + const semanticHitC = await readFile(join(outDir, "lib.lib.c"), "utf8"); + await rm(join(cacheRoot, "early-lib"), { recursive: true, force: true }); + await runBuild(true); + expect(await readFile(join(outDir, "lib.lib.c"), "utf8")).toBe(semanticHitC); + await runBuild(); + expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); + + // A line-shifting edit cannot safely reuse line-only annotations (not even + // synthetic byte-zero locations). It must match a forced frontend miss. await writeFile(join(dir, "lib.ts"), [ - "// harmless rebuild comment", + "// line-shifting rebuild comment", await readFile(join(dir, "lib.ts"), "utf8"), ].join("\n")); - await runBuild(); - expect((await readdir(outDir)).sort()).toEqual(["contract.json", "lib.lib.a"]); + await runBuild(true); + const shiftedC = await readFile(join(outDir, "lib.lib.c"), "utf8"); + await rm(join(cacheRoot, "early-lib"), { recursive: true, force: true }); + await runBuild(true); + expect(await readFile(join(outDir, "lib.lib.c"), "utf8")).toBe(shiftedC); - // Imported trivia is semantically unchanged too, but the cached C text - // cannot be line-rebased through the entry-only annotation table. That - // shape must take the normal frontend path and match a forced cache miss. + // Move to a multi-source graph and seed its cache. Imported trivia is + // semantically unchanged too, but cached C annotations cannot be rebased + // through the entry-only line table. That shape must take the normal + // frontend path and match a forced cache miss. + await writeFile(join(dir, "lib.ts"), (await readFile(join(dir, "lib.ts"), "utf8")) + .replace( + "export interface Model", + "import { initialValue } from \"./helper.js\";\nexport interface Model", + ) + .replace("value: 1", "value: initialValue()")); + await runBuild(true); await writeFile(join(dir, "helper.ts"), [ "// harmless helper comment", await readFile(join(dir, "helper.ts"), "utf8"), diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts index aa09a660..a7b39ed6 100644 --- a/packages/compiler/src/library/early-cache.test.ts +++ b/packages/compiler/src/library/early-cache.test.ts @@ -342,7 +342,51 @@ test("semantic library cache refuses token and directive edits", async () => { expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); }); -test("semantic C cache refuses comment-only edits outside the entry", async () => { +test("semantic C cache accepts only line-preserving single-source edits", async () => { + const f = await fixture(); + const sourceBefore = await readFile(f.source, "utf8"); + const semanticMod = { + irVersion: 6, + sourceFile: f.source, + functions: [{ + name: "__main", + params: [], + returnType: { kind: "void" }, + locals: [], + body: [], + loc: { file: f.source, start: 0, end: sourceBefore.length }, + }], + entry: "__main", + } satisfies IrModule; + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "c", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + semantic: { mod: semanticMod, sources: new Map([[f.source, sourceBefore]]) }, + }); + + await writeFile(f.source, `/* harmless */ ${sourceBefore}`); + expect(await readSemanticLibraryCache(f.root, f.options, null)).not.toBeNull(); + await writeFile(f.source, `// inserted line\n${sourceBefore}`); + expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); +}); + +test("semantic C cache refuses comment-only edits in multi-source graphs", async () => { const f = await fixture(); const imported = join(dirname(f.source), "helper.ts"); const entrySource = await readFile(f.source, "utf8"); @@ -389,7 +433,10 @@ test("semantic C cache refuses comment-only edits outside the entry", async () = }, }); - await writeFile(imported, `// inserted comment\n${importedSource}`); + await writeFile(f.source, `// inserted entry comment\n${entrySource}`); + expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); + await writeFile(f.source, entrySource); + await writeFile(imported, `// inserted imported comment\n${importedSource}`); expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); }); diff --git a/packages/compiler/src/library/early-cache.ts b/packages/compiler/src/library/early-cache.ts index b1371f36..99a8035f 100644 --- a/packages/compiler/src/library/early-cache.ts +++ b/packages/compiler/src/library/early-cache.ts @@ -9,7 +9,7 @@ import { compilerReleaseVersion } from "./sidecar.js"; import type { IrModule } from "../ir/nodes.js"; import { IR_VERSION } from "../ir/serialize.js"; import { frontendInputsSemanticallyMatch, frontendInputsStillMatch, validFrontendInputSnapshot, type FrontendInputExclusions, type FrontendInputSnapshot } from "../frontend/input-tracker.js"; -import { rebaseSourceLocations, semanticallyEqualSource } from "./semantic-source.js"; +import { rebaseSourceLocations, semanticallyEqualSource, sourceLineRebaseIsIdentity } from "./semantic-source.js"; const gzipAsync = promisify(gzip); const gunzipAsync = promisify(gunzip); @@ -410,14 +410,20 @@ export async function readSemanticLibraryCache( ), ); if (semantic === null || semantic.changed.length === 0) return null; - // C source annotations are rendered through the entry source's line - // table, including locations originating in imported modules. Rebasing - // cached imported locations therefore cannot reproduce a fresh lowering - // exactly; take the normal frontend path for that uncommon edit shape. - if ( - stamp.native.backend === "c" && - semantic.changed.some((change) => change.path !== resolve(options.entryPath)) - ) return null; + // C source annotations are rendered through the entry source's line table, + // including imported offsets stamped with the entry path and synthetic + // byte-zero locations. Their line-only text cannot be rebased exactly for + // multi-source graphs or line-shifting edits. Keep TU reuse to the safe + // single-source, line-preserving subset; take the normal frontend path for + // the other uncommon trivia edits. + if (stamp.native.backend === "c") { + const entry = resolve(options.entryPath); + const change = semantic.changed.find((candidate) => candidate.path === entry); + if ( + previousSources.size > 1 || semantic.changed.length !== 1 || change === undefined || + !sourceLineRebaseIsIdentity(entry, change.previous, change.current) + ) return null; + } const mod = deserializeV8(irJson) as IrModule; if (mod.irVersion !== IR_VERSION) return null; rebaseSourceLocations(mod, previousSources, semantic.currentSources); diff --git a/packages/compiler/src/library/semantic-source.test.ts b/packages/compiler/src/library/semantic-source.test.ts index 66316580..187ace65 100644 --- a/packages/compiler/src/library/semantic-source.test.ts +++ b/packages/compiler/src/library/semantic-source.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "vitest"; -import { createSourceLineRebaser, rebaseSourceLocations, semanticallyEqualSource, semanticSourceDigest } from "./semantic-source.js"; +import { createSourceLineRebaser, rebaseSourceLocations, semanticallyEqualSource, semanticSourceDigest, sourceLineRebaseIsIdentity } from "./semantic-source.js"; test("ordinary TypeScript comments and formatting preserve semantic identity", () => { const before = "// old note\nexport function value(): number { return 1; }\n"; @@ -85,6 +85,12 @@ test("source lines rebase across changed comment trivia", () => { expect(rebase(2)).toBe(5); expect(rebase(3)).toBe(6); expect(rebase(4)).toBe(7); + expect(sourceLineRebaseIsIdentity("/entry.ts", before, after)).toBe(false); + expect(sourceLineRebaseIsIdentity( + "/entry.ts", + "// old\nexport function value() { return 1; }\n", + "/* replacement */\nexport function value() { return 1; }\n", + )).toBe(true); }); test("source locations at adjacent token boundaries rebase past inserted comments", () => { diff --git a/packages/compiler/src/library/semantic-source.ts b/packages/compiler/src/library/semantic-source.ts index c8b0090d..777dfdc6 100644 --- a/packages/compiler/src/library/semantic-source.ts +++ b/packages/compiler/src/library/semantic-source.ts @@ -204,6 +204,25 @@ export function createSourceLineRebaser( }; } +/** True when every existing source line keeps its physical line number. This + * is the safe subset for reusing a C TU whose annotations retain line numbers + * but not enough provenance to distinguish synthetic byte-zero locations. */ +export function sourceLineRebaseIsIdentity( + path: string, + previous: string, + current: string, +): boolean { + const rebase = createSourceLineRebaser(path, previous, current); + let line = 1; + if (rebase(line) !== line) return false; + for (let offset = 0; offset < previous.length; offset++) { + if (previous[offset] !== "\n") continue; + line++; + if (rebase(line) !== line) return false; + } + return true; +} + /** Rebase every SrcLoc-shaped object in a deserialized cache payload. */ export function rebaseSourceLocations( value: T, From 15039d0c613ecf7894bd950e5c30f1f1b5bfb937 Mon Sep 17 00:00:00 2001 From: Chris Tate Date: Thu, 20 Aug 2026 01:46:09 -0500 Subject: [PATCH 4/4] Fix cached C newline annotations --- .../compiler/src/library/early-cache.test.ts | 43 +++++++++++++++++++ .../src/library/semantic-source.test.ts | 9 ++++ .../compiler/src/library/semantic-source.ts | 18 ++++++++ 3 files changed, 70 insertions(+) diff --git a/packages/compiler/src/library/early-cache.test.ts b/packages/compiler/src/library/early-cache.test.ts index a7b39ed6..f1d9ba0d 100644 --- a/packages/compiler/src/library/early-cache.test.ts +++ b/packages/compiler/src/library/early-cache.test.ts @@ -386,6 +386,49 @@ test("semantic C cache accepts only line-preserving single-source edits", async expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); }); +test("semantic C cache refuses non-LF separator normalization", async () => { + const f = await fixture(); + for (const separator of ["\r", "\u2028", "\u2029"]) { + const sourceBefore = [ + "export function first(): number { return 1; }", + "export function value(): number { return 2; }\n", + ].join(separator); + await writeFile(f.source, sourceBefore); + const tracker = new FrontendInputTracker(); + tracker.run(() => trackedReadFile(f.source)); + await publishEarlyLibraryCache(f.root, f.options, { + cPath: f.cPath, + irPath: f.irPath, + sidecarPath: f.sidecarPath, + native: { + backend: "c", + regex: false, + assert: false, + inspect: false, + symbol: false, + searchParams: false, + emitter: false, + zlib: false, + copying: false, + textDecoderLegacy: false, + }, + frontend: tracker.snapshot(), + semantic: { + mod: { + irVersion: 6, + sourceFile: f.source, + functions: [], + entry: "__main", + }, + sources: new Map([[f.source, sourceBefore]]), + }, + }); + + await writeFile(f.source, sourceBefore.replace(separator, "\n")); + expect(await readSemanticLibraryCache(f.root, f.options, null)).toBeNull(); + } +}); + test("semantic C cache refuses comment-only edits in multi-source graphs", async () => { const f = await fixture(); const imported = join(dirname(f.source), "helper.ts"); diff --git a/packages/compiler/src/library/semantic-source.test.ts b/packages/compiler/src/library/semantic-source.test.ts index 187ace65..cdadf6b2 100644 --- a/packages/compiler/src/library/semantic-source.test.ts +++ b/packages/compiler/src/library/semantic-source.test.ts @@ -93,6 +93,15 @@ test("source lines rebase across changed comment trivia", () => { )).toBe(true); }); +test("C source lines reject normalization of non-LF separators", () => { + for (const separator of ["\r", "\u2028", "\u2029"]) { + const before = `export const first = 1;${separator}export const second = 2;\n`; + const after = before.replace(separator, "\n"); + expect(semanticallyEqualSource("/entry.ts", before, after)).toBe(true); + expect(sourceLineRebaseIsIdentity("/entry.ts", before, after)).toBe(false); + } +}); + test("source locations at adjacent token boundaries rebase past inserted comments", () => { const before = "export function value() { return left+right; }\n"; const after = "export function value() { return left+/* note */right; }\n"; diff --git a/packages/compiler/src/library/semantic-source.ts b/packages/compiler/src/library/semantic-source.ts index 777dfdc6..7955f4be 100644 --- a/packages/compiler/src/library/semantic-source.ts +++ b/packages/compiler/src/library/semantic-source.ts @@ -212,6 +212,24 @@ export function sourceLineRebaseIsIdentity( previous: string, current: string, ): boolean { + const oldTokens = semanticTokens(path, previous); + const newTokens = semanticTokens(path, current); + if (oldTokens === null || newTokens === null || !tokensEqual(oldTokens, newTokens)) { + return false; + } + const oldStarts = lineStarts(previous); + const newStarts = lineStarts(current); + // The C emitter counts only LF bytes. TypeScript also treats bare CR and + // the Unicode separators as line breaks, so normalizing one of those to LF + // preserves semantic tokens while moving later annotations to a different + // emitter line. Check every corresponding token, including multiple + // TypeScript lines that the emitter previously collapsed into one. + for (let index = 0; index < oldTokens.length; index++) { + if ( + lineAt(oldStarts, oldTokens[index]!.start) !== + lineAt(newStarts, newTokens[index]!.start) + ) return false; + } const rebase = createSourceLineRebaser(path, previous, current); let line = 1; if (rebase(line) !== line) return false;